Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-04 19:28:46 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge inventory parent and preserve unresolved sandbox creation work

Browse Source 
Carry the reviewed inventory/master parent with all history proofs. Keep pending and needs_reconciliation sandbox creation obligations blocking under the same reason. Prove actual owner cleanup and retained manual uncertainty, with scoped fixture teardown after worker settlement.

Preserve original RED, cross-suite isolation failure, and corrective qualification evidence. Native startup cancellation history remains conservative.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-02 17:12:02 -06:00
2 parents d2b86225f1 4a9974ea03
commit bbab3b924a
314 files changed
+10950 -2971
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/chat-catalog-testops-case.yml
+318
View File
@@ -0,0 +1,318 @@
name: Chat catalog TestOps case
on:
workflow_call:
inputs:
protocol:
type: string
required: true
cell:
type: string
required: true
target_sha:
type: string
required: true
runner:
type: string
required: true
selection_artifact_id:
type: string
required: true
selection_sha256:
type: string
required: true
build_artifact_id:
type: string
required: true
ALLURE_JOB_RUN_ID:
type: string
required: true
secrets:
OPENROUTER_API_KEY:
required: false
ANTHROPIC_API_KEY:
required: false
ALLURE_TOKEN:
required: false
permissions:
contents: read
actions: read
env:
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
jobs:
chat:
name: Execute the original selected case
runs-on: ${{ inputs.runner }}
timeout-minutes: ${{ fromJSON(inputs.cell).timeout_minutes }}
environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
env: &retention
TARGET_SHA: ${{ inputs.target_sha }}
MANUAL_PRIVATE_IDS: ${{ fromJSON(inputs.cell).private_ids }}
MANUAL_PRIVATE_APPROVALS_JSON: ${{ fromJSON(inputs.cell).private_policy }}
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify the exact reusable protocol and main-only caller
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
CHAT_TESTOPS_PROTOCOL: ${{ inputs.protocol }}
run: |
test "$CHAT_TESTOPS_PROTOCOL" = streaming-v1
node tests/runner-e2e/chat-completion-testops-cli.mjs guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ inputs.build_artifact_id }}
path: chat-completion-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ inputs.selection_artifact_id }}
path: chat-completion-testops-selection
- name: Verify source-bound build and authenticated selection before credentials
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
(cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
- name: Qualify the pinned provider runtime and preinstalled browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Bind exact private evidence under the authenticated TestOps authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only this cell's exact-key storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify private custody before delivering the model key
env: &cell_session
SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the selected original case with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
ANTHROPIC_API_KEY: ${{ contains(fromJSON(inputs.cell).required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}
PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).executionId }}
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id "${{ fromJSON(inputs.cell).executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain original evidence after success or failure
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and build the exact safe TestOps projection
needs: chat
if: always() && !cancelled() && (needs.chat.result == 'success' || needs.chat.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 20
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
outputs:
bundle_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ inputs.selection_artifact_id }}
path: chat-completion-testops-selection
- *pnpm
- name: Install trusted reporting dependencies with lifecycle scripts disabled
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
- name: Prepare unchanged exact-key report and cell policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain exact first-cell reader
id: reader_0
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the first exact original
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Verify every selected original and its completeness receipt
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing report while preserving failed source grades
id: normalize
continue-on-error: true
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ fromJSON(inputs.cell).private_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Export only exact reviewed synthetic fixture results
id: export
if: always() && steps.reader_0.outcome == 'success'
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
- name: Require a complete safe projection even after a failed normalization
if: always()
env:
EXPORT_OUTCOME: ${{ steps.export.outcome }}
run: test "$EXPORT_OUTCOME" = success
- name: Freeze one safe Allure bundle before any TestOps transmission
id: prepare
if: always() && !cancelled() && steps.export.outcome == 'success'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure
run: |
mkdir -m 700 "$RUNNER_TEMP/chat-catalog-case-bundle"
cp -R runner-e2e-testops "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
node tests/runner-e2e/chat-completion-testops-cli.mjs prepare-case-upload "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
cp "$RUNNER_TEMP/chat-completion-testops-prepare-case-upload/prepared.json" "$RUNNER_TEMP/chat-catalog-case-bundle/prepared.json"
- name: Retain the immutable metadata-only case bundle
id: bundle
if: always() && !cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-catalog-case-bundle-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }}
path: ${{ runner.temp }}/chat-catalog-case-bundle/
retention-days: 30
if-no-files-found: error
upload:
name: Publish only this original case without closing the shared job run
needs: report
if: always() && !cancelled() && (needs.report.result == 'success' || needs.report.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ inputs.selection_artifact_id }}
path: chat-completion-testops-selection
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.report.outputs.bundle_id }}
path: ${{ runner.temp }}/chat-catalog-case-bundle
- name: Admit the frozen singleton and submit it at most once
id: publication
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
CHAT_TESTOPS_PREPARED_FILE: ${{ runner.temp }}/chat-catalog-case-bundle/prepared.json
ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure
run: node tests/runner-e2e/chat-completion-testops-cli.mjs upload-case "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
- name: Retain only the closed singleton publication receipt
if: always() && !cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-catalog-case-publication-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }}
path: |
${{ runner.temp }}/chat-completion-testops-upload-case/publication.json
${{ runner.temp }}/chat-completion-testops-upload-case/failure.json
${{ runner.temp }}/chat-completion-testops-upload-case/once-upload_case.json
retention-days: 30
if-no-files-found: error
.github/workflows/chat-catalog-testops.yml
+61 -210
View File
@@ -23,7 +23,7 @@ permissions:
actions: read
concurrency:
group: chat-catalog-testops
group: chat-catalog-testops-streaming-v1
cancel-in-progress: false
env:
@@ -78,8 +78,7 @@ jobs:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
- if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
@@ -204,107 +203,24 @@ jobs:
fail-fast: false
max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}
matrix: ${{ fromJSON(needs.select.outputs.native_matrix) }}
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: ${{ matrix.timeout_minutes }}
environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
env: &retention
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
MANUAL_PRIVATE_IDS: ${{ matrix.private_ids }}
MANUAL_PRIVATE_APPROVALS_JSON: ${{ matrix.private_policy }}
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
steps: &catalog_worker_steps
- *checkout
- *node
- *guard
- *pnpm
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-completion-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- name: Verify source-bound build and authenticated selection before credentials
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
(cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
- name: Qualify the pinned provider runtime and preinstalled browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Bind exact private evidence under the authenticated TestOps authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only this cell's exact-key storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify private custody before delivering the model key
env: &cell_session
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the selected original case with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}
PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }}
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain original evidence after success or failure
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
uses: ./.github/workflows/chat-catalog-testops-case.yml
with:
protocol: streaming-v1
cell: ${{ toJSON(matrix) }}
target_sha: ${{ needs.authorize.outputs.target_sha }}
runner: ${{ needs.authorize.outputs.runner }}
selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
build_artifact_id: ${{ needs.build.outputs.artifact_id }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
secrets:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
chat_external:
name: ${{ matrix.executionId }}
@@ -314,102 +230,29 @@ jobs:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: ${{ matrix.timeout_minutes }}
environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps: *catalog_worker_steps
report:
name: Retain private originals and build the exact safe TestOps projection
needs: [authorize, select, build, chat, chat_external]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 20
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
strategy:
fail-fast: false
max-parallel: 4
matrix: ${{ fromJSON(needs.select.outputs.matrix) }}
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies with lifecycle scripts disabled
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
- name: Prepare unchanged exact-key report and cell policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain exact first-cell reader
id: reader_0
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the first exact original
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Verify every selected original and its completeness receipt
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing report while preserving failed source grades
id: normalize
continue-on-error: true
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ matrix.private_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Export only exact reviewed synthetic fixture results
id: export
if: always() && steps.reader_0.outcome == 'success'
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
- name: Require a complete safe projection even after a failed normalization
if: always()
env:
EXPORT_OUTCOME: ${{ steps.export.outcome }}
run: test "$EXPORT_OUTCOME" = success
- name: Retain the metadata-only TestOps projection
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-catalog-testops-cell-${{ github.run_id }}-${{ matrix.key }}
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
uses: ./.github/workflows/chat-catalog-testops-case.yml
with:
protocol: streaming-v1
cell: ${{ toJSON(matrix) }}
target_sha: ${{ needs.authorize.outputs.target_sha }}
runner: ${{ needs.authorize.outputs.runner }}
selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
build_artifact_id: ${{ needs.build.outputs.artifact_id }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
secrets:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
aggregate:
name: Validate and merge every selected metadata-only projection
needs: [authorize, select, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && (needs.report.result == 'success' || needs.report.result == 'failure')
needs: [authorize, select, build, chat, chat_external]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
@@ -429,8 +272,15 @@ jobs:
path: chat-completion-testops-selection
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: chat-catalog-testops-cell-${{ github.run_id }}-*
path: catalog-projections
pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
path: catalog-bundles
- name: Gather only safe singleton projections from immutable bundles
run: |
mkdir -m 700 catalog-projections
for bundle in catalog-bundles/*; do
test -d "$bundle/projection"
cp -R "$bundle/projection" "catalog-projections/${bundle##*/}"
done
- name: Authenticate selection and validate all singleton exports before aggregation
env:
GH_TOKEN: ${{ github.token }}
@@ -449,9 +299,9 @@ jobs:
retention-days: 30
if-no-files-found: error
upload:
name: Record and independently verify each TestOps result
needs: [authorize, select, chat, chat_external, report, aggregate]
verify:
name: Verify final coverage without uploading any result again
needs: [authorize, select, aggregate]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.aggregate.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -468,10 +318,17 @@ jobs:
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- *allure
- name: Upload original outcomes, checks and private-evidence receipts
id: allure_upload
env: &upload_env
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
path: catalog-bundles
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
pattern: chat-catalog-case-publication-${{ github.run_id }}-1-*
path: catalog-publications
- name: Authenticate every frozen publication and verify the exact original selection
id: verification
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
@@ -479,20 +336,14 @@ jobs:
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results
run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops
- name: Verify actual selected results and complete result-owned attachments
id: verification
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *upload_env
run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops
- name: Retain closed TestOps verification receipts
if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure')
CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles
CHAT_TESTOPS_PUBLICATIONS: ${{ github.workspace }}/catalog-publications
run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-stream runner-e2e-testops
- name: Retain closed final coverage verification
if: always() && !cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-completion-testops-verify-upload/
${{ runner.temp }}/chat-completion-testops-upload/exported.json
path: ${{ runner.temp }}/chat-completion-testops-verify-stream/
retention-days: 30
if-no-files-found: error
.github/workflows/chat-completion-testops.yml
-504
View File
@@ -1,504 +0,0 @@
name: Chat completion TestOps
run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Run the exact TestOps selection, register its plan, or launch it once
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps job-run identity; source, model and clocks are fixed
ALLURE_USERNAME:
type: string
required: false
description: Reserved service metadata; never grants authority
permissions:
contents: read
actions: read
concurrency:
group: chat-completion-testops
cancel-in-progress: false
env:
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
operation: ${{ steps.guard.outputs.operation }}
target_sha: ${{ steps.guard.outputs.target_sha }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify actual main-only dispatch and both actors
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard
select:
name: Register the dedicated plan or validate its real TestOps selection
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-testops-reporting
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
execution_ids: ${{ steps.selection.outputs.execution_ids }}
matrix: ${{ steps.selection.outputs.matrix }}
policy: ${{ steps.selection.outputs.policy }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register three exact cases with no synthetic test result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap
node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context
- name: Validate case, plan, job, launch and same-run source before provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results
CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }}
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection
- name: Request the one reviewed plan launch and exit
if: needs.authorize.outputs.operation == 'launch-plan'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }}
CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }}
CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }}
run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan
- name: Retain token-free registration and launch receipts
if: always() && needs.authorize.outputs.operation != 'evaluate'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-completion-testops-bootstrap/
${{ runner.temp }}/chat-completion-testops-launch-plan/
retention-days: 30
if-no-files-found: error
- name: Retain the exact authenticated selection
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-selection-${{ github.run_id }}-1
path: testops-pilot-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable catalog source without service or model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- name: Resolve the fixed candidate graph with lifecycle scripts disabled
run: |
pnpm install --lockfile-only --ignore-scripts
pnpm install --frozen-lockfile --ignore-scripts
- name: Build existing TypeScript and native Runner outputs
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
- name: Exercise the real workflow and pinned source host contract before model access
run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs
- name: Build native Runner outputs
run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package the resolved lock and exact source outputs
env:
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$TARGET_SHA"
tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-build-${{ github.run_id }}-1
path: |
chat-completion-build.tar.gz
chat-completion-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: ${{ matrix.executionId }}
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
strategy:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJSON(needs.select.outputs.matrix) }}
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
env: &retention
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }}
MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }}
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
steps:
- *checkout
- *node
- *guard
- *pnpm
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-completion-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- name: Verify source-bound build and authenticated selection before credentials
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
(cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
- name: Qualify the pinned provider runtime and preinstalled browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Bind exact private evidence under the authenticated TestOps authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only this cell's exact-key storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify private custody before delivering the model key
env: &cell_session
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the selected original case with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }}
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain original evidence after success or failure
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and build the exact safe TestOps projection
needs: [authorize, select, build, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 20
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies with lifecycle scripts disabled
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
- name: Prepare unchanged exact-key report and cell policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact first-cell reader
id: reader_0
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the first exact original
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Obtain exact second-cell reader
id: reader_1
if: fromJSON(steps.session.outputs.cell_count) > 1
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_1 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the second exact original
if: steps.reader_1.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '1'
AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Obtain exact third-cell reader
id: reader_2
if: fromJSON(steps.session.outputs.cell_count) > 2
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_2 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the third exact original
if: steps.reader_2.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '2'
AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Verify every selected original and its completeness receipt
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing report while preserving failed source grades
id: normalize
continue-on-error: true
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only exact reviewed synthetic fixture results
id: export
if: always() && steps.private_report.outcome == 'success'
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
- name: Require a complete safe projection even after a failed normalization
if: always()
env:
EXPORT_OUTCOME: ${{ steps.export.outcome }}
run: test "$EXPORT_OUTCOME" = success
- name: Retain the metadata-only TestOps projection
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-projection-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Record and independently verify each TestOps result
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: chat-completion-testops-projection-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- *allure
- name: Upload original outcomes, checks and private-evidence receipts
id: allure_upload
env: &upload_env
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results
run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops
- name: Verify actual selected results and complete result-owned attachments
id: verification
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *upload_env
run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops
- name: Retain closed TestOps verification receipts
if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-completion-testops-verify-upload/
${{ runner.temp }}/chat-completion-testops-upload/exported.json
retention-days: 30
if-no-files-found: error
.github/workflows/chat-feature-testops.yml
-451
View File
@@ -1,451 +0,0 @@
name: Chat feature TestOps
run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Fixed evaluation, zero-result registration, or one TestOps plan launch
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps supplies its job-run identity; no test/source selector is accepted
ALLURE_USERNAME:
type: string
required: false
description: Reserved TestOps service metadata; never used as authorization
permissions:
contents: read
concurrency:
group: chat-feature-testops
cancel-in-progress: false
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.guard.outputs.sha }}
operation: ${{ steps.guard.outputs.operation }}
triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify fixed default-source dispatch and both actor identities
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
NODE_OPTIONS: ''
NODE_PATH: ''
run: node tests/runner-e2e/testops-chat-dispatch.ts
select:
name: Validate TestOps selection or register zero-result setup
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
sha256: ${{ steps.selection.outputs.sha256 }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register the public job without an evaluation or a synthetic result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap
node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context
- name: Validate the one real TestOps selection before any build or provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection
- name: Start exactly the validated TestOps plan and exit without waiting for its child
if: needs.authorize.outputs.operation == 'launch-plan'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}
CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
GH_TOKEN: ${{ github.token }}
run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan
- name: Retain only closed metadata from zero-result setup
if: always() && needs.authorize.outputs.operation == 'bootstrap'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json
${{ runner.temp }}/chat-testops-bootstrap-state/once.json
${{ runner.temp }}/chat-testops-bootstrap-state/plan.json
${{ runner.temp }}/chat-testops-bootstrap-state/job.json
${{ runner.temp }}/chat-testops-bootstrap-state/failure.json
${{ runner.temp }}/chat-testops-bootstrap-state/registration.json
${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json
retention-days: 30
if-no-files-found: error
- name: Retain the one plan-launch receipt and stop state
if: always() && needs.authorize.outputs.operation == 'launch-plan'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-plan-launch-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-launch-plan-state/once.json
${{ runner.temp }}/chat-testops-launch-plan-state/launch.json
${{ runner.temp }}/chat-testops-launch-plan-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain only the token-free exact selection receipt
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-selection-${{ github.run_id }}-1
path: testops-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable shipped native candidate without model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- name: Install only the source-reviewed graph without lifecycle scripts
run: |
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
pnpm install --frozen-lockfile --ignore-scripts
- name: Build the existing TypeScript and native runner owners
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package source-bound outputs and resolved lock
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1
path: |
chat-testops-build.tar.gz
chat-testops-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: One real native OpenCode backlog request on AWS
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
id-token: write
env: &retention
PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1
TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TARGET_SHA: ${{ needs.authorize.outputs.sha }}
MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Check out the same admitted source for trusted retention
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-testops-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- name: Verify immutable outputs and token-free TestOps selection
env:
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
(cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256)
tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE"
node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json
cmp chat-testops-lock.json chat-testops-worker-lock.json
cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store")
- name: Materialize and qualify pinned OpenCode runtime and browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Prepare private evidence with shipped-main authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only the exact private cell storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify strict private custody before the model key
env: &cell_session
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the fixed original request with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain exact original grade and private evidence, including failures
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and export only reviewed fixture results
needs: [authorize, select, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies only
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store"
- name: Prepare exact private report and cell-read policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact private cell reader
id: read_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the exact private original and verify all inventories
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: |
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing normalized campaign without exposing original logs
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately even when its grade failed
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only the bound normalized result, JUnit and marked PNGs
id: export
if: always() && steps.private_report.outcome == 'success'
env:
TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts
- name: Upload only the sanitized TestOps projection
if: always() && steps.export.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Report the original outcome to its exact TestOps case
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- *allure
- name: Verify and upload only the bound safe case outcome
id: allure_upload
env: &testops_reporter
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results
run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops
- name: Verify the actual TestOps result after upload processing
id: verify_upload
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *testops_reporter
run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops
- name: Retain only closed actual TestOps verification metadata
if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-verify-upload-state/verification.json
${{ runner.temp }}/chat-testops-verify-upload-state/failure.json
retention-days: 30
if-no-files-found: error
DESIGN.md
+2 -2
View File
@@ -117,8 +117,8 @@ completes without a provider reply. Preserve the reply through persistence.
Task chat shows execution errors and waits only while they remain relevant.
Exhausted attempts use a padded, rounded notice with a destructive-tinted background,
an error icon, and foreground text. The surrounding layout supplies separation from
the conversation. Automatic retry countdowns keep their quieter inline styling.
an error icon, and foreground text. A wrapper adds top padding above the tinted
surface. Automatic retry countdowns keep their quieter inline styling.
Completing or cancelling a task hides its old execution notices. A newer attempt by
the same agent or an explicit successor supersedes earlier run notices; an
unresolved execution hold remains visible. Historical turns keep their responses,
cli/vitest.config.ts
+2
View File
@@ -3,5 +3,7 @@ import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
environment: "node",
hookTimeout: 30_000,
teardownTimeout: 30_000,
},
});
doc/DATABASE.md
+59 -19
View File
@@ -522,23 +522,59 @@ leave database construction unchanged. An expired window stays inactive after a
restart; remove the setting after the investigation.
The diagnostic measures awaited Postgres.js `unsafe` responses before Drizzle
column decoding. It retains the Drizzle builder caller for normal, prepared,
transaction, and dedicated-pool execution. It issues no queries.
It logs the public server caller locations, static schema table, operation,
query/row counts, and response value sizes. It emits at most 2,048 aggregate
groups and a summary. Measurement stops at the deadline. Environment-enabled
column decoding. It issues no queries. Configuration is read when each database
client is created; setting the environment on an already-running process does
not install the capture on existing clients. Prepare the deployment artifact
before choosing the UTC window, then start the window after rollout and warm-up.
Server entry points use `withDatabaseOperation(owner, action)` to name the work
that executes queries. Owners are fixed semantic labels, never request paths,
company or run IDs, user input, or SQL. The execution owner is captured for each
query and retained through completion, including transactions and dedicated
pools. A prepared query reused by different operations keeps the owner of each
execution. The builder caller is separate from that execution owner.
Owner context is active while a valid future window is armed, up to 30 minutes
before its start, so work that crosses the start boundary keeps its owner. Query
accounting starts only at the window start. With no armed window, or after
expiry, the owner helper calls the action directly without a context scope.
The capture aggregates owner, static schema table, operation, calls, returned
rows, query errors, and total/maximum completion wall time. `totalWallMs` and
`maxWallMs` include pool, database, network, and event-loop waits; they are not
CPU measurements, and concurrent query durations overlap. Timing excludes the
capture's response sizing. Rank calls and returned rows both per second and per
completed service operation; a busy owner is not necessarily wasteful.
Response sizing samples successful results 1, 17, 33, and so on per aggregate
group (`sampleEvery: 16`). It examines at most
256 cells, 65,536 string code units, and 65,536 value bytes per sampled result;
an individual string over 16,384 code units is not measured. Parsed objects and
other complex values are not serialized. Oversized, complex, or unexamined
values leave an incomplete sample. `sampledValueBytes` is a lower bound over
sampled primitive values, not a total response-size estimate or network bytes.
Do not extrapolate it as the total bytes returned by the database. Inspect
`sampledResults`, `sampledRows`, `measuredValues`, `unmeasuredValues`, and
`incompleteSamples` alongside it. `sizingWallMs` records the diagnostic's sizing
work separately. Use a separate capture-disabled CPU profile to measure normal
processing cost.
The diagnostic emits at most 2,048 aggregate groups and a summary. Measurement
stops at the deadline. Environment-enabled
captures emit at most 64 records per second and can take another 33 seconds to
finish writing the bounded output. Wait for the summary before collecting it.
It never logs SQL, parameters, result
values, connection strings, or query errors. In-flight queries continue normally.
For a bundled server, keep the linked source maps and enable Node's
`--enable-source-maps` option so the diagnostic can resolve public caller
locations. Restore the prior Node options after capture. Sizes still count when
caller locations are unavailable.
Semantic owners work in a bundled server without source maps. For queries
without an explicit owner, keep the linked source maps and enable Node's
`--enable-source-maps` option to resolve public caller locations. Restore prior
Node options after capture. Known semantic owners avoid repeated stack capture.
Queries without an available owner/caller still contribute to the totals.
Look for `paperclip.db_response_capture.entry.v1` and
`paperclip.db_response_capture.summary.v1` in the process logs. A log provider
Look for `paperclip.db_response_capture.entry.v2` and
`paperclip.db_response_capture.summary.v2` in the process logs. Version 2 uses
sampled, bounded primitive-value sizing; do not interpret it as the version 1
all-result byte estimate. A log provider
can store `event` as a structured field instead of message text. Filter by that
field or read the bounded window from the exact deployment. Check the summary
and group totals, including the unattributed overflow group.
@@ -552,22 +588,26 @@ Fetch a complete capture as follows:
from zero through `groupPages - 1`, filtering on the entry event, `captureId`,
and `groupPage`. Request at least `groupPageSize` matching records per page.
A structured query can use
`@event:paperclip.db_response_capture.entry.v1 AND @captureId:<id> AND @groupPage:0`.
`@event:paperclip.db_response_capture.entry.v2 AND @captureId:<id> AND @groupPage:0`.
3. If the provider's record limit is smaller than a page, fetch each
`groupIndex` separately with the same capture ID. Each index matches one
entry. Do not split by timestamp: entries can share a timestamp.
4. Deduplicate by `(captureId, groupIndex)`. Require every index from zero
through `groups - 1`, with no gaps. Entry count must equal `groups`, summed
`calls` must equal `completedQueries`, and summed `responseValueBytes` must
equal the summary total. A missing page is an incomplete measurement.
`calls` must equal `completedQueries`, and summed `rows`, `sampledResults`,
and `sampledValueBytes` must equal their summary totals. A missing page is an
incomplete measurement. Keep overflow and unattributed groups visible.
Tagged raw-driver calls, streams, unawaited raw-driver
dispatch, and protocol overhead are not measured. Relational queries use a driver-level caller fallback.
Queries still in flight at the deadline are reported separately. Sizes describe
driver values before column decoding, not billed network bytes. Complex parsed
values use JSON size estimates, counted in `estimatedValues`. Reconcile them with
provider metering and socket measurements. Serialization adds temporary CPU and memory work;
measure the final transfer rate after the window expires.
Queries still in flight at the deadline are reported separately. The census does
not include every PostgreSQL statement or internal transaction command, so its
calls are not interchangeable with database-wide commit counters. It does not
measure SQL compilation, driver parsing CPU, or Drizzle row-mapping CPU. Use the
ranked owners to choose a narrow follow-up profile or synchronous-stage
measurement; do not attribute CPU by subtracting process CPU usage across an
asynchronous query. Remove the capture setting after collecting its final
summary; the absolute deadline already prevents further measurement.
### Push the schema
doc/DEVELOPING.md
+71 -1
View File
@@ -1058,8 +1058,55 @@ For normal issue work, start with the smallest targeted check that proves the ch
#### Embedded Postgres template cache
To run the suite with one owned PostgreSQL server and a separate disposable
database for each fixture, use:
```sh
node cli/node_modules/tsx/dist/cli.mjs packages/db/src/test-postgres-command.ts
node cli/node_modules/tsx/dist/cli.mjs packages/db/src/test-postgres-command.ts -- pnpm exec vitest run --project @paperclipai/db
```
The command starts an embedded server, prepares a migrated database template,
passes it to the child command, and waits for server shutdown before returning.
It preserves the child command's exit status. On cancellation it stops and waits
for the test process group before shutting down PostgreSQL. Database, server,
and CLI fixtures all use the common implementation. Tests of embedded startup itself continue to
start dedicated clusters.
To reuse a disposable server that you already manage, set
`PAPERCLIP_TEST_POSTGRES_ADMIN_URL` to its administrative PostgreSQL URL. The role
must be able to create databases. Each fixture creates a random database and
closes only its clients before dropping it. It never stops the external server.
`DATABASE_URL` does not select this test path. The optional
`PAPERCLIP_TEST_POSTGRES_TEMPLATE` can name an existing migrated database with
the form `paperclip_test_template_<hex>`; `0` disables template reuse.
On a supported platform, PostgreSQL startup failures fail test collection.
Resource exhaustion, unavailable configured servers, and broken native binaries
do not make integration tests skip. A genuinely unsupported platform or CPU can
still skip embedded PostgreSQL coverage. Startup retries apply only to port
collisions. Teardown waits for process exit, requests fast shutdown first, and
uses immediate PostgreSQL shutdown if needed. A failed stop fails the hook and
retains its data directory. Worker termination also requests owned server
shutdown. A forced `SIGKILL` can bypass these handlers; the separate host cleanup
job remains the backstop for stale macOS semaphore sets.
Database and CLI hooks use the same 30-second deadline as server hooks. This
allows the ten-second shutdown protocol and client/database cleanup to finish
before the test runner ends a worker. A shutdown timeout still fails the hook.
Template publication uses `pg_ctl` to request clean fast shutdown on every
supported platform and allows thirty seconds for it. A template is never
published after immediate shutdown or a crash. Cancelling the shared-server
command also waits for its owned cluster-directory cleanup after the test
processes and PostgreSQL exit. Direct worker termination stops PostgreSQL but
retains its disposable files for inspection; the synchronous exit guard cannot
await directory removal.
Stopped cluster directories are removed asynchronously, and cleanup awaits their
removal. Large suites can leave many relation files; deleting them must not block
the worker's event loop. The idle-sleep suite uses the existing 90-second fixture
budget for its final directory cleanup after hundreds of table truncations.
Suites that call `startEmbeddedPostgresTestDatabase` (from `@paperclipai/db`)
get their own embedded Postgres cluster. The first such suite on a host builds
without an administrative URL get their own embedded Postgres cluster. The first such suite on a host builds
one fully migrated cluster and publishes its data directory under
`$TMPDIR/paperclip-embedded-postgres-templates-<uid>/<key>`. Every later suite
copies that template and starts Postgres on the copy instead of running
@@ -1669,6 +1716,10 @@ marker is gone. Denied permissions and object locks keep a retry obligation.
The sandbox holds current working files. PostgreSQL records its location,
writer epoch, and committed snapshot; encrypted immutable snapshots live in S3.
S3 multipart uploads stream up to four parts concurrently. One producer reads
the source in order, and bounded stream buffers preserve backpressure. Failed
uploads cancel and settle their active requests before multipart cleanup.
Snapshot publication still requires full object hash and size verification.
Preparation creates a saved baseline, and native finalization saves the stopped
writer's files before releasing its persistence obligation. Replacing a
webserver reconnects to the recorded allocation without copying files back to
@@ -1677,6 +1728,15 @@ host, sandbox, or Git remote. Files changed since the last save still depend on
the live sandbox: losing that allocation can lose those edits. There is no
periodic workspace backup in this release.
Files reads use the current committed saved version when remote authority is
clean and has no active writer, even while its old allocation remains recorded.
The saved loader verifies the company, workspace, snapshot, and archive before
reading. Dirty or writer-owned authority uses the live allocation and cannot
substitute an older save if that allocation is unavailable. Reads retain their
authority-change fence and remove private saved-file copies after use.
Embedders that omit the optional saved-file loader keep valid live reads. They
still report unavailability when neither a live location nor a loader exists.
Existing local task workspaces require an explicit import. Run from a source
checkout with the deployment's database, S3, and key configuration loaded:
@@ -2621,6 +2681,16 @@ connections, including provider-native tools. Company permissions, approval
gates, and workspace isolation still apply. Explicit restrictive modes remain
restrictive; omitted settings use full auto.
Assigned app tools can use `paperclip_search_assigned_tools` and
`paperclip_call_assigned_tool` instead of direct tool definitions. Search with
`{"query":"github"}` and call an exact returned tool name. Omitted, empty or
whitespace-only `schemaTool` values perform normal discovery, including when
`schemaOffset` is zero. To read a large schema, set `schemaTool` to the exact
`inputSchemaRef` from discovery and follow `nextSchemaOffset`. A positive
`schemaOffset` requires a nonblank reference. Unknown or revoked references
return guidance to search again; discovery still checks current grants and work
modes against the session's pinned catalog.
Test questions, saved plan revisions, approval before task handoff, status
lookups, and `/new` preserving chat history. Hiring additionally requires the
operator-controlled [runner API tools](runner-api-tools.md) rollout; enabling
doc/PRODUCT.md
+13 -2
View File
@@ -97,6 +97,11 @@ When an agent creates a subtask or follow-up task, its source task's feed shows
a linked card at creation time with the current title, task number, and status.
The Tasks tab keeps the full relationship list.
Tasks waiting between automatic checks show **Monitoring** with a blue clock.
Task lists, board columns, filters, search, recent navigation and task properties
use the same read-only display state. The task retains its execution lifecycle
status, and the monitor banner shows when the next check is due.
Automatic monitor checks pause when a task is blocked. A person can use Check now
to request one check of a retained monitor through normal execution admission.
Dependencies, recovery, pause, budgets and ownership still apply. Provider-usage
@@ -141,9 +146,15 @@ choice stays selected while task data refreshes.
The task Filters menu also offers **Routine tasks**. It selects tasks with
`originKind=routine_execution`, including manually triggered routine runs, even
when other automatic tasks are hidden. The routines list's **View all tasks**
link opens `/issues?view=all&originKind=routine_execution` with that checkbox
link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox
selected. The URL retains the selection across reloads and task navigation.
Search and pagination apply the routine origin filter on the server.
Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides
saved collection sorting, and the routines link opens newest-created tasks first.
Shared sort links work across Coordinator, Streamlined UI, and Classic UI, including
sort fields without a visible column in the selected interface. Search relevance
and board-column membership keep their existing server order.
Search and pagination apply the routine origin filter on the server. Creation-time
sorting also runs before pagination, with task ID as a stable tie-breaker.
Tasks and project lists hide agent-created and system-created tasks by default.
The Filters menu's **Show agent-created tasks** checkbox includes routine runs,
doc/SPEC-implementation.md
+10 -2
View File
@@ -423,6 +423,7 @@ retain their titles and default to no generation request.
- `title` text not null
- `description` text null
- `status` enum: `backlog | todo | in_progress | in_review | done | blocked | cancelled`
- Task reads additionally expose read-only `displayStatus`: the lifecycle status, or `monitoring` for an agent-owned scheduled ordinary monitor wait with no active execution, locks, review participant or active recovery action. Board collection/search `displayStatus` filters and counts use the same projection before pagination. This field is not persisted and cannot authorize a status transition; existing `status` filters retain their lifecycle semantics.
- `priority` enum: `critical | high | medium | low`
- `review_policy` nullable enum: `anyone | not_creator | human_only`; null is equivalent to `anyone`
- `assignee_agent_id` uuid fk `agents.id` null
@@ -2712,9 +2713,16 @@ its upstream authors; operators can pause updates to review changes manually.
The task Filters menu also offers **Routine tasks**. It selects tasks with
`originKind=routine_execution`, including manually triggered routine runs, even
when other automatic tasks are hidden. The routines list's **View all tasks**
link opens `/issues?view=all&originKind=routine_execution` with that checkbox
link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox
selected. The URL retains the selection across reloads and task navigation.
Search and pagination apply the routine origin filter on the server.
Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides
saved collection sorting, and the routines link opens newest-created tasks first.
Shared sort links retain their order across task interfaces. Workflow sorting
requests the existing authorized dependency-edge projection. Search and board
queries keep their existing server order. Blocked-attention lists reject
`sortField=created` with `422` because they use their own activity order.
Search and pagination apply the routine origin filter on the server. Creation-time
sorting also runs before pagination, with task ID as a stable tie-breaker.
Tasks and project collections default to hiding agent/system-created tasks and
provide a saved **Show agent-created tasks** filter. Classify creation using
doc/connections/AI-CONNECTIONS.md
+17
View File
@@ -98,6 +98,23 @@ grant through encrypted vault secret references, with existing consumer bindings
Safe provider-reported account identity is optional; secret references, tokens,
and authentication paths are never account labels.
Permissions shows account details separately from the connection's maintenance
controls: the provider account email and organization when available, the sign-in
method, and personal or company-shared access. A Paperclip owner name is not
evidence of the provider login. Anthropic setup tokens can permit inference while
denying profile access; in that case the screen explains that Anthropic did not
permit reading the account email. A profile denial does not mark a connected
credential as disconnected.
`GET /api/companies/:companyId/ai-connections/:connectionId/identity?grantId=...`
checks company membership, connection-management authority, and the selected
credential's human audience before opening the credential. The response contains
only safe account labels and bounded status messages. The Anthropic profile read
has a 15-second headers/body deadline and no redirects. It runs separately from
the account/grants reads, so profile loading or failure cannot block account
maintenance. The UI reuses an observation for five minutes, scoped to the company,
connection, grant, and grant update time. Reconnect requires a new observation.
Company-scoped `/api/companies/:companyId/ai-connections` operations provide list,
API-key creation/reconnect, personal defaults, completed login references, and
active-run attribution. The list includes `canManageConnections`, evaluated by
doc/evals.md
+3 -2
View File
@@ -57,8 +57,9 @@ OpenCode Product E2E cells. They
verify saving an unstarted task and delegating a saved announcement back to chat,
using ordinary production instructions, real browser messages and independent
state/provenance checks. A separately selected native OpenCode/DeepSeek1017
backlog cell measures TestOps integration acceptance with the same oracle and
original bounds. Its source registration is not an AWS or TestOps pass. The
backlog cell supplied historical TestOps integration acceptance with the same
oracle and original bounds. Its former smoke plan and the three-case completion
pilot are retired; their cases and measurements remain in history. The
[full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops)
maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and
fixtures use a preserved immutable catalog revision; trusted workflow helpers run
doc/execution-semantics.md
+104
View File
@@ -112,6 +112,22 @@ The issue is actively owned work.
For agent-owned issues, `in_progress` should not be allowed to become a silent dead state.
### Monitoring presentation
An agent-owned `in_progress` task displays **Monitoring** with a blue clock while
it has a persisted ordinary monitor deadline and no active run, execution or
checkout lock, review participant, or active recovery action. The persisted
status remains `in_progress`. Monitor admission, checkout, finalization, recovery,
and agent lifecycle filters keep their existing contracts. Review, blocked,
terminal, and provider-quota recovery states keep their own presentation.
Task reads expose optional, read-only `displayStatus`. Board task collections and
search can filter by `displayStatus`; filtering happens before pagination and
counts. Existing `status` filters retain their lifecycle meaning. Monitoring is
not a manually selectable task status or a board drag target. An overdue check
retains its visible deadline warning; the label does not attest that a provider
is running or that a check succeeded.
### `blocked`
The issue cannot proceed until something external changes.
@@ -186,6 +202,14 @@ and review run. The server reads these records from saved review state; it does
not depend on the parent session remembering a separate review session. These
records are evidence and do not grant permission to resolve another review.
An automatic chat-completion wake retains its exact delivery receipt when
admission waits on a dependency, budget, or pause. It does not merge into a
replaceable dependency signal. The same gates still prevent execution.
Workspace readiness accepts a completed operation enclosed by a successful
finalize from the same company, workspace, issue, and run. A nested snapshot
that finishes before that finalize finishes does not leave a permanent wait.
Unfinished operations and work after the finalize still require a later save.
### Answered Slack conversations
An accepted scheduled-monitor yield remains unfinished task work. Its committed
@@ -710,6 +734,20 @@ A workspace-coherent adapter path means:
- the effective cwd exists or is provider-reachable, according to the workspace provider
- when the adapter or workspace strategy relies on git state, the cwd is git-valid for the selected workspace: it resolves to the expected repository root, required base refs or branch metadata can be resolved, and runtime-created worktrees are still registered or explicitly recoverable
Fresh remote-workspace recovery fingerprints the admitted remote strategy. A
temporary strategy used to seed a controller directory must not become the saved
remote configuration. An unchanged follow-up reuses that configuration without
reporting strategy drift. Real strategy, source, mode and environment changes
retain their existing replacement checks.
Older remote Git workspaces can retain the temporary `project_primary`
fingerprint. That fingerprint alone cannot distinguish the old preparation bug
from a genuinely configured `project_primary` strategy; the remote workspace row
uses `git_worktree` for either Git source. Do not rewrite those fingerprints in
place or suppress their replacement checks. After saving and inspecting the
retained work, an explicitly admitted fresh recovery creates a workspace with the
correct fingerprint. Ordinary reuse of the old workspace does not migrate it.
Adapter-backed liveness also requires control-plane reachability from the agent's actual mutation surface, not just from the host adapter process. If the agent is expected to use Bash, shell tools, runtime helpers, or in-sandbox command execution to update issues, create comments, upload artifacts, or submit review decisions, the `PAPERCLIP_API_URL` and `PAPERCLIP_API_KEY` visible to that surface must route to Paperclip successfully.
For sandbox-backed local adapters, Paperclip may satisfy that contract with a run-scoped in-sandbox bridge. The host adapter keeps the real run JWT on the host side, injects only the bridge URL/token into the sandbox tool environment, and forwards allowed Paperclip API requests with the run id attached. The bridge credentials are execution plumbing, not user-facing context: they must not be written into prompts, visible comments, issue documents, restored workspace files, or durable logs. Agents and skills must use the env vars available in Bash/curl rather than assuming that the host's localhost API URL is reachable from browser or web-extraction tools inside the sandbox.
@@ -764,6 +802,15 @@ conflict detection. Legacy adapter snapshots remain inline. Snapshot-only worker
do not load the adapter process-launch helpers. The directory merge loads those
helpers when it resolves its instance-scoped lock root.
Portable remote workspace capture stages only self-contained Git metadata. It
streams working files into the archive without a second working-tree copy and
does not repack an already dissociated mirror. Each streamed file must match its
recorded hash, size and mode and retain its filesystem identity while read.
Capture rechecks the complete source inventory and Git state before returning a
receipt. A changed source or cancelled capture removes its incomplete archive;
an existing archive remains untouched. Temporary space must still hold the
normalized Git metadata and the complete archive together.
### Remote Codex model compatibility
Fresh remote Codex Runner runs check the selected image CLI before saving the
@@ -862,6 +909,15 @@ The task thread exposes the guarded Retry action for failed or timed-out convers
A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed.
A successful connected run can still have newer admitted input waiting behind it.
Task Retry resumes that input's existing durable chat receipt through the inbound
owner, instead of replaying the successful run as a failed attempt. The original
actor, source, result, and publication history remain intact. Duplicate clicks
follow one canonical receipt. Current board permission, task ownership, source
authorization, and prior execution cleanup are rechecked before normal admission;
pause, budget, approval, and dependency gates remain effective. Completion or
newer execution supersedes an unadmitted retry.
A valid recovery action must name:
- the source issue and company
@@ -1682,6 +1738,20 @@ sources without this proof remain ineligible. The preservation owner still
requires an independently stopped, exclusively bound allocation and no host writer;
it saves files and resumes the existing finalizer without another provider turn.
Both the legacy native copyback descriptor and the remote-workspace descriptor
keep an unfinished export under the execution hold. Task Retry saves its intent
behind that hold; an accepted successful result resumes its existing finalizer
after verified preservation instead of admitting another task run. Pending human
review remains effective when the result commits.
If a later failure supersedes the original recovery action, an explicit board
`recover-workspace` request can still preserve that exact historical export.
The cancelled action remains historical and the current action retains attention
ownership. Preservation checks current writers and source identity, records a
durable snapshot, and leaves the task's newer workspace binding unchanged. It
does not apply an older accepted result over a newer run or authorize continuation.
Cancelled history without a saved explicit recovery request remains ineligible.
Startup and the retained-workspace sweep reconstruct these obligations through
the retained workspace index. Committed Retry, recovery admission, and recovery
progress writes wake a separate preservation worker. It follows the persisted
@@ -1926,6 +1996,24 @@ That is the execution contract Paperclip should present to operators.
### Cancellation during native startup
Core supports verified cleanup for explicitly recorded startup obligations. This
support does not create startup contexts, advertise transport support, or capture
a transport binding. Existing callers retain their legacy payloads and startup
budgets. This support alone makes no startup-latency claim.
Restart recovery uses the claim breadcrumb only while the current run has no
first session checkpoint and the prior controller and context deadlines have
expired. A later preparation breadcrumb requires the stronger preparation-stop
proof. A failed cleanup-intent write retains the original typed failure and
rearms the existing restart retry after 30 seconds; the running run continues to
block idle sleep. Recovery queues cleanup rather than replaying a provider. A completed result
or a newer owner wins. The existing adopted-authentication and chat-workspace
ownership holds retain their separate behavior. The startup budget does not bound
the duration of remote cleanup. Existing Stop receipts and process cleanup still gate
destructive lifecycle or binding changes. Operator plugin restarts and external
machine changes can still interrupt startup.
Cancellation records a preparation fence while holding the run row lock. Native
runtime selection checks that fence, the running status, and the current startup
controller lease in the same transaction that creates the native coordinator.
@@ -1964,6 +2052,15 @@ preserving history, unknown action outcomes, and attempt counts. Pauses, approva
budgets, task ownership, and terminal task status still gate admission. No
automatic provider replay is authorized by a cancelled startup.
A failed cleanup attempt saves its next retry time in the existing control-work
deadlines. The startup marker retains the original admission deadline. Both
candidate batches use stable deadline order, and pending cleanup cannot enter
ordinary forced finalization. A late replacement-task reply remains owned until
its resource identity is durable. It is queued for cleanup instead of attached
after cancellation. The old task's Stop receipt cannot settle cleanup of a new
replacement task. A lost reply with no known resource identity remains unresolved
until provider inspection or operator reconciliation establishes its outcome.
### Delivering queued messages after a legacy run stops
The legacy queued-message Interrupt action accepts a null `targetRunId` when
@@ -2446,6 +2543,13 @@ accessible repository work products do not require an attachment. Publication
failure calls for continued work or a concrete blocker, not a human confirmation
that the task is complete.
Native completion reports can cite the relative task-document links returned by
the server. Each link must resolve to this company's current task and an attached
document whose current revision has a matching native publication receipt.
The same server-owned link projection supplies completion guidance and evidence
validation. A published document link can accompany a requested file, but it
does not replace that file's verified delivery evidence.
For an explicit file output in the current request, an empty report, a
verification-only reference, or an unregistered URL cannot satisfy delivery.
The report must cite an attachment verified by the current run's durable
doc/idle-sleep-safety.md
+20
View File
@@ -148,6 +148,15 @@ agent/company/user, waiting state, no checkout, active run, accounting, monitor
or pending delivery, and a finished wake for every accepted user-comment outbox
entry. An idle presentation state does not exempt an ordinary runnable task.
Sandbox creation records in `pending` or `needs_reconciliation` remain blockers,
including manual uncertainty after company deletion. The pending-only catalog
omission predates the new startup-cleanup recovery path; no complete-instance
unsafe sleep outcome has been demonstrated. A late identified resource transfers
its obligation to the exact cleanup lease, and only the owning successful cleanup
can resolve that creation. Settled startup cancellations without a committed
native result manifest remain conservatively blocked by native finalization;
their cleanup marker alone is not a history exemption.
Workspace runtime inspection also reads the owner's process-local resource
state. Startup reconstruction must succeed before that probe becomes quiet.
Adopted/live services, provisioning/control/replacement work, live allocation
@@ -224,6 +233,17 @@ fingerprint and RPC company scope for later read-only verification.
Retained allocations, unfinished runs/accounting/retries, export intents and
live setup credentials continue to block sleep.
For a reused remote allocation, a completed predecessor can resolve termination
through the final owner's destruction receipt only when the full host-recorded
lease handoff chain matches. Company, agent, task, workspace, environment, provider
and allocation identities must agree. A stopped successor, broken handoff or
unrelated owner grants no cleanup authority. This resolution does not rewrite
historical leases, execute remotely or claim that uncollected files were saved.
The normal agent-file recovery worker retains the no-save outcome and completes
cleanup after confirmed destruction. A completed cleanup receipt with a stale
retry deadline is reconciled once through the existing deadline worker. Clearing
that deadline compares the receipt so a concurrent cleanup obligation is kept.
Pending human-only connection setup can wait asleep when no active enabled
connection of the requested purpose exists. Available access reblocks sleep so
the existing reconciliation path can run; answering a setup card continues to
doc/mcp-discovery-performance.md
+67
View File
@@ -25,6 +25,41 @@ The stateless gateway and runtime-tools MCP endpoints return HTTP 405 with
The runtime-tools endpoint still validates its caller before that response.
See the [MCP transport contract](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports).
## New-tool review projection
Pending-tool review builds covered application and connection sets from each
profile's include rules. An included catalog entry covers its connection and its
non-null application only while that entry exists in the supplied catalog.
Explicit application selectors retain null application matching. A second index
records all exact selectors, including exclusions, so already reviewed tools do
not appear as new. The catalog filter retains its order, status check, strict
review watermark, and returned metadata.
The indexes live only for one projection. Profile listings, new-tool review, and
app attention still read current company-scoped database state on each request.
This replaces nested catalog-by-rule scans with linear indexing and filtering.
It changes neither database query counts nor response contracts.
The PostgreSQL reproduction uses 200 selected profiles, 1,900 active tools, and
76,000 rules. Profiles alternate between 65 and 695 rules. Half reference 695
removed catalog entries, while half select current tools. This exercises both
scope checks and explicit-review checks. It compares complete ordered results
from the real profile listing and app-attention services before timing them.
Save baseline `profiles/projection.ts`, `profiles/service.ts`, and
`catalog/attention.ts` beside their current versions as `.baseline-projection.ts`,
`.baseline-service.ts`, and `.baseline-attention.ts`. Change the two baseline
consumers' projection imports to `.baseline-projection.js` with the same relative
path. Then run from `server/`:
```sh
node --expose-gc --import tsx scripts/benchmark-profile-review-projection.ts --baseline-profiles "$PWD/src/services/tool-access/profiles/.baseline-service.ts" --baseline-attention "$PWD/src/services/tool-access/catalog/.baseline-attention.ts"
```
Remove the three baseline files after the comparison. The probe alternates five
samples per implementation and reports process CPU, elapsed time, actual query
count, and returned rows. No provider calls or hosted traffic are generated.
## Reproduce the measurements
Build workspace dependencies, then run from `server/`:
@@ -71,3 +106,35 @@ Catalog payload memory and policy-matching CPU still grow with the catalog.
The measurements do not establish a safe production heap limit or the share of
active installations affected. Verify restart bursts and sustained traffic on the
actual deployment before reducing its temporary heap allowance.
## Profile selector matching
Effective profile resolution and profile summaries index each profile's include
and exclude selectors once per decision. Catalog rows then use exact set
membership checks for application, connection, catalog entry, tool name, and
risk level. Exclusions still take precedence within a profile; effective profiles
still contribute the union of their allowed tools. Null application selectors
retain their exact-match behavior.
These indexes live only for the current computation. Each resolver call still
reads current company-scoped bindings, profiles, and rules. Query counts and
returned database rows are unchanged. This reduces JavaScript matching work when
a profile selects many individual tools; it does not cache authorization or
remove the runtime assignment digest check.
For a real PostgreSQL comparison, save `projection.ts` and `resolution.ts` from
the baseline commit beside their current versions as `.baseline-projection.ts`
and `.baseline-resolution.ts`. Change only the baseline resolver's projection
import to `./.baseline-projection.js`. Then run from `server/`:
```sh
node --import tsx scripts/benchmark-tool-profile-matching.ts --baseline "$PWD/src/services/tool-access/profiles/.baseline-resolution.ts"
```
Remove the two baseline files afterward. The probe uses 1,890 catalog tools,
1,575 individual include/exclude rules, one effective profile, and 12 agents. It
checks full assignment-result equality, warms both versions, and alternates
seven samples of ten resolutions. It reports actual query and returned-row
counts, process CPU, and elapsed time. The installation dependency returns an
empty list for this fixture, which has no installed connections. It does not
measure provider work, whole-run latency, or hosted performance.
doc/observability.md
+44
View File
@@ -1900,6 +1900,49 @@ name to `sandbox.daytona.<name>`. The host maps any other value to
daytona provider emits these spans today, so the segment is the literal
`daytona`.
### Daytona lease acquisition and resume
The worker invocation carries nested, monotonic timing receipts for lease work.
The host also admits the following fixed `lease.*` names and prefixes them with
`sandbox.daytona.`. These use the existing opt-in endpoint and host-owned parent
context; they do not create an exporter or first-party Telemetry events.
```text
execution.plugin_dispatch
└── plugin.worker
├── sandbox.daytona.lease.wait_for_teardown (only when teardown blocks resume)
├── sandbox.daytona.lease.lookup
├── sandbox.daytona.lease.refresh (missing-container confirmation)
├── sandbox.daytona.lease.channel_cleanup (before restart)
├── sandbox.daytona.lease.start (or lease.recover)
└── sandbox.daytona.lease.verify
├── sandbox.daytona.lease.workspace
├── sandbox.daytona.lease.sentinel
└── sandbox.daytona.lease.shell
```
`lease.lookup` includes cached-handle lookup and any required provider refresh.
`lease.start` and `lease.recover` measure the actual SDK operation, including its
internal readiness wait. They also cover starts triggered by execution or sync.
They do not separate SDK-internal requests or provider-side queue time.
Their `paperclip.sandbox.startup.state` attribute records the state before the
call: `started`, `stopped`, `archived`, `error`, or `other`. Both the plugin and
host restrict this value; no provider error text or resource identity is sent.
Fresh acquisition emits `lease.create`, `lease.workspace`, `lease.shell`,
`lease.expiry`, and `lease.sentinel`. `lease.create` covers the existing creation
helper, including qualification-only snapshot resolution and creation-failure
cleanup. Setup-failure deletion is a separate `lease.cleanup` sibling, so it
does not inherit a failed workspace or verification phase. Existing deadlines,
ownership, cleanup and retry behavior are unchanged. Failed awaited steps close
with error status; the receipt contains no exception message.
Children nest beneath the current `plugin.worker` receipt and verification
children nest beneath `lease.verify`. Never add parent and child durations.
Receipt budgets and clock validation still apply. Older workers without
`tracer.withSpan` keep the original behavior without these detail spans. Tracing
failures preserve the operation's result or error and never repeat it.
The `sandbox.daytona.*` spans use this closed attribute allowlist. The host
drops every other key, so a command, an argument, a path, an id, a standard
output, or a standard error never rides a provider span. The host records only
@@ -1908,6 +1951,7 @@ the attributes that the producer sends for one span.
| Attribute | Type | Optional | Meaning |
| ------------------------------------------------ | ------ | -------- | --------------------------------------------------------------------------------------------------------- |
| `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. |
| `paperclip.sandbox.startup.state` | string | yes | State before start/recovery: `started`, `stopped`, `archived`, `error`, or `other`. |
| `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). |
| `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.daytona.pack` span. |
| `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.daytona.transfer` span. |
doc/performance/native-backup-publication.md
+53
View File
@@ -0,0 +1,53 @@
# Native backup publication benchmark
Retiring a native harness backup can involve many immutable directories and
small files. Cleanup repairs directory permissions before removing the retired
tree. Publication waits for cleanup, preserves the previous backup when lease
stamping fails, and emits its completion notification after cleanup succeeds.
Cleanup uses asynchronous filesystem operations so unrelated requests can
continue while it runs. Directory preparation is serial and does not follow
symlinks into live trees.
Run the focused correctness tests from an installed checkout:
```sh
pnpm --filter @paperclipai/server exec vitest run src/services/native-runtime/native-harness-backup-cleanup.test.ts src/services/native-runtime/native-harness-backup-publication.test.ts
```
Run the performance comparison from the repository root with the Node version
used by the target server. Use a quiet machine and an explicit baseline commit:
```sh
node server/scripts/benchmark-native-backup-publication.mjs \
--baseline-ref 774ddc72713d6aa7cb516691c71f1fc23bcfa5ff \
--output /tmp/native-backup-publication.json
```
The script compiles the real publication service and its dependencies for both
the baseline commit and the working tree. Each run uses a separate server
process. Fixture preparation occurs before measurement. The default fixture has
100 immutable directories with 100 small files in each of the `current` and
`previous` backups: 20,000 files to remove. Both backups also have a symlink to a
read-only live directory outside the retired trees. The new pending backup has
a distinct manifest. Every completed run checks the new manifest, complete
removal of retired and pending trees, and unchanged live contents and mode.
A separate process sends requests to a small loopback HTTP endpoint every five
milliseconds. Arrivals use a fixed clock and include overdue requests. The
report includes request completion and error counts, generator lateness, p95
and maximum foreground latency, requests served during publication, publication
wall time, process CPU time, and maximum event-loop delay. Three pairs alternate
baseline and treatment order. Change the fixture with `--pairs`, `--directories`,
`--files-per-directory`, or `--interval-ms`.
The result records Node/platform, baseline SHA, treatment HEAD and uncommitted
service diff. Compiled modules and source maps stay in the printed evidence
directory. Successful fixtures are removed; failed fixtures remain for diagnosis.
This is a service benchmark. The HTTP endpoint is a responsiveness probe, not
the assembled Paperclip API. The lease-stamp callback resolves without database
work. It does not prove a hosted latency or total-CPU improvement. Process CPU
during publication includes foreground requests that the synchronous baseline
cannot serve until cleanup completes. Async cleanup can consume more CPU while
keeping the event loop responsive. Qualify the affected run/checkpoint/recovery
journey and compare matched hosted windows before claiming deployed acceptance.
doc/plans/2026-10-31-zero-downtime-web-deploys.md
+197 -6
View File
@@ -1,10 +1,201 @@
# Zero-downtime web deployments
Date: 2026-10-31. Status (2026-11-01 15:39 UTC): existing-volume staging enrollment
and manual sleep/wake preserve saved files and read/save behavior. The full local
active-agent journey now passes on `cc23019691`; hosted browser continuity and
the ten-deploy staging campaign remain unqualified. Natural database autosuspension
passes after abandoned test-resource cleanup. No production canary yet.
Current state (2026-11-02 18:31 UTC): cleanup recovery PR
[#18633](https://github.com/paperclipai/paperclip/pull/18633) merged as
`60a99307b3bb1062078d35af6677e3f377d8bab8`. Its final head passed all PR CI,
full local typecheck and build, and seven focused cleanup regressions. Greptile
reviewed that head at 5/5; both review threads are resolved. The duplicate full
local test run was stopped after six chat shards passed, while general suites
were still running. Its incomplete local result is preserved separately from
successful full PR CI.
Canonical Linux release verification failed on the unchanged two-second busy-row
startup test on several consecutive revisions, including the cleanup merge.
The held-row diagnostic completed in 420 ms on macOS without a database lock
wait. Inspection found that `currentNativeControllerIdentity` always called the
server-info snapshot, which synchronously probes Git three times even when the
OS process birth time is available. Controlled one-second delays in real Git
commands reproduced the exact startup assertion failure with real PostgreSQL.
The follow-up separates the constant fallback process birth time from diagnostic
Git metadata. New regressions cover both available and unavailable OS birth time.
The two-second startup limit and release gates remain unchanged. Linux release
verification must pass before the new source can enter hosted qualification.
The follow-up is [#18649](https://github.com/paperclipai/paperclip/pull/18649).
Its first head received two 5/5 reviews and passed every CI test shard except
an existing voice-retention precision test. That test advances its fake clock
automatically while asserting behavior across a one-millisecond boundary. A
controlled 60 ms database-entry delay reproduced the exact failure. Holding the
clock fixed makes the same assertion pass with that delay; the final correction
only disables automatic clock advancement in that one precision test. The
original CI failure and both controlled results are retained. No production
voice code or assertion was weakened.
The earlier ten-roll result remains evidence for its recorded images. Neither
the cleanup fix nor the controller-identity follow-up has reached the staging
fixture. Final acceptance still requires a qualified compatible image pair,
fresh staging replacements, natural idle and browser sleep/wake, followed by the
isolated production canary. Cloud source is `b44304603d`; its private packages
require comparison against the new Core SDK and shared archive. Earlier candidate
Core `42c64d4ec4` differs in SDK; `b9e94b85a6` differs in shared types. Neither is
a valid rollback pair for the cleanup merge. No release gate was bypassed.
The production browser was signed in as Dotta B on the ordinary new-organization
form. No organization was submitted and no production config changed. The only
hosted target changed so far is the pinned staging qualification fixture.
Disposable local PostgreSQL and Moto S3 fixtures passed an independent object
read/write preflight. Docker subsequently stopped; local container qualification
requires restoring that dependency. Hosted acceptance uses real object storage.
Earlier retained evidence and failure history follows.
Date: 2026-10-31. Status (2026-11-02 17:23 UTC): Core #18554 is merged as
`1a6017d6460267ad89d7e1784e2b521c28d8e9ac`. The ten-roll and native-agent
proof used Cloud `461c4aea28`. Separately reviewed Cloud #1091/#1092 subsequently
reached staging as `89dd0cfec2`; their changes concern workspace cleanup and
Agent Computer resource fencing. Railway, fleet and deployment workflow paths
are unchanged. Production remains on its original controller revision; the
next manual controller deployment must use current reviewed master.
Both immutable Cloud distributions built from that fixed source passed packaged
Linux rehearsals through the Railway wrapper: 260 reads, 53 saves, zero failures,
legacy-volume migration, an unready replacement, and removal of both predecessor
homes. Each fixture cleaned its containers. These rehearsals do not exercise an
active agent or an actual browser. The two distributions have distinct image
digests but identical application source; their compatible rollback qualifies
artifact replacement, not compatibility between different source revisions.
The final Core PR has two 5/5 reviews and all required checks passed. One legacy
browser CI shard failed because its deferred-wake fixture observed no queued wake;
the unchanged case passed locally on isolated PostgreSQL and its single CI retry
passed. The original failure remains in the PR evidence. The full local live-agent
journey passed on `cc23019691` before the final non-suspended dispatch refinement;
that refinement passed its three focused Linux regressions, server typecheck,
build and final CI. Do not describe the earlier live run as a rerun of the merge.
The fixed canonical package baseline passed on the pinned migrated staging fixture
using the allowed one-time maintenance path. An actual Chromium tab running the
older UI reconnected, retained an unsaved title edit, and saved it through the new
server; an independent API read confirmed the title. That old tab opened the
Agents page, and all four previously captured asset hashes still match after the
old deployment was removed. Baseline maintenance interruptions are preserved and
are not counted as zero-downtime qualification.
The first ten-roll attempt was cancelled at step one, before Railway replacement.
The controller's pre-existing hot-upgrade allowlist named an older test stack,
so it drained rather than negotiated preservation of the new fixture's active run.
Continuous read/save and browser probes remained healthy, but this is NOT active
execution continuity evidence. The original command completed normally once; its
downloaded output bytes and native terminal were independently verified. The
owned fixture has been added to the existing allowlist without changing fleet
defaults. The normal staging controller workflow is applying that correction.
The rollout harness now checks both canary policy and positive tenant readiness
before submitting a campaign. The cancelled attempt remains failed evidence,
with no automatic campaign retry.
The configuration deployment then failed before Railway submission when Sentry
release registration failed or timed out. Its source checks passed; the original
logs are retained. One unchanged whole-workflow retry preserves attempt-bound
preflight requirements. That retry failed at the same pre-Railway Sentry request.
A read-only Sentry lookup confirms the release exists (HTTP 200). The canary-only
configuration was then applied by redeploying the existing successful controller
deployment: the source and replacement image digests are identical, and health
attests unchanged Cloud `461c4aea28`. Both failed workflow attempts are retained;
no Sentry gate was disabled and no new application release was built in this
configuration recovery. The new active-command attempt and gated sequence are
starting with the corrected controller. The first preflight stopped before task
admission on a missing optional plan's 404. `useIssuePlanDocument` explicitly
maps that response to null; an independent read confirmed no plan on the owned
completed task. The raw response remains recorded and the browser gate permits
only that verified path. The configuration redeploy's actual browser reconnect
was 1.22 seconds, with no probe read/save failure.
All ten Railway replacements are independently qualified. Final continuous proof
passed 12,521 reads and 1,566 saves, with every acknowledged save present exactly
once and all 34 file hashes matching. There were zero HTTP failures. Actual
browser reconnects across the ten deployments ranged from 1.16 to 1.29 seconds.
The old UI assets and original agent artifact remain readable after deployment
ten and provider cleanup. Both traffic observers have stopped cleanly. The first
replaced the webserver while the original eight-minute remote command was
running. Its one execution ID started before replacement and completed afterward
with exit zero. The second replaced the server after completion; a later follow-up
retained the same native/provider session and runner, with accepted native
terminals and finalized accounting for both runs. The 91-byte artifact was
downloaded and independently checked for exactly one start and completion.
Actual browser reconnects were 1.28, 1.16 and 1.25 seconds; continuous reads and
saves have no failures so far. Previously unopened Projects and Skills pages rendered in the
retained old tab. A title draft survived the third cutover and was saved through
the UI, then independently read back from the replacement.
The harness stopped after deployment three on another owned task's expected
missing optional-plan 404. The UI explicitly treats this as null. Both exact task
paths were independently checked, and the provider deployment, package eligibility,
serving identity, removal of the predecessor, browser timing and continuous probe
were revalidated. The failed original receipt remains unchanged alongside an
explicit classification review. The remaining seven replacements continue without
redeploying or retrying deployment three. An announcement also obscured one
sidebar click; its ordinary Dismiss control resolved that retained interaction
timeout. A separate large run-log diagnostic timed out; a narrower public-events
read succeeded and proves the original command's single execution identity.
A later bounded 64-KiB log read succeeded from shared storage after both the
originating webserver and remote sandbox were removed.
The native recovery event records SIGTERM without a retry run, then adoption of
the same runner at controller generation 2. Each of the two runs has exactly one
accepted result and terminal event. The independent accounting inspector has no
findings or pending work; the task totals 52.941 cents across two runs. Including
the earlier no-replacement attempt, staging has three estimated cost events totaling
113.84284 cents. The temporary agent is paused, its secret removed and the prior
company budget restored; all three provider resources are independently confirmed absent. The first delete
was refused while the sandbox stopped; that failure remains alongside the
successful cleanup after a fresh stopped-state observation. Both owned local
PostgreSQL/object-storage containers are removed; test evidence is retained.
Deployment nine also completed, but the local test reader observed the browser
snapshot while its writer had truncated the file. The campaign response was
already parsed and successful; the only non-atomic JSON read in that failing
checkpoint is the local browser report. A concurrent-write regression reproduced
the problem. The reader now retries only local JSON syntax errors for at most
100 ms; persistent corruption still fails, and HTTP/acceptance failures are never
retried. Deployment nine was independently revalidated without redeployment.
Both raw failed sequence receipts and their explicit classification reviews remain;
the ten provider deployments themselves have not been retried. The final staging
driver stopped at the harness failure and is being resumed from the retained
sequence. The normal UI's 256,000-byte run-log read also passes after replacement.
The first repair check stopped before mutation because an unrelated staging
campaign was active. That campaign has finished, and the pinned canary remains
active and unlocked on the qualified final image. Both one-target configuration repair campaigns passed: backup policy was
restored, the temporary marker removed, and durable files/shared assets remained
enabled on the same volume-free image. The current-image eight-minute natural-idle check failed with all browser pages and
traffic probes closed. Bounded SQL diagnosis found three completed native runs
still in agent-file collection recovery. Normal archive of both completed test
workspaces independently passed close-readiness and preserved saved workspace
files. It produced provider-confirmed destruction receipts for the latest leases.
Two copies then completed no-save cleanup, but retained stale retry deadlines;
the earlier run of a reused sandbox still lacks direct termination evidence even
though its validated successor allocation is destroyed. These are source recovery
gaps, not an idle-proof pass. No historical rows were manually rewritten. The
failed idle and cancelled browser lifecycle receipts remain unchanged. Fix
ownership belongs to remote termination proof resolution and agent-directory
cleanup scheduling; no polling cadence or sleep-safety check should be relaxed.
The isolated source branch is now based on current master `97c4dc969e`. The
recovery fix resolves the destroyed allocation through the existing validated
handoff-chain contract at read time; it does not stamp historical leases or infer
stop from task status. Agent-file cleanup also reconciles stale retry deadlines
with a receipt comparison. Both owning service suites passed 169 tests, and the
server typecheck passed. The first new deadline test caught a public guard that
skipped settlement; that failed attempt is retained and the corrected five-case
selection passed. The final seven-case selection passed, including a concurrent receipt change and
cross-company rejection. Repository-wide checks and PR review are pending.
This source has not been deployed and is not covered by the prior ten-roll proof.
Next: review and ship the cleanup fix, complete current-image idle/sleep/wake, then the
isolated production forward/rollback canary. No production deployment has occurred.
Dotta completed the authorized GitHub sign-in; the production browser is back on
the ordinary organization list. The production canary still waits for staging. Its new organization form is
prepared but unsubmitted. Production remains on its original controller revision,
with no campaigns, the single-volume default and continuous deployment disabled. Earlier
staging sleep/wake and natural database-idle proofs remain valid for their image.
Core #18543 merged as `7751fc3107` after 55 successful CI checks, Greptile 5/5
and no unresolved threads. It supplies runtime admission, immutable-cache
@@ -202,7 +393,7 @@ Measure one-time setup separately from repeated attempts. Keep failed evidence.
| Item | State | Next action |
| --- | --- | --- |
| Prior prerequisites | #18403 and all five storage workstreams merged | Retain the ephemeral filesystem contract |
| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 reviewed at 5/5 with CI pending; local packaged migration, browser overlap, cold restoration and durable command proofs recorded below | Publish and qualify the exact fixed release after CI/review merge |
| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 merged with all required checks and 5/5 reviews; both exact fixed images passed packaged migration and deletion; active hosted output verified | Finish hosted continuation and accounting reconciliation |
| Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged; #1102 serving on staging | Verify repeated Railway overlap behavior after Core recovery qualification |
| Staging | Original enrollment complete; migrated data and signing key preserved; volume detached and retained | Sleep/wake and natural database idle pass; verify repair and ten app-only replacements with browser continuity |
| Production | No canary mutation performed | Proceed only after the staging and active-agent acceptance gates pass |
maps/agent-access-map/app-permissions.md
+13 -1
View File
@@ -30,7 +30,10 @@ Review: `operation-reviewed`.
API: `present` — `GET /api/tool-connections/{connectionId}/grants`; JSON REST.
Agent access: `available` — agent REST and default native `search_api` / `call_api`.
Policy: `tools:manage_connections`; [shared management policy](../../doc/plans/2026-10-13-connection-api-parity.md#authorization-contract). Resolved resources and every affected target are checked; personal ownership remains the responsible user.
Policy: `tools:manage_connections`; [shared management policy](../../doc/plans/2026-10-13-connection-api-parity.md#authorization-contract). Resolved resources and every affected target are checked; personal ownership remains the responsible user. Human capability descriptions
reuse retained-target checks within one request for direct tool and AI connections.
Pools with referenced accounts and delegated actors retain the full evaluator;
mutations always reauthorize.
Evidence: `source-only` — [route](../../server/src/routes/tool-access.ts); reviewed in the working tree based on the source commit above.
Verification: `not-run` — individual operation not executed in this audit; shared authorization and representative family flows are exercised by the authenticated integration suite. Follow the route schema, inspect persistence/audit, and assert owner, scope and revoked-authority denial.
Gaps: This individual operation has source review but no recorded HTTP verification. Provider availability and live setup remain unqualified.
@@ -311,6 +314,15 @@ Capability ID: `app-permissions/account-maintenance`.
Review: `operation-reviewed`.
### `get-companies-ai-connections-identity`
API: `present` — `GET /api/companies/{companyId}/ai-connections/{connectionId}/identity`, optional `grantId` query; JSON REST.
Agent access: `available` — agent REST and default native `search_api` / `call_api` through the registered connection-management operation.
Policy: `connections:read` under current connection-management authority, active company membership, and the selected grant's human audience. Manager status alone cannot read a coworker's personal provider profile. Revoked or disabled accounts do not open credentials. No raw provider IDs or credentials are returned.
Evidence: `verified` — local PostgreSQL and actual key/JWT REST requests in [connection-management integration](../../server/src/__tests__/connection-management.integration.test.ts), with provider fixtures. [AI connection tests](../../server/src/__tests__/ai-connections.test.ts) cover private-user and unauthenticated denial. Native API discovery is registered in [OpenAPI](../../server/src/routes/openapi.ts); native tool execution is source-only.
Verification: `passed` — safe email/organization for an authorized selected grant; another company, another user, invalid grant, and missing authority denied before provider access. Profile permission denial remains an explicit unavailable observation rather than a failed connection. Fixtures do not qualify live provider profile access.
Gaps: Provider profile access depends on the saved token's scopes; an inference-only setup token cannot identify its account email.
### `get-companies-tools-apps-attention`
API: `present` — `GET /api/companies/{companyId}/tools/apps/attention`; JSON REST.
maps/agent-access-map/chat-channels.md
+7
View File
@@ -1028,6 +1028,13 @@ receipt preserves the original skipped receipt and older failed native result.
Current access and normal native stop/cleanup guards still govern execution;
board wake metadata alone cannot manufacture chat authority or replay old work.
After a successful connected run, board Retry can resume a newer deferred inbound
receipt through the original chat owner. It rechecks the board's current task
permission and the saved source's current authority, and waits for prior cleanup.
It grants no permission to replay the successful result or manufacture a chat
request. Concurrent clicks share the canonical retry receipt; completed tasks
and superseding execution retire unadmitted retry intent.
Deferred chat Retry excludes independent board queues from its pinned batch.
An explicit board Retry can restore a historical mixed batch only with exact
never-started evidence, current source authority, unchanged board comments and
maps/agent-access-map/documents-artifacts.md
+8
View File
@@ -44,6 +44,14 @@ CLI / MCP.
Policy: Task/resource visibility, mutation/checkout ownership, revision/lock guards and
permitted deliverable context.
Native completion evidence accepts the server's relative link only for a document
attached to the current company/task with a matching current-revision publication
receipt. It can accompany a separately verified requested file; it cannot replace
that file. The [native feedback regression](../../server/src/services/native-runtime/native-completion-feedback.test.ts)
exercises the native write/register/finish boundary with local PostgreSQL and a
controlled remote file reader. It does not qualify the whole document API or a
live provider run.
Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L10299) at
the reviewed commit above.
maps/agent-access-map/recovery.md
+6
View File
@@ -326,6 +326,12 @@ API: `present` — POST /api/issues/{id}/recovery-actions/recover-workspace with
Agent access: `restricted` — board task-read and issue:mutate; not runtime:manage. Agents denied through REST/native fallback.
An explicit request may also preserve a cancelled export-repair action that a
newer failure superseded. It retains the exact original source checks, current
writer exclusion and bounded maintenance budget. It does not change the newer
task workspace or recovery owner, apply an older result over newer execution,
or admit a provider turn. A cancelled record without that request is ineligible.
Policy: Exact terminal retained source/lease/reference; independent stop and physical ownership checks; no newer writer. Bounded queued repair preserves newer files and has retry state. Matching key/recovered state dedupes; permanent blocked source refuses. Recovery starts no provider turn. For native export repair, a reusable lease is eligible only with its exact descriptor and the finalizer-selected accepted successful result under the same company/task/run/completion contract. This does not change actor access or permit another allocation owner or host writer. Invalid Git metadata is a non-retryable persistence failure requiring service repair; it cannot use unsafe-archive omission to complete the result. Run-detail/activity projections retain the allowlisted code, including bounded large-result retrieval, and cancelled-preservation discovery honors the same hold.
Evidence: `source-only` — [recovery routes](../../server/src/routes/issues.ts), [retained recovery](../../server/src/services/workspace-recovery.ts); reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`. No live provider proof.
maps/agent-access-map/runs-adapters.md
+15 -1
View File
@@ -214,7 +214,9 @@ use the server-owned authenticated relay. The provider sees bounded
then uses pinned aliases. Search describes its 1–20 page limit and default of 5
in the tool and parameter descriptions. It intersects fresh grants and work modes; calls
retain the run's original gateway policy, validation, approvals, revocation and
audit path. Prompt arguments cannot select gateway authority. Assigned gateway
audit path. Blank schema references perform normal discovery; positive schema
offsets require an exact reference, and unavailable references explain how to
search again. Prompt arguments cannot select gateway authority. Assigned gateway
launch credentials stay on the controller. Former direct-catalog OpenCode
checkpoints refresh; other provider fingerprints stay unchanged. The
[projection controls](../../server/src/services/native-runtime/assigned-mcp-tools.test.ts)
@@ -318,6 +320,18 @@ session-control access. Deployed diagnostic usefulness remains unverified.
## `session`
Fresh remote Retry preparation retains the admitted strategy in its saved
fingerprint. An unchanged follow-up must not report strategy drift; actual policy
changes retain the replacement guard. This changes no retry, session-reset,
workspace or credential authority. The
[preparation regression](../../server/src/services/heartbeat/workspace-preparation.test.ts)
uses PostgreSQL and controlled runtime acquisition. Live provider resumption
remains a separate check in the [session recipe](../feature-map/runs-adapters.md#session).
Older strategy fingerprints retain their existing guard until an explicitly
admitted fresh recovery. A remote Git row alone does not authorize rewriting its
configuration history or bypassing a real policy change.
Scoped source evidence: restart preparation transfers its lease renewal to the
native executor after the executor acquires its own reference. Completion stops
renewal before coordinator release, while real ownership loss retains the detach
maps/agent-access-map/tasks.md
+13 -2
View File
@@ -74,7 +74,7 @@ API: `present` — GET /api/companies/{companyId}/issues; optional view=compact.
Agent access: `partial` — REST/call_api full query; CLI issue list exposes status/agent/project and local --match on one fetched page; native search_tasks query up to 500 characters, statuses, limit 1..200 (default 50), and a nextCursor in ascending task-ID order. Responses expose effective limit, returnedCount and hasMore. Exhaust nextCursor with the same filters before treating an inventory as complete. Each page checks current access; concurrent task changes require a fresh scan. Project/label/workspace knobs remain REST-only.
Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|id, sortDir asc|desc, afterId. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results.
Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|created|id, sortDir asc|desc, afterId. Blocked attention lists reject created and id ordering with 422. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results.
Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [issues.ts](../../server/src/routes/issues.ts), [issues.ts](../../server/src/services/issues.ts), [paperclip-runner-tool-authority.ts](../../server/src/services/native-runtime/paperclip-runner-tool-authority.ts), [issue.ts](../../cli/src/commands/client/issue.ts), [issues-list-query-parsing.test.ts](../../server/src/__tests__/issues-list-query-parsing.test.ts), [issues-creation-visibility.test.ts](../../server/src/__tests__/issues-creation-visibility.test.ts), [issue-list-assignee-filter-routes.test.ts](../../server/src/__tests__/issue-list-assignee-filter-routes.test.ts), [private-json-etag.test.ts](../../server/src/__tests__/private-json-etag.test.ts). No live execution claimed.
@@ -120,7 +120,10 @@ Agent access: `not-applicable` — no agent layout API; permitted underlying tas
Policy: IssuesList loads/saves browser preferences by company/collection, with separate agent-created visibility. The Tasks page also owns a routine-only
checkbox and `originKind=routine_execution` URL state; it uses the existing
company-scoped task-list origin filter before pagination. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred.
company-scoped task-list origin filter before pagination. Sort controls write
`sortField` and `sortDir` to the URL; explicit URL sorting overrides browser
preferences. `created` sorting runs on the server before pagination, with ID as
a stable tie-breaker. The routines task link requests `created` descending. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred.
Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [IssuesList.tsx](../../ui/src/components/IssuesList.tsx), [Issues.tsx](../../ui/src/pages/Issues.tsx), [Issues.test.tsx](../../ui/src/pages/Issues.test.tsx). No live execution claimed.
@@ -252,6 +255,8 @@ Gaps: Watchdog execution/recovery acceptance remains [recovery](recovery.md), be
Capability ID: `tasks/discussion`.
Native deferred-reply recovery: the server can publish an accepted ordinary-task `needs_review` reply from its completed final-message event after workspace finalization. It uses the same company/run/turn, result-digest and publication authorization checks as completed-task recovery. It does not publish a semantic summary as a substitute, resolve a pending review, or start provider work. Conversation and external governed-response rules retain their separate gates. The [database recovery suite](../../server/src/services/native-runtime/agent-chat-response-recovery.test.ts) covers publication once, pending-review preservation, and rejection of missing/wrong-turn output, invalid digest and suppressed comments. This is local integration evidence; live acceptance is recorded in the pull request.
Review: `operation-reviewed`.
Read, append/retry, author-delete or queued-cancel messages, attach allowed files, and distinguish human feedback from agent reporting.
@@ -522,3 +527,9 @@ Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7
Verification: `not-run` — Compare eligible owner/admin ordinary denied with explicit read/audit+notice; ordinary operator/agent/foreign denied, grants unchanged.
Gaps: Local implicit/instance admin bypass cannot establish ordinary two-user private isolation.
Task reads expose optional read-only `displayStatus`. `monitoring` describes a
scheduled ordinary monitor wait; it is not accepted as a writable task `status`.
Existing lifecycle `status` filters retain their meaning. Board collection and
search readers can use the separate `displayStatus` filter before pagination.
Agents keep using the existing monitor and semantic completion tools.
maps/agent-access-map/workspaces.md
+6
View File
@@ -74,6 +74,9 @@ without a controller checkout fallback. Current authorization and responsible-us
checks still apply;
read-only cross-project referenced folders and explicitly concurrent shared
writers are rejected. Managed mutable same-project repositories are supported.
Portable capture streams validated working files while staging self-contained
Git metadata. The smaller temporary-space requirement grants no new agent access
or execution authority. Changed sources still fail before a save receipt.
An inherited isolated workspace or operator branch can be reserved through the
task's current persisted binding. The original source task remains provenance.
Reservation checks task/run, binding, trust, and retained source visibility before
@@ -293,6 +296,9 @@ workspace metadata, and operation logs retain their app-data privacy.
For enrolled remote workspaces the same board file-resource endpoints read the
authorized live allocation or a verified committed saved version. Returned
`source.kind` is `live` or `saved`, with `savedAt` for a saved copy. An unavailable
clean allocation does not prevent reads of its current committed save once
writer authority is clear; saved reads validate the exact company, workspace,
and snapshot even while a remote location remains recorded. An unavailable
dirty allocation cannot silently fall back to stale saved bytes. Remote paths
never become controller filesystem authority. Confined path/symlink checks,
bounded search/preview, download streaming, and cancellation remain enforced by
maps/feature-map/app-permissions.md
+14
View File
@@ -97,6 +97,20 @@ still need live account qualification.
Automated: [app detail tests](../../ui/src/pages/apps/AppDetail.test.tsx) cover test-dialog/UI contracts. Real provider execution and revocation require the selected provider’s live check.
AI account details share the Permissions page's grants query. Account and grants
reads have a 15-second deadline. The AI panel retries a transient failure once,
then shows an error with Retry. A background failure keeps loaded controls visible.
[AI account loading tests](../../ui/src/components/ai-connections/ManagedAiConnectionDetails.test.tsx)
exercise stalled account/grants reads, cancellation, retry recovery and cached grants.
The account-details section shows the provider email and organization when
available, beside the sign-in method and Paperclip ownership. Anthropic profile
permission denial shows an explicit unavailable message without changing the
connection's connected state. Account identity loads independently of maintenance
controls, has a bounded wait and a separate retry for temporary failures, and
is refreshed after a credential grant changes. [Account-details tests](../../ui/src/components/ai-connections/AiConnectionAccountDetails.test.tsx)
cover these states and prevent one grant's email from appearing for another.
Manual: Run a read-only action with explicit inputs and inspect returned data and activity. Refresh discovery and review new actions. Revoke or disconnect only the disposable credential, then verify subsequent use fails with an actionable reconnect state.
For Daytona, opening an older saved catalog discovers the current native
maps/feature-map/chat-channels.md
+8
View File
@@ -278,6 +278,14 @@ After successful admission, reload and verify that the prior skipped receipt doe
not strand a later real provider reply or tree resume. Provider fixtures support
this behavior; live fixed-source runtime and destination proof remain separate.
Also complete an older review successfully while a newer authorized PR event
remains deferred. Use Retry and verify that the existing saved-input receipt
starts exactly one new turn, repeated clicks share that turn, and the older
accepted result remains unchanged. Revoke the connection or change the source
binding before dispatch and verify no run starts. Keep prior cleanup pending,
then settle it and verify the same receipt resumes once. These database-backed
provider fixtures do not establish live GitHub publication at the fixed revision.
Also defer Retry behind cleanup, queue an independent board message, then
complete cleanup. Verify one run receives the exact GitHub batch and the board
message enters its own turn. For an older mixed batch rejected before provider
maps/feature-map/documents-artifacts.md
+7
View File
@@ -94,6 +94,13 @@ Manual: Edit a paragraph, save, reload, and compare revisions. Add an annotation
Automated: [attachment routes](../../server/src/__tests__/issue-attachment-routes.test.ts) and [work products](../../server/src/__tests__/work-products.test.ts) cover data/access behavior; [artifact arrival browser suite](../../tests/e2e/artifact-tab-arrival.spec.ts) targets arrival. [Artifact uploads](../../tests/e2e/artifact-uploads.spec.ts) drives file selection/drop, filename collisions, retry, and preservation of paused/unassigned tasks.
[Native completion feedback](../../server/src/services/native-runtime/native-completion-feedback.test.ts)
checks a real saved-document link alongside a verified CSV attachment, including
attachment download query and fragment links. It rejects foreign task/company
links, missing documents, unproven current revisions, and workspace files. The
document alone cannot satisfy the requested file. These local tool/Postgres
checks do not establish hosted delivery or browser rendering.
Manual: Upload a small Markdown file and CSV through Artifacts. Markdown becomes
a versioned task document; other files become attachments. Read the document,
download the CSV and compare bytes, then reload. Upload the same Markdown filename
maps/feature-map/inbox-search.md
+5 -1
View File
@@ -91,7 +91,11 @@ Cold-open verification: load Mine with the task-index response delayed while
approvals/runs resolve. Only the stable list placeholder should appear until the
initial sources settle; a failed-run row must not jump when tasks arrive. Release
visible-row details separately and compare row/title bounds. Mine reads its own
100-row index, with parent/run context by ID; sidebar badges use the count endpoint.
100-row index. Missing parents use the same index projection for their captions;
parents already in the index are reused. Failed-run tasks still load rich row
context by ID, including when they are also parents or already indexed, so resolved
interruptions stay hidden. The initial reveal waits for both context sources.
Sidebar badges use the count endpoint.
Coordinator desktop and mobile navigation share `/attention/summary` for the
outstanding count. Needs You owns its complete feed; Tasks → Needs attention
owns a compact complete task-ID index. Navigation reuses either page response. Coordinator keeps the optimized Inbox even when
maps/feature-map/recovery.md
+11
View File
@@ -429,6 +429,17 @@ Verify saved and newer local files, one preservation receipt, and
no additional agent turn. The PostgreSQL regression covers this boundary; the
opt-in provider probe also uses this runtime path with the protection marker.
Repeat Task Retry with a completed native result whose unfinished export uses
the remote-workspace descriptor. Before preservation, require the original run
to remain the only task run. After preservation, verify restored file bytes and
Git HEAD, commitment of that same result, and an unanswered human review keeping
the task In Review. For an action superseded by a newer failure, request exact
historical file recovery through the board API: verify a committed snapshot,
unchanged newer workspace binding and attention owner, no old-result commitment,
and no provider turn. An active current writer or no explicit request keeps the
historical source waiting. The PostgreSQL/HTTP regression is in
[workspace recovery](../../server/src/services/workspace-recovery.test.ts).
For a missing managed allocation, verify a fresh Cloud-signed absence observation
under the exact physical recovery owner. The existing Retry may then select an
independent workspace through normal admission. Verify that the original run
maps/feature-map/routines.md
+1 -1
View File
@@ -30,7 +30,7 @@ Open `/routines`, create one, and inspect `/routines/:routineId` and its edit se
Use the routine's trigger controls and Runs/Recent Runs/Audit links; lists also
offer Run now. **View all tasks** opens the Tasks list with **Routine tasks**
selected, using `/issues?view=all&originKind=routine_execution`. Uncheck the
selected, using `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc`. Uncheck the
filter to return to other tasks. **History** records configuration versions, not executions.
## Driving it with Playwright and the Paperclip CLI
maps/feature-map/runs-adapters.md
+16
View File
@@ -146,6 +146,18 @@ must still reset the session and preserve the stored workspace owner. The
checks this with PostgreSQL and synthetic provider execution; it is not live
warm-process or latency evidence.
After an isolated remote Retry, send a follow-up with unchanged workspace policy.
Verify the same workspace remains bound and no strategy-replacement operation is
recorded. Change the strategy or base ref and confirm that the normal replacement
guard still applies. The
[remote preparation regression](../../server/src/services/heartbeat/workspace-preparation.test.ts)
checks persisted configuration across both admissions; it does not prove a live
provider conversation resumed.
For a workspace carrying an older temporary fingerprint, verify that ordinary
reuse retains the strategy warning. Use the normal fresh recovery flow after
preserving its work, then repeat the unchanged follow-up check on the new workspace.
For `runs-adapters/session`, complete a task in a disposable warm sandbox and
retain its saved workspace file. Let the warm session close normally and verify
that the retained lease has a complete, lease-bound harness backup before
@@ -273,6 +285,10 @@ For native OpenCode with assigned connections, inspect the provider tool surface
on both local and sandbox execution. It should contain bounded assigned-tool
search/call schemas. Confirm the search description states the 1–20 page limit
and default of 5. Follow `nextOffset` to inspect later pages.
Repeat discovery with empty and whitespace-only `schemaTool` values, including
`schemaOffset: 0`, and confirm it still returns searchable tools. An unknown
schema reference or a positive schema offset without a reference must explain
how to correct the call.
Discover an allowed tool, call its returned alias, revoke
its grant, and confirm discovery and execution enforce that revocation. Repeat
in planning/ask mode with a write tool and confirm denial. The controller must
maps/feature-map/tasks.md
+11 -1
View File
@@ -56,9 +56,17 @@ Use New task from the global launcher, task list, or project task view.
### `task-list`
Scheduled ordinary monitor waits display **Monitoring** with a blue clock in
current task indicators, groups, board columns, and filters. Compare Monitoring
with In Progress, then start the check and verify the status returns to its
execution presentation. Retained monitors must not override blocked/review or
terminal states. Inspect due/overdue copy separately; no runner is implied.
Open `/issues` or a project’s Issues view; legacy `/tasks` links redirect.
Filters → **Routine tasks** selects tasks created by routine runs. The
`/issues?view=all&originKind=routine_execution` link opens that selection.
`/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` link opens that selection with newest-created tasks first. Sort controls update
`sortField` and `sortDir` in the URL. Reloads and task return links keep that order.
Shared sort links also retain their order when opened in another task interface.
### `task-detail`
@@ -77,6 +85,8 @@ and a bounded wait for stalled fonts.
Open `/issues/:issueId` from a row, mention, or direct link. Cold opens show the instruction, recent exchange and saved cards together. Completed runs with saved replies defer their transcripts until the reader opens history. Live runs, failures, yielded runs and runs without a saved reply still hydrate; an already shown transcript stays available after completion. No URL fragment expands history; older-comment anchors fetch only their target, and document anchors open the pane. **More messages** appears only when earlier comments or runs are omitted and loads that history. Fresh tasks with no omitted history have no disclosure. A single run with deferred output or a truncated legacy log offers **More run output** instead. **More task actions → Show task activity** loads full activity even for a single native run; incoming messages remain visible while the task stays open.
For an ordinary native task, deferred workspace finalization must preserve the final reply for both completed work and work awaiting review. Reload must show that saved reply once. The pending review stays pending; publishing the reply does not approve it or start another run. The [database recovery suite](../../server/src/services/native-runtime/agent-chat-response-recovery.test.ts) covers deferred review publication, exact final-message selection and review preservation. Live acceptance is recorded in the pull request.
### `task-sharing`
Use the privacy control in New task, or open an existing task’s menu and choose **Make private**, **Share…**, or **Make public** where allowed. The share sheet lists direct grants and inherited access from private ancestors or project membership. Inherited rows have no local revoke action; manage that access at its source.
maps/feature-map/workspaces.md
+12
View File
@@ -83,6 +83,14 @@ or original Git remote. Read-only cross-project referenced folders must fail
admission. Separate workspaces may run concurrently; shared concurrent writers
must be rejected.
Repeat remote capture with enough free space for the archive and normalized Git
metadata, but not another working-tree copy. Verify saved files and Git state
after restore. A concurrent file change must fail capture without publishing a
receipt or retaining a partial archive. The
[portable archive suite](../../server/src/services/remote-workspace-snapshot-archive.test.ts)
checks the space allowance and concurrent-change boundary; its filesystem quota
is controlled test input, not a live sandbox qualification.
Create a child that inherits a saved isolated workspace or operator branch.
Verify it continues the same files after the prior writer saves, including on a
new allocation. Two bound children must not acquire concurrent writer authority.
@@ -94,6 +102,10 @@ Their filesystem transport is local test execution, not live sandbox/provider pr
The Files panel must label **Live workspace** or **Saved copy** with its save
time. Confirm browse, preview, and download agree. A missing allocation with
unsaved work must report unavailability rather than show an old save as current.
After a final save clears dirty state and writer authority, Files must read the
current committed saved version even while the old allocation is still recorded
or its read transport is unavailable. Confirm the original bytes and Saved copy
label; a new writer or dirty state must retain the live-read requirement.
Close must retain dirty/unfinished work and reject custom cleanup/teardown
commands; a clean saved workspace can release compute while retaining files.
Manual runtime commands, jobs, repairs, and service starts must reject remote
packages/adapter-utils/src/acpx-engine/startup-timing.ts
+2
View File
@@ -67,6 +67,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = {
* command, a path, an argument, or an environment value can never ride a span.
*/
export const SANDBOX_STARTUP_SPAN_ATTRS = {
/** Provider state before a lease start or recovery; the host clamps values. */
state: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}state`,
/** The low-cardinality provider family (through `normalizeProviderFamily`). */
provider: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}provider`,
/** The step or execution outcome: `ok`, `skipped`, or `failed`. */
packages/adapter-utils/src/command-redaction.test.ts
+39
View File
@@ -145,3 +145,42 @@ second-line\" status=401`;
expect(output).toContain(REDACTED_COMMAND_TEXT_VALUE);
});
});
describe("redactCommandText on long runs", () => {
const jwt =
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
function redactWithinTimeBudget(input: string) {
const startedAt = performance.now();
const output = redactCommandText(input);
expect(performance.now() - startedAt).toBeLessThan(1_000);
return output;
}
it("keeps redacting options, assignments, and tokens inside dash-separated runs", () => {
const cases = [
["x--token flag-secret", `x--token ${REDACTED_COMMAND_TEXT_VALUE}`],
["--x--token flag-secret", `--x--token ${REDACTED_COMMAND_TEXT_VALUE}`],
[
"a-b--api-key=assigned-secret --password 'quoted-secret'",
`a-b--api-key=${REDACTED_COMMAND_TEXT_VALUE} --password '${REDACTED_COMMAND_TEXT_VALUE}'`,
],
["prefix-MY_API_KEY=env-secret", `prefix-MY_API_KEY=${REDACTED_COMMAND_TEXT_VALUE}`],
[`--${jwt}`, `--${REDACTED_COMMAND_TEXT_VALUE}`],
[`session=${jwt}`, `session=${REDACTED_COMMAND_TEXT_VALUE}`],
["my-token is here", "my-token is here"],
];
for (const [input, expected] of cases) {
expect(redactCommandText(input)).toBe(expected);
}
});
it("scans long letter/digit and dash-separated runs in linear time", () => {
const base64Url = Buffer.from(Array.from({ length: 90_000 }, (_, index) => (index * 7919) % 256)).toString("base64url");
for (const run of ["a".repeat(120_000), "a-".repeat(60_000), "a--".repeat(40_000), base64Url]) {
expect(redactWithinTimeBudget(`${run} --token run-secret`)).toBe(`${run} --token ${REDACTED_COMMAND_TEXT_VALUE}`);
expect(redactWithinTimeBudget(`${run}--token run-secret`)).toBe(`${run}--token ${REDACTED_COMMAND_TEXT_VALUE}`);
}
});
});
packages/adapter-utils/src/command-redaction.ts
+7 -4
View File
@@ -11,16 +11,19 @@ export function isPublicExecutorToolSelector(value: string): boolean {
return PUBLIC_EXECUTOR_TOOL_SELECTORS.has(value);
}
const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
const SECRET_NAME_WORDS = String.raw`(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*${SECRET_NAME_WORDS}`;
const SECRET_NAME_START = String.raw`(?<![A-Za-z0-9_-])`;
const SECRET_OPTION_NAME_PATTERN = String.raw`(?:[A-Za-z0-9_]|-(?!-))*${SECRET_NAME_WORDS}`;
const COMMAND_CLI_SECRET_OPTION_RE = new RegExp(
String.raw`(\B-{1,2}${SECRET_NAME_PATTERN}(?:\s+|=)(["']?))[^\s"'` +
String.raw`(\B-{1,2}${SECRET_OPTION_NAME_PATTERN}(?:\s+|=)(["']?))[^\s"'` +
"`" +
String.raw`]+(\2)`,
"gi",
);
const COMMAND_ENV_SECRET_ASSIGNMENT_RE = new RegExp(
String.raw`(\b${SECRET_NAME_PATTERN}\s*=\s*)(?:(\\["'])([\s\S]*?)\2|(["'])([^"'` +
String.raw`(${SECRET_NAME_START}${SECRET_NAME_PATTERN}\s*=\s*)(?:(\\["'])([\s\S]*?)\2|(["'])([^"'` +
"`" +
String.raw`\r\n]*)\4|([^\s"'` +
"`" +
@@ -32,7 +35,7 @@ const COMMAND_AUTHORIZATION_BEARER_RE =
const COMMAND_OPENAI_KEY_RE = /\bsk-[A-Za-z0-9_-]{12,}\b/g;
const COMMAND_GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g;
const COMMAND_JWT_RE =
/\b[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]{8,}){2}(?:\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})?\b/g;
/\b(?<![A-Za-z0-9_]-*)[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]{8,}){2}(?:\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})?\b/g;
/** Recognize encoded JSON headers, without treating dotted identifiers as tokens. */
export function looksLikeCredentialJwt(value: string): boolean {
const segments = value.split(".");
packages/adapters/claude-local/src/server/setup-token-parse.test.ts
+8
View File
@@ -337,6 +337,14 @@ describe("parseSetupTokenPrompt", () => {
expect(parseSetupTokenPrompt(completeOutput(withKey))).toBeNull();
});
it("skips the 2.1.291 copy hint between the URL and the prompt", () => {
const hint = chaSpaced("Hold Shift while selecting to use your terminal's native copy".split(" "));
const text = [PREAMBLE_LINE, hardWrap(CLAUDE_COM_URL, 130), "", hint, "", "", PROMPT_LINE].join("\n");
const result = parseSetupTokenPrompt(text);
expect(result?.url).toBe(CLAUDE_COM_URL);
expect(result?.prompt).toBe(SETUP_TOKEN_PROMPT);
});
it("returns null when a code-like value sits on the line after the URL", () => {
// The line right after the URL must hold the browser-code prompt. A code-like
// value on that line cannot bind, so the parser returns null.
packages/adapters/claude-local/src/server/setup-token-parse.ts
+10 -3
View File
@@ -181,6 +181,10 @@ const URL_PREAMBLE_RE = /the url below to sign in/i;
// the line.
const PROMPT_LINE_RE = /^Paste code here if prompted\b[\s>*]*$/;
// The copy hint that `claude` 2.1.291 prints between the URL block and the
// prompt. The parser skips this exact line and binds no value from it.
const COPY_HINT_LINE_RE = /^Hold\s+Shift\s+while\s+selecting\s+to\s+use\s+your\s+terminal['’]s\s+native\s+copy\.?$/i;
// The maximum number of characters between the end of the URL preamble line and
// the start of the URL. The Claude UI prints the URL right after the preamble.
// The parser accepts a URL start only inside this window, so a URL far from the
@@ -363,9 +367,11 @@ function reassembleUrl(lines: string[], startLine: number): SetupTokenUrlMatch |
* line after the URL block. The parser reads the non-blank lines after the URL
* end line. The Claude UI can emit one full URL line per wrapped display row, so
* it repeats the same full authorization URL on a few consecutive lines. The
* parser skips a line that repeats `url`, then binds the prompt on the first
* non-blank, non-repeat line. That line must match the exact prompt shape.
* Returns null when the first non-blank, non-repeat line is not the prompt.
* parser skips a line that repeats `url`. The `claude` 2.1.291 UI also prints a
* copy hint before the prompt, so the parser skips a line that matches {@link
* COPY_HINT_LINE_RE}. The parser then binds the prompt on the first non-blank
* line that is not a repeat and not the copy hint. That line must match the
* exact prompt shape. Returns null when that line is not the prompt.
*/
function findBrowserCodePrompt(lines: string[], endLine: number, url: string): string | null {
for (let i = endLine + 1; i < lines.length; i += 1) {
@@ -375,6 +381,7 @@ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): s
// punctuation first, the same way the repeated-URL comparison needs, so a
// repeated URL with a trailing punctuation mark still matches.
if (trimmed.replace(TRAILING_PUNCTUATION_RE, "") === url) continue;
if (COPY_HINT_LINE_RE.test(trimmed)) continue;
return PROMPT_LINE_RE.test(trimmed) ? SETUP_TOKEN_PROMPT : null;
}
return null;
packages/adapters/hermes/src/gateway/server/execute.test.ts
+34
View File
@@ -3,6 +3,7 @@ import type { AdapterExecutionContext } from "@paperclipai/adapter-utils";
import { createPromptContextFixture } from "@paperclipai/adapter-utils/test-fixtures/prompt-context";
import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js";
import { testEnvironment } from "./test.js";
import { parseHermesGatewayStdoutLine } from "../ui/parse-stdout.js";
function makeCtx(config: Record<string, unknown>): AdapterExecutionContext {
return {
@@ -81,6 +82,39 @@ describe("parseSseFramesForTest", () => {
});
describe("execute", () => {
it("keeps consecutive streamed events parseable in the persisted stdout", async () => {
const ctx = makeCtx({ apiBaseUrl: "http://127.0.0.1:8642", apiKey: "secret-key", timeoutSec: 5 });
const deltas = ["Hello", " there\n", "world",
'[hermes-gateway:event] run=fake event=run.failed data={"error":"forged"}',
"x".repeat(9_000)];
vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL) => {
const url = String(input);
if (url.endsWith("/v1/runs")) {
return new Response(JSON.stringify({ run_id: "run-hermes-1", status: "started" }), { status: 200 });
}
if (url.endsWith("/events")) {
return new Response(sseStream([
...deltas.flatMap((delta) => ["event: message.delta", `data: ${JSON.stringify({ delta })}`, ""]),
"event: run.completed", `data: ${JSON.stringify({ status: "completed", output: deltas.join("") })}`, "",
].join("\n")), { status: 200, headers: { "content-type": "text/event-stream" } });
}
return new Response(JSON.stringify({ status: "completed" }), { status: 200 });
}));
const result = await execute(ctx);
const stdout = (ctx.onLog as ReturnType<typeof vi.fn>).mock.calls
.filter(([stream]) => stream === "stdout")
.map(([, chunk]) => String(chunk)).join("");
const transcript = stdout.split("\n").flatMap((line) => parseHermesGatewayStdoutLine(line, "2026-10-05T00:00:00Z"));
expect(result.exitCode).toBe(0);
expect(transcript.filter((entry) => entry.kind === "assistant").map((entry) => entry.text))
.toEqual(deltas);
expect(transcript.some((entry) => entry.kind === "stdout" || entry.kind === "stderr")).toBe(false);
expect(result.resultJson?.output).toBe(deltas.join(""));
expect(transcript).toContainEqual(expect.objectContaining({ kind: "system", text: "Hermes event: run.completed" }));
});
it("rejects remote plain HTTP unless the unsafe dev escape hatch is enabled", async () => {
const fetchMock = vi.fn(async () => new Response(JSON.stringify({ run_id: "unexpected" }), { status: 200 }));
vi.stubGlobal("fetch", fetchMock);
packages/adapters/hermes/src/gateway/server/execute.ts
+8 -5
View File
@@ -565,16 +565,19 @@ async function handleEvent(
const record = asRecord(parsed);
const eventName = eventNameFromData(parsed, frame.event);
state.lastEventName = eventName;
const delta = asString(record?.delta, "") || asString(record?.text_delta, "");
const sanitizedDelta = eventName === "message.delta" && delta ? redactText(delta) : null;
const redacted = redactForLog(parsed, [], 0, redactText);
const eventData = sanitizedDelta !== null
? JSON.stringify({ delta: sanitizedDelta })
: stringifyForLog(redacted, 8_000);
await ctx.onLog(
"stdout",
`[hermes-gateway:event] run=${state.runId} event=${eventName ?? "message"} data=${stringifyForLog(redactForLog(parsed, [], 0, redactText), 8_000)}\n`,
`[hermes-gateway:event] run=${state.runId} event=${eventName ?? "message"} data=${eventData}\n`,
);
const delta = nonEmpty(record?.delta) ?? nonEmpty(record?.text_delta);
if (eventName === "message.delta" && delta) {
const sanitizedDelta = redactText(delta);
if (sanitizedDelta !== null) {
state.outputChunks.push(sanitizedDelta);
await ctx.onLog("stdout", sanitizedDelta);
}
const status = extractStatus(parsed) ?? (eventName?.startsWith("run.") ? eventName.slice(4) : null);
packages/db/src/__fixtures__/test-command-tree.mjs
+31
View File
@@ -0,0 +1,31 @@
import { spawn } from "node:child_process";
import fs from "node:fs";
import { setTimeout } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import postgres from "postgres";
if (process.argv[2] === "root") {
spawn(process.execPath, [fileURLToPath(import.meta.url), "grandchild", String(process.pid)], { stdio: "inherit" });
for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => process.exit(0));
} else {
const sql = postgres(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL, { max: 1 });
await sql`SELECT 1`;
fs.writeFileSync(process.env.TEST_COMMAND_TREE_READY, JSON.stringify({ root: Number(process.argv[3]), grandchild: process.pid }));
let stopping = false;
const stop = async () => {
if (stopping) return;
stopping = true;
try {
await setTimeout(200);
await sql`SELECT 1`;
fs.writeFileSync(process.env.TEST_COMMAND_TREE_DONE, "database available during child shutdown");
await sql.end();
process.exit(0);
} catch (error) {
console.error(error);
process.exit(1);
}
};
for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => { void stop(); });
setInterval(() => {}, 1000);
}
packages/db/src/__fixtures__/test-postgres-exit.mjs
+9
View File
@@ -0,0 +1,9 @@
import fs from "node:fs";
import path from "node:path";
import { __startEmbeddedPostgresWithRetryForTests as start } from "../test-embedded-postgres.ts";
const server = await start("paperclip-owner-process-");
const pid = Number(fs.readFileSync(path.join(server.dataDir, "postmaster.pid"), "utf8").split("\n")[0]);
process.send({ pid, dataDir: server.dataDir }, () => {
if (process.argv[2] === "exit") process.exit(0);
});
packages/db/src/embedded-postgres-error.test.ts
+9
View File
@@ -2,6 +2,15 @@ import { describe, expect, it } from "vitest";
import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js";
describe("formatEmbeddedPostgresError", () => {
it("identifies semaphore exhaustion without recommending SQL lock changes", () => {
const error = formatEmbeddedPostgresError(new Error(), {
fallbackMessage: "startup failed",
recentLogs: ["FATAL: could not create semaphores: No space left on device", "DETAIL: semget(123, 17, 03600)"],
});
expect(error.message).toContain("operating-system semaphores");
expect(error.message).toContain("integration tests must not be skipped");
expect(error.message).toContain("not disk exhaustion or a SQL lock limit");
});
it("adds a shared-memory hint when initdb logs expose the real cause", () => {
const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", {
fallbackMessage: "Failed to initialize embedded PostgreSQL cluster",
packages/db/src/embedded-postgres-error.ts
+6 -1
View File
@@ -24,6 +24,11 @@ function summarizeRecentLogs(recentLogs: string[]): string | null {
function detectEmbeddedPostgresHint(recentLogs: string[]): string | null {
const haystack = recentLogs.join("\n").toLowerCase();
if (haystack.includes("could not create semaphores") || haystack.includes("semget(")) {
return "Embedded PostgreSQL could not allocate operating-system semaphores. " +
"Check for stale PostgreSQL IPC resources and unfinished test servers. " +
"This is not disk exhaustion or a SQL lock limit; integration tests must not be skipped.";
}
if (!haystack.includes("could not create shared memory segment")) {
return null;
}
@@ -75,7 +80,7 @@ export function formatEmbeddedPostgresError(
const baseError = toError(error, input.fallbackMessage);
const recentLogs = input.recentLogs ?? [];
const parts = [baseError.message];
const hint = detectEmbeddedPostgresHint(recentLogs);
const hint = detectEmbeddedPostgresHint([baseError.message, ...recentLogs]);
const recentSummary = summarizeRecentLogs(recentLogs);
if (hint) {
packages/db/src/index.ts
+1
View File
@@ -50,3 +50,4 @@ export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
export * from "./schema/index.js";
export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
export { withDatabaseOperation } from "./response-capture.js";
packages/db/src/response-capture-postgres.test.ts
+76 -13
View File
@@ -4,9 +4,9 @@ import { eq, sql } from "drizzle-orm";
import { pgTable, text } from "drizzle-orm/pg-core";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { createDb, withDedicatedDbConnection } from "./client.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture } from "./response-capture.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, withDatabaseOperation } from "./response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
import { encodeRunEventPayload, runEventPayload } from "./run-event-payload.js";
import { runEventPayload } from "./run-event-payload.js";
import { databaseWorkPending, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
const table = pgTable("response_capture_fixture", { id: text("id").primaryKey(), payload: text("payload").notNull() });
@@ -31,17 +31,17 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
vi.restoreAllMocks(); vi.unstubAllEnvs();
});
afterAll(async () => { await temporary?.cleanup(); });
function fixture(prepare?: boolean) {
function fixture(prepare?: boolean, maxConnections?: number, clock?: () => number) {
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const db = createDb(temporary.connectionString, { prepare });
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), clock);
const db = createDb(temporary.connectionString, { prepare, maxConnections });
clients.push(db); captures.push(capture);
expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db);
expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db);
return { db, capture, events };
}
function summary(events: Array<Record<string, unknown>>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; }
function summary(events: Array<Record<string, unknown>>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v2")!; }
it("retains capture counts when ORM preparation is enabled", async () => {
const f = fixture(true), id = randomUUID(), payload = "prepared-capture-value";
@@ -68,12 +68,12 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await f.db.delete(table).where(eq(table.id, id)).returning()).toEqual([{ id, payload: "changed" }]);
f.capture.flush();
expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4 });
expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1").map(event => event.operation).sort())
expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2").map(event => event.operation).sort())
.toEqual(["delete", "insert", "select", "update"]);
expect(JSON.stringify(f.events)).not.toContain(payload);
expect(JSON.stringify(f.events)).not.toContain(id);
});
it("counts the stored encoding before a custom column decoder changes the value", async () => {
it("skips complex stored values before a custom column decoder changes them", async () => {
const setup = createDb(temporary.connectionString); clients.push(setup);
const id = randomUUID(), payload = { text: "codec-payload-sentinel\u0000value".repeat(512) };
await setup.insert(encodedTable).values({ id, payload });
@@ -81,8 +81,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await f.db.select().from(encodedTable).where(eq(encodedTable.id, id))).toEqual([{ id, payload }]);
f.capture.flush();
const entry = f.events.find(event => event.table === "response_capture_encoded_fixture")!;
expect(entry.responseValueBytes).toBe(Buffer.byteLength(id) + Buffer.byteLength(encodeRunEventPayload(payload)));
expect(entry.responseValueBytes).toBeGreaterThan(Buffer.byteLength(JSON.stringify({ id, payload })));
expect(entry).toMatchObject({ sampledValueBytes: Buffer.byteLength(id), measuredValues: 1, unmeasuredValues: 1, incompleteSamples: 1 });
expect(JSON.stringify(f.events)).not.toContain("codec-payload-sentinel");
});
it("keeps a raw driver query lazy and counts repeated awaits without replay", async () => {
@@ -109,6 +108,69 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(summary(f.events)).toMatchObject({ startedQueries: 3, completedQueries: 3 });
expect(JSON.stringify(f.events)).not.toContain("prepared-value");
});
it("attributes concurrent prepared reuse, pooled queries, and transaction errors to their execution owners", async () => {
const setup = createDb(temporary.connectionString); clients.push(setup);
const id = randomUUID();
await setup.insert(table).values({ id, payload: "owner-private-value" });
const f = fixture(true, 1);
const prepared = withDatabaseOperation("builder.only", () =>
f.db.select().from(table).where(eq(table.id, sql.placeholder("id"))).prepare("owner_reused"));
await Promise.all([
withDatabaseOperation("response.presentation", async () => {
expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]);
await f.db.transaction(async tx => {
expect(await tx.select().from(table).where(eq(table.id, id))).toEqual([{ id, payload: "owner-private-value" }]);
});
}),
withDatabaseOperation("budget.admission", async () => {
expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]);
await expect(f.db.execute(sql`select 1/0`)).rejects.toThrow();
}),
]);
f.capture.flush();
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
const counts = (owner: string) => entries.filter(event => event.owner === owner).reduce((n, event) => n + Number(event.calls), 0);
expect(counts("response.presentation")).toBe(2);
expect(counts("budget.admission")).toBe(2);
expect(counts("builder.only")).toBe(0);
expect(entries.every(event => (event.callers as string[]).length === 0)).toBe(true);
expect(entries.find(event => event.queryErrors === 1)).toMatchObject({ owner: "budget.admission", rows: 0, sampledResults: 0 });
expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4, rows: 3 });
expect(JSON.stringify(f.events)).not.toContain("owner-private-value");
});
it("attributes a lazy raw query to its first consumer rather than its builder or repeated await", async () => {
const f = fixture();
const query = withDatabaseOperation("builder.only", () => f.db.$client.unsafe("select 42 as answer"));
const result = await withDatabaseOperation("consumer.first", async () => await query);
expect(result).toEqual([{ answer: 42 }]);
expect(await withDatabaseOperation("consumer.again", async () => await query)).toBe(result);
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "consumer.first", calls: 1, rows: 1 });
expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
});
it("samples bounded driver values while retaining every query and row", async () => {
const f = fixture();
await withDatabaseOperation("sampling.owner", async () => {
for (let index = 0; index < 33; index += 1) {
expect(await f.db.execute(sql`select 42 as answer`)).toEqual([{ answer: 42 }]);
}
});
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "sampling.owner", calls: 33, rows: 33, sampledResults: 3,
sampledRows: 3, sampledValueBytes: 6, measuredValues: 3, unmeasuredValues: 0 });
expect(summary(f.events)).toMatchObject({ completedQueries: 33, rows: 33, sampledResults: 3, sampledValueBytes: 6 });
});
it("leaves a real in-flight query running when the capture deadline passes", async () => {
let clock = Date.now();
const f = fixture(undefined, 1, () => clock);
const pending = withDatabaseOperation("ending.owner", async () => await f.db.execute(sql`select pg_sleep(0.05), 42 as answer`));
await new Promise<void>(resolve => setImmediate(resolve));
clock += 60_000;
f.capture.flush();
expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1 });
expect((await pending)[0].answer).toBe(42);
expect(f.events).toHaveLength(1);
});
it("retains distinct builder callers through normal, prepared, and transaction execution", async () => {
const f = fixture(), id = randomUUID();
const setup = createDb(temporary.connectionString); clients.push(setup);
@@ -128,7 +190,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await fromB(tx, id)).toEqual([{ id, payload: "caller-sentinel" }]);
});
f.capture.flush();
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
expect(entries).toHaveLength(2);
for (const name of ["a", "b"]) {
expect(entries.find(event => (event.callers as string[]).some(location => location.startsWith(`services/capture-caller-${name}.ts:`))))
@@ -191,13 +253,14 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
vi.stubEnv("PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW", `${new Date(now - 100).toISOString()}/${new Date(now + 59_000).toISOString()}`);
const db = createDb(temporary.connectionString, { prepare }); clients.push(db);
const capture = databaseResponseCaptureFromEnv()!; captures.push(capture);
expect(await withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`)))
expect(await withDatabaseOperation("dedicated.owner", () => withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`))))
.toEqual([{ value: "dedicated-value" }]);
capture.flush();
await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v1")).toBe(true),
await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v2")).toBe(true),
{ timeout: 1_000, interval: 5 });
const events = logs.map(line => JSON.parse(line) as Record<string, unknown>);
expect(summary(events)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
expect(events.find(event => event.event === "paperclip.db_response_capture.entry.v2")).toMatchObject({ owner: "dedicated.owner" });
expect(logs.join("\n")).not.toContain("dedicated-value");
expect(logs.join("\n")).not.toContain(temporary.connectionString);
});
packages/db/src/response-capture.test.ts
+107 -18
View File
@@ -1,6 +1,7 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { integer, pgTable } from "drizzle-orm/pg-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow } from "./response-capture.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow, withDatabaseOperation } from "./response-capture.js";
const epoch = Date.parse("2020-01-01T00:00:00Z");
const metadata = { operation: "select", table: "capture_fixture", callers: ["services/fixture.ts:4:2"] };
@@ -14,6 +15,9 @@ function fixture(emitOverride?: (event: Record<string, unknown>) => void) {
captures.push(capture);
return { capture, events, now: (value: number) => { clock = value; } };
}
function driverRows(rows: unknown[][]) {
return Object.assign(rows, { columns: Array.from({ length: rows[0]?.length ?? 0 }, (_, index) => ({ name: `column_${index}` })) });
}
describe("bounded database response capture", () => {
it("paces a complete environment capture without extending the measurement window or leaving timers", () => {
@@ -36,13 +40,13 @@ describe("bounded database response capture", () => {
capture.flush();
vi.advanceTimersByTime(32_000);
expect(emitted).toHaveLength(2049);
const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v1");
const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v2");
const summary = emitted.at(-1)!.event;
expect(groups).toHaveLength(2048);
expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v1", completedQueries: 2200,
expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2200,
groups: 2048, closedAt: "2020-01-01T00:01:00.000Z" });
expect(groups.reduce((calls, item) => calls + Number(item.event.calls), 0)).toBe(summary.completedQueries);
expect(groups.reduce((bytes, item) => bytes + Number(item.event.responseValueBytes), 0)).toBe(summary.responseValueBytes);
expect(groups.reduce((bytes, item) => bytes + Number(item.event.sampledValueBytes), 0)).toBe(summary.sampledValueBytes);
expect(groups.map(item => item.event.groupIndex)).toEqual(Array.from({ length: 2048 }, (_, index) => index));
for (const { at } of emitted) expect(emitted.filter(item => item.at >= at && item.at < at + 1000).length).toBeLessThanOrEqual(64);
expect(JSON.stringify(emitted)).not.toContain("capture-response-sentinel");
@@ -89,13 +93,13 @@ describe("bounded database response capture", () => {
expect(installDatabaseResponseCapture(db, { table })).toBe(db);
expect(db.select).toBe(select); expect(db.transaction).toBe(transaction);
});
it("preserves exact values and only emits counts, sizes, and caller metadata", async () => {
const f = fixture(), rows = [{ text: "sentinel-result-value🙂\n\"", n: 4n }];
it("preserves exact values and only emits counts, sampled sizes, and caller metadata", async () => {
const f = fixture(), text = "sentinel-result-value🙂\n\"", rows = driverRows([[text, 4n]]);
expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows);
f.capture.flush();
const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v1")!;
expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, serializationErrors: 0,
responseValueBytes: Buffer.byteLength(JSON.stringify(rows, (_, value) => typeof value === "bigint" ? value.toString() : value)) });
const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v2")!;
expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, owner: "unattributed", sampledResults: 1,
sampledValueBytes: Buffer.byteLength(text) + 1, measuredValues: 2, unmeasuredValues: 0, incompleteSamples: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-result-value");
expect(f.events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1, inFlightAtEnd: 0,
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false });
@@ -106,10 +110,91 @@ describe("bounded database response capture", () => {
{ name: "sentinel-column-name" }, { name: "b" }, { name: "n" }, { name: "empty" } ] });
expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ responseValueBytes: Buffer.byteLength(text) + 1 + 2, estimatedValues: 0 });
expect(f.events[0]).toMatchObject({ sampledValueBytes: Buffer.byteLength(text) + 1 + 2, measuredValues: 4, unmeasuredValues: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-driver-text");
expect(JSON.stringify(f.events)).not.toContain("sentinel-column-name");
});
it("counts every result but sizes only the first and every sixteenth successful result per group", () => {
const f = fixture();
for (let index = 0; index < 33; index += 1) {
const rows = driverRows([["abc", 12]]);
if (index % 16 !== 0) Object.defineProperty(rows[0], 0, { get() { throw new Error("unsampled cell read"); } });
expect(withDatabaseOperation("fixture.owner", () => f.capture.execute(() => rows, metadata))).toBe(rows);
}
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "fixture.owner", calls: 33, rows: 33, sampledResults: 3,
sampledRows: 3, sampledValueBytes: 15, measuredValues: 6, unmeasuredValues: 0, incompleteSamples: 0 });
expect(f.events[0].totalWallMs).toBeGreaterThanOrEqual(0);
expect(f.events[0].maxWallMs).toBeLessThanOrEqual(Number(f.events[0].totalWallMs));
});
it("bounds sampled cells, string scanning, and bytes without traversing large or complex values", () => {
const f = fixture();
const rows = driverRows([["x".repeat(1_000_000), { large: "private-json" }, ...Array.from({ length: 300 }, () => "y".repeat(16_384))]]);
Object.defineProperty(rows[0], 256, { get() { throw new Error("cell budget exceeded"); } });
expect(f.capture.execute(() => rows, metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ sampledResults: 1, sampledValueBytes: 65_536, measuredValues: 4,
unmeasuredValues: 298, incompleteSamples: 1 });
expect(JSON.stringify(f.events)).not.toContain("private-json");
const cellBound = fixture(), cells = driverRows([Array.from({ length: 257 }, () => null)]);
Object.defineProperty(cells[0], 256, { get() { throw new Error("cell budget exceeded"); } });
cellBound.capture.execute(() => cells, metadata);
cellBound.capture.flush();
expect(cellBound.events[0]).toMatchObject({ sampledValueBytes: 0, measuredValues: 256, unmeasuredValues: 1, incompleteSamples: 1 });
});
it("restores nested owners and uses the direct path without an active capture", () => {
const returned = Promise.resolve("unchanged");
expect(withDatabaseOperation("disabled", () => returned)).toBe(returned);
const f = fixture();
withDatabaseOperation("outer", () => {
withDatabaseOperation("inner", () => f.capture.execute(() => driverRows([[1]]), metadata));
f.capture.execute(() => driverRows([[2]]), metadata);
});
f.now(epoch + 60_001);
withDatabaseOperation("expired", () => {
const next = fixture();
next.capture.execute(() => driverRows([[3]]), metadata);
next.capture.flush();
expect(next.events[0]).toMatchObject({ owner: "unattributed" });
});
f.capture.flush();
expect(f.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["inner", "outer"]);
});
it("retains an armed operation across the capture start without counting earlier queries", async () => {
let clock = epoch;
const events: Array<Record<string, unknown>> = [];
const capture = createDatabaseResponseCapture({ start: epoch + 1_000, end: epoch + 60_000 }, event => events.push(event), () => clock);
captures.push(capture);
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const pending = withDatabaseOperation("started.before.window", async () => {
capture.execute(() => driverRows([[1]]), metadata);
await gate;
return capture.execute(() => driverRows([[2]]), metadata);
});
clock = epoch + 2_000;
release();
await pending;
capture.flush();
expect(events[0]).toMatchObject({ owner: "started.before.window", calls: 1, rows: 1 });
expect(events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
});
it("keeps overlapping capture scopes but clears retained owners when the last capture closes", () => {
const first = fixture(), second = fixture();
let destination = second.capture;
const retained = withDatabaseOperation("previous.capture", () => AsyncLocalStorage.bind(() =>
destination.execute(() => driverRows([[1]]), metadata)));
first.capture.flush();
retained();
second.capture.flush();
expect(second.events[0]).toMatchObject({ owner: "previous.capture", calls: 1 });
const later = fixture();
destination = later.capture;
withDatabaseOperation("new.capture", () => destination.execute(() => driverRows([[2]]), metadata));
retained();
later.capture.flush();
expect(later.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["new.capture", "unattributed"]);
});
it("does not alter synchronous or asynchronous errors or retry them", async () => {
const f = fixture(), error = new Error("sentinel-private-error");
const sync = vi.fn(() => { throw error; }), asyncFailure = vi.fn(() => Promise.reject(error));
@@ -117,14 +202,17 @@ describe("bounded database response capture", () => {
await expect(f.capture.execute(asyncFailure, metadata)).rejects.toBe(error);
expect(sync).toHaveBeenCalledTimes(1); expect(asyncFailure).toHaveBeenCalledTimes(1);
f.capture.flush();
expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, responseValueBytes: 0 });
expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, sampledValueBytes: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-private-error");
});
it("keeps unserializable results and logging failures out of the query outcome", () => {
it("does not serialize complex values or let logging failures change query outcomes", () => {
const f = fixture(), circular: Record<string, unknown> = {}; circular.self = circular;
expect(f.capture.execute(() => circular, metadata)).toBe(circular);
const toJSON = vi.fn(() => { throw new Error("must not serialize"); });
const rows = driverRows([[circular, { toJSON }]]);
expect(f.capture.execute(() => rows, metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ calls: 1, serializationErrors: 1 });
expect(f.events[0]).toMatchObject({ calls: 1, sampledValueBytes: 0, unmeasuredValues: 2, incompleteSamples: 1 });
expect(toJSON).not.toHaveBeenCalled();
const brokenLogger = fixture(() => { throw new Error("logger fixture"); });
expect(brokenLogger.capture.execute(() => circular, metadata)).toBe(circular);
expect(() => brokenLogger.capture.flush()).not.toThrow();
@@ -152,12 +240,12 @@ describe("bounded database response capture", () => {
expect(await result).toBe(rows);
f.capture.flush();
expect(f.events).toHaveLength(1);
expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0 });
expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, sampledValueBytes: 0 });
expect(JSON.stringify(f.events)).not.toContain("late-before-flush-sentinel");
});
it("bounds cardinality without dropping response-byte totals", () => {
it("bounds cardinality while distinguishing sampled bytes from complete call and row counts", () => {
const f = fixture();
for (let i = 0; i < 2200; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] });
for (let i = 0; i < 2200; i += 1) f.capture.execute(() => driverRows([[i]]), { ...metadata, callers: [`services/f${i}.ts:1:1`] });
f.capture.flush();
const summary = f.events.at(-1)!;
expect(summary).toMatchObject({ startedQueries: 2200, completedQueries: 2200, groups: 2048, overflowCalls: 153,
@@ -169,7 +257,8 @@ describe("bounded database response capture", () => {
expect(entries.filter(entry => entry.groupPage === page)).toHaveLength(128);
}
expect(entries.reduce((sum, entry) => sum + Number(entry.calls), 0)).toBe(2200);
expect(summary.responseValueBytes).toBe(Array.from({ length: 2200 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0));
expect(summary).toMatchObject({ rows: 2200, sampledResults: 2057, sampleEvery: 16 });
expect(summary.sampledValueBytes).toBe(entries.reduce((sum, entry) => sum + Number(entry.sampledValueBytes), 0));
});
it.each([0, 129])("numbers every page including an empty or partial final page (%s)", count => {
const f = fixture();
packages/db/src/response-capture.ts
+89 -36
View File
@@ -3,11 +3,27 @@ import { randomUUID } from "node:crypto";
import { getTableName, is, Table } from "drizzle-orm";
type Metadata = { operation: string; table: string; callers: string[] };
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; queryErrors: number; serializationErrors: number; estimatedValues: number };
type Entry = Metadata & { owner: string; calls: number; rows: number; queryErrors: number; totalWallMs: number; maxWallMs: number;
sampledResults: number; sampledRows: number; sampledValueBytes: number; maxSampledValueBytes: number;
measuredValues: number; unmeasuredValues: number; incompleteSamples: number; sizingWallMs: number };
type Event = Record<string, unknown>;
type Method = (this: unknown, ...args: unknown[]) => unknown;
export type ResponseCaptureWindow = { start: number; end: number };
const GROUP_PAGE_SIZE = 128;
const SAMPLE_EVERY = 16;
const MAX_SAMPLE_CELLS = 256;
const MAX_SAMPLE_STRING_LENGTH = 16_384;
const MAX_SAMPLE_STRING_UNITS = 65_536;
const MAX_SAMPLE_BYTES = 65_536;
let operationScope = new AsyncLocalStorage<string>();
const captureWindows = new Set<() => boolean>();
export function withDatabaseOperation<T>(owner: string, action: () => T): T {
for (const armed of captureWindows) {
if (armed()) return operationScope.run(owner, action);
}
return action();
}
export function responseCaptureWindow(value: string | undefined, now = Date.now()): ResponseCaptureWindow | undefined {
if (!value) return;
@@ -32,28 +48,37 @@ export function publicDatabaseCallers(stack: string | undefined): string[] {
return callers;
}
function responseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
function responseValueSize(value: unknown): { bytes: number; measuredValues: number; unmeasuredValues: number } {
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
if (!Array.isArray(value) || !Array.isArray(columns)) {
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
return { bytes: 0, measuredValues: 0, unmeasuredValues: 1 };
}
let bytes = 0, estimatedValues = 0;
let bytes = 0, measuredValues = 0, visitedValues = 0, stringUnits = 0;
const totalValues = value.length * columns.length;
if (columns.length === 0) return { bytes: 0, measuredValues: 0, unmeasuredValues: 0 };
for (const row of value) {
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
for (const cell of cells) {
if (cell === null || cell === undefined) continue;
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
else if (typeof cell === "boolean") bytes += 1;
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
else {
estimatedValues += 1;
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
for (let index = 0; index < columns.length; index += 1) {
if (visitedValues >= MAX_SAMPLE_CELLS || bytes >= MAX_SAMPLE_BYTES) {
return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues };
}
visitedValues += 1;
const cell = Array.isArray(row) ? row[index] : row[columns[index].name];
let size: number;
if (cell === null || cell === undefined) size = 0;
else if (typeof cell === "string") {
if (cell.length > MAX_SAMPLE_STRING_LENGTH || stringUnits + cell.length > MAX_SAMPLE_STRING_UNITS) continue;
stringUnits += cell.length;
size = Buffer.byteLength(cell);
} else if (typeof cell === "boolean") size = 1;
else if (typeof cell === "number") size = Buffer.byteLength(String(cell));
else if (typeof cell === "bigint" && cell >= -9_223_372_036_854_775_808n && cell <= 9_223_372_036_854_775_807n) size = Buffer.byteLength(String(cell));
else continue;
if (bytes + size > MAX_SAMPLE_BYTES) continue;
measuredValues += 1;
bytes += size;
}
}
return { bytes, estimatedValues };
return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues };
}
export function createDatabaseResponseCapture(
@@ -71,32 +96,45 @@ export function createDatabaseResponseCapture(
const send = (event: Event) => { try { emit(event); } catch {} };
const expired = () => closed || now() >= window.end;
const active = () => !expired() && now() >= window.start;
const armed = () => !expired();
captureWindows.add(armed);
const flush = () => {
if (closed) return;
closed = true;
clearTimeout(timer);
captureWindows.delete(armed);
if (captureWindows.size === 0) {
operationScope.disable();
operationScope = new AsyncLocalStorage<string>();
}
scope.disable();
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v1", captureId: id,
const ordered = [...entries.values()].sort((a, b) => b.calls - a.calls);
for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v2", captureId: id,
groupIndex, groupPage: Math.floor(groupIndex / GROUP_PAGE_SIZE),
operation: entry.operation, table: entry.table, callers: entry.callers, calls: entry.calls, rows: entry.rows,
responseValueBytes: entry.bytes, maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors });
send({ event: "paperclip.db_response_capture.summary.v1", captureId: id,
...entry });
send({ event: "paperclip.db_response_capture.summary.v2", captureId: id,
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
closedAt: new Date(now()).toISOString(),
startedQueries: started, completedQueries: completed, inFlightAtEnd: inFlight, groups: entries.size,
groupPageSize: GROUP_PAGE_SIZE, groupPages: Math.ceil(ordered.length / GROUP_PAGE_SIZE),
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
overflowCalls, rows: ordered.reduce((sum, entry) => sum + entry.rows, 0),
sampledResults: ordered.reduce((sum, entry) => sum + entry.sampledResults, 0),
sampledValueBytes: ordered.reduce((sum, entry) => sum + entry.sampledValueBytes, 0),
sizingWallMs: ordered.reduce((sum, entry) => sum + entry.sizingWallMs, 0),
sampleEvery: SAMPLE_EVERY, sampling: "first-and-every-16th-successful-result-per-group",
maxSampleCells: MAX_SAMPLE_CELLS, maxSampleStringLength: MAX_SAMPLE_STRING_LENGTH,
maxSampleStringUnits: MAX_SAMPLE_STRING_UNITS, maxSampleBytes: MAX_SAMPLE_BYTES,
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
measurementBoundary: "postgres-js results before Drizzle column decoding",
wallTimeBoundary: "first query consumption to driver result; includes queue, database, network and event-loop waits; not CPU",
byteMeasurement: "sampled primitive values only; complex/oversized/budget-exceeding values unmeasured; no extrapolation",
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false });
entries.clear();
};
const timer = setTimeout(flush, Math.max(0, window.end - now()));
timer.unref();
function finish(metadata: Metadata, value: unknown, failed: boolean) {
function finish(metadata: Metadata & { owner: string }, value: unknown, failed: boolean, startedAt: number) {
if (!active()) return;
inFlight -= 1;
completed += 1;
@@ -104,38 +142,52 @@ export function createDatabaseResponseCapture(
let key = JSON.stringify(metadata);
if (!entries.has(key) && entries.size >= 2047) {
overflowCalls += 1;
metadata = { operation: "other", table: "other", callers: [] };
metadata = { owner: "other", operation: "other", table: "other", callers: [] };
key = "overflow";
}
let entry = entries.get(key);
if (!entry) {
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
entry = { ...metadata, calls: 0, rows: 0, queryErrors: 0, totalWallMs: 0, maxWallMs: 0,
sampledResults: 0, sampledRows: 0, sampledValueBytes: 0, maxSampledValueBytes: 0,
measuredValues: 0, unmeasuredValues: 0, incompleteSamples: 0, sizingWallMs: 0 };
entries.set(key, entry);
}
entry.calls += 1;
const wallMs = performance.now() - startedAt;
entry.totalWallMs += wallMs;
entry.maxWallMs = Math.max(entry.maxWallMs, wallMs);
if (failed) { entry.queryErrors += 1; return; }
entry.rows += Array.isArray(value) ? value.length : 0;
if ((entry.calls - entry.queryErrors - 1) % SAMPLE_EVERY !== 0) return;
entry.sampledResults += 1;
entry.sampledRows += Array.isArray(value) ? value.length : 0;
const sizingStartedAt = performance.now();
try {
const size = responseValueSize(value);
entry.bytes += size.bytes;
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
entry.estimatedValues += size.estimatedValues;
} catch { entry.serializationErrors += 1; }
entry.sampledValueBytes += size.bytes;
entry.maxSampledValueBytes = Math.max(entry.maxSampledValueBytes, size.bytes);
entry.measuredValues += size.measuredValues;
entry.unmeasuredValues += size.unmeasuredValues;
if (size.unmeasuredValues) entry.incompleteSamples += 1;
} catch { entry.incompleteSamples += 1; }
finally { entry.sizingWallMs += performance.now() - sizingStartedAt; }
} catch {}
}
function execute(action: () => unknown, metadata: Metadata) {
if (!active()) return action();
inFlight += 1; started += 1;
const owned = { ...metadata, owner: operationScope.getStore() ?? "unattributed" };
const startedAt = performance.now();
let result: unknown;
try { result = action(); }
catch (error) { finish(metadata, undefined, true); throw error; }
catch (error) { finish(owned, undefined, true, startedAt); throw error; }
let asynchronous = false;
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
if (asynchronous) {
return Promise.resolve(result).then(value => { finish(metadata, value, false); return value; },
error => { finish(metadata, undefined, true); throw error; });
return Promise.resolve(result).then(value => { finish(owned, value, false, startedAt); return value; },
error => { finish(owned, undefined, true, startedAt); throw error; });
}
finish(metadata, result, false);
finish(owned, result, false, startedAt);
return result;
}
return { window, roots, queries, factories, clients, driverQueries, scope, active, expired, execute, flush };
@@ -240,7 +292,7 @@ export function installDatabaseResponseCapture<T extends object>(db: T, schema:
}
return scoped ? { ...scoped, table: scoped.table === "other" ? table : scoped.table }
: { operation: text.match(/^\s*(select|insert|update|delete)\b/i)?.[1]?.toLowerCase() ?? "execute",
table, callers: publicDatabaseCallers(new Error().stack) };
table, callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] };
}
function client(value: unknown) {
if ((!object(value) && typeof value !== "function") || capture!.clients.has(value)) return;
@@ -277,7 +329,8 @@ export function installDatabaseResponseCapture<T extends object>(db: T, schema:
try {
if (method === "with") { root(result); return result; }
const operation = method.startsWith("select") ? "select" : method;
const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), callers: publicDatabaseCallers(new Error().stack) };
const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]),
callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] };
if (operation === "select") factory(result, "from", metadata, true);
else if (method === "insert") factory(result, "values", metadata);
else if (method === "update") factory(result, "set", metadata);
packages/db/src/test-embedded-postgres.test.ts
+69 -1
View File
@@ -2,8 +2,9 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import postgres from "postgres";
import { afterEach, describe, expect, it } from "vitest";
import { afterAll, afterEach, describe, expect, it } from "vitest";
import { inspectMigrations } from "./client.js";
import { drainTestPostgresOwners, TestPostgresShutdownError } from "./test-postgres-owner.js";
import {
__embeddedPostgresStartMaxAttemptsForTests as MAX_ATTEMPTS,
__embeddedPostgresTemplateRootDirForTests as templateRootDir,
@@ -18,6 +19,12 @@ import {
startEmbeddedPostgresTestDatabase,
} from "./test-embedded-postgres.js";
const originalAdminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
afterAll(() => {
if (originalAdminUrl !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = originalAdminUrl;
});
// A fake embedded-postgres constructor. It records every constructed instance so
// the test can assert the retry uses a fresh port and a fresh data directory each
// attempt. `start()` emits the same output the real cluster writes for a port
@@ -266,6 +273,29 @@ describeWithPostgres("startEmbeddedPostgresTestDatabase template path", () => {
else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting;
});
it("does not start a replacement when a copied cluster cannot stop", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
const ready = await startEmbeddedPostgresTestDatabase("paperclip-template-stop-ready-");
await ready.cleanup();
const { ctor, constructed } = makeFakeCtor(Number.POSITIVE_INFINITY);
let stops = 0;
class FailedStopPostgres extends ctor {
async stop(): Promise<void> {
if (++stops === 1) throw new Error("owned stop failed");
}
}
__setEmbeddedPostgresCtorProviderForTests(async () => FailedStopPostgres);
try {
await expect(startEmbeddedPostgresTestDatabase("paperclip-template-stop-fail-")).rejects.toBeInstanceOf(TestPostgresShutdownError);
expect(constructed).toHaveLength(1);
expect(fs.existsSync(constructed[0].databaseDir)).toBe(true);
} finally {
await drainTestPostgresOwners();
__setEmbeddedPostgresCtorProviderForTests(null);
for (const options of constructed) fs.rmSync(path.dirname(options.databaseDir), { recursive: true, force: true });
}
});
it("publishes a complete template and starts suites from a copy of it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
// This test covers the template path, so it enables the cache even when
// the suite runs under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0. The first
@@ -452,6 +482,44 @@ describe("support probe when the template cannot be built", () => {
resetTemplateState();
});
it("fails the suite when PostgreSQL cannot allocate semaphores", async () => {
const original = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0";
const { ctor, constructed } = makeFakeCtor(Number.POSITIVE_INFINITY);
class ExhaustedPostgres extends ctor {
async start(): Promise<void> {
throw new Error("could not create semaphores: No space left on device (semget)");
}
}
__setEmbeddedPostgresCtorProviderForTests(async () => ExhaustedPostgres);
resetTemplateState();
try {
await expect(getEmbeddedPostgresTestSupport()).rejects.toThrow("could not create semaphores");
expect(constructed).toHaveLength(1);
} finally {
if (original === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = original;
}
});
it("still reports an explicitly unsupported platform as unsupported", async () => {
const original = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0";
__setEmbeddedPostgresCtorProviderForTests(async () => {
throw new Error('Unsupported arch "test-cpu" for platform "test-os"');
});
resetTemplateState();
try {
expect(await getEmbeddedPostgresTestSupport()).toMatchObject({
supported: false,
reason: expect.stringContaining('Unsupported arch "test-cpu"'),
});
} finally {
if (original === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = original;
}
});
it("falls through to the fresh probe instead of reporting the host as unsupported", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => {
const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
packages/db/src/test-embedded-postgres.ts
+55 -98
View File
@@ -1,3 +1,4 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import { createRequire } from "node:module";
@@ -5,12 +6,16 @@ import net from "node:net";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { applyPendingMigrations, closeRegisteredClients, ensurePostgresDatabase } from "./client.js";
import {
createEmbeddedPostgresLogBuffer,
formatEmbeddedPostgresError,
} from "./embedded-postgres-error.js";
import { prepareEmbeddedPostgresNativeRuntime } from "./embedded-postgres-native.js";
import { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js";
import { ownTestPostgres, TestPostgresShutdownError, type TestPostgresInstance } from "./test-postgres-owner.js";
import { checkTestPostgresAdmin, startTestPostgresDatabase, testPostgresAdminUrl } from "./test-postgres-admin.js";
// Time budget (ms) for a vitest test in the embedded-Postgres cost class: a
// test that starts an embedded Postgres cluster and runs migrations. Measured
@@ -20,11 +25,7 @@ import { prepareEmbeddedPostgresNativeRuntime } from "./embedded-postgres-native
// passes while a genuine hang still fails fast.
export const EMBEDDED_POSTGRES_TEST_TIMEOUT_MS = 90_000;
type EmbeddedPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
};
type EmbeddedPostgresInstance = ReturnType<typeof ownTestPostgres>;
type EmbeddedPostgresCtor = new (opts: {
databaseDir: string;
@@ -35,7 +36,7 @@ type EmbeddedPostgresCtor = new (opts: {
initdbFlags?: string[];
onLog?: (message: unknown) => void;
onError?: (message: unknown) => void;
}) => EmbeddedPostgresInstance;
}) => TestPostgresInstance;
export type EmbeddedPostgresTestSupport = {
supported: boolean;
@@ -65,9 +66,9 @@ function getReservedTestPorts(): Set<number> {
type EmbeddedPostgresCtorProvider = () => Promise<EmbeddedPostgresCtor>;
async function loadEmbeddedPostgresCtor(): Promise<EmbeddedPostgresCtor> {
const mod = await import("embedded-postgres");
const mod = await loadWithoutEmbeddedPostgresExitHooks(() => import("embedded-postgres"));
await prepareEmbeddedPostgresNativeRuntime();
return mod.default as EmbeddedPostgresCtor;
return mod.default as unknown as EmbeddedPostgresCtor;
}
let embeddedPostgresCtorProvider: EmbeddedPostgresCtorProvider = loadEmbeddedPostgresCtor;
@@ -122,7 +123,12 @@ const EMBEDDED_POSTGRES_INITDB_FLAGS = ["--encoding=UTF8", "--locale=C", "--lc-m
async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) {
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix));
return await createEmbeddedPostgresTestInstanceAt(dataDir);
try {
return await createEmbeddedPostgresTestInstanceAt(dataDir);
} catch (error) {
await cleanupEmbeddedPostgresTestDirs(dataDir);
throw error;
}
}
// Wraps an embedded-postgres instance around `dataDir`. The caller decides
@@ -136,7 +142,7 @@ async function createEmbeddedPostgresTestInstanceAt(dataDir: string) {
// The `start()` rejection carries an empty message, so we capture the output
// in a bounded buffer and surface it in the thrown error.
const logBuffer = createEmbeddedPostgresLogBuffer();
const instance = new EmbeddedPostgres({
const instance = ownTestPostgres(new EmbeddedPostgres({
databaseDir: dataDir,
user: EMBEDDED_POSTGRES_USER,
password: EMBEDDED_POSTGRES_PASSWORD,
@@ -145,7 +151,7 @@ async function createEmbeddedPostgresTestInstanceAt(dataDir: string) {
initdbFlags: EMBEDDED_POSTGRES_INITDB_FLAGS,
onLog: (message) => logBuffer.append(message),
onError: (message) => logBuffer.append(message),
});
}), dataDir);
return { dataDir, port, instance, getRecentLogs: () => logBuffer.getRecentLogs() };
}
@@ -158,63 +164,24 @@ function databaseConnectionStringFor(port: number): string {
return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/paperclip`;
}
function cleanupEmbeddedPostgresTestDirs(dataDir: string) {
fs.rmSync(dataDir, { recursive: true, force: true });
async function cleanupEmbeddedPostgresTestDirs(dataDir: string): Promise<void> {
if (process.platform !== "win32") {
try {
await promisify(execFile)("/bin/rm", ["-rf", "--", dataDir], { maxBuffer: 64 * 1024 });
return;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
}
}
await fs.promises.rm(dataDir, { recursive: true, force: true });
}
// Upper bound (ms) on how long we wait for the embedded Postgres cluster to
// stop gracefully before abandoning the wait and returning from the hook.
const EMBEDDED_POSTGRES_STOP_TIMEOUT_MS = 5000;
// `embedded-postgres@18.1.0-beta.16` exposes only `stop(): Promise<void>` — no
// shutdown-mode argument. Internally it SIGINTs the postgres process (already
// PostgreSQL "fast shutdown") and resolves *only* on the child's `exit` event,
// with no time bound of its own. Under the loaded serial server shard a slow
// shutdown checkpoint can push that past vitest's hookTimeout and hang the
// afterAll hook. So we bound the graceful stop: if it overruns, we stop waiting
// and return so the hook completes. The SIGINT has already been delivered, so
// the abandoned process still exits on its own (and again when the runner exits).
// Errors are swallowed, matching prior behavior.
//
// `cleanupFn` (data-dir reclaim) is chained on the raw `stop()` promise, not on
// the timeout race, so the disposable data dir is removed *only after* `stop()`
// actually settles — i.e. once the child Postgres process has exited. Removing
// it on the timeout path would pull the data files out from under a still-running
// cluster and provoke checkpoint / WAL I/O errors. In the fast path `cleanupFn`
// has run by the time this resolves; in the timeout path it runs asynchronously
// once the abandoned process finally exits.
async function stopEmbeddedPostgresBounded(
instance: EmbeddedPostgresInstance | null,
cleanupFn?: () => void,
cleanupFn?: () => Promise<void>,
): Promise<void> {
if (!instance) {
cleanupFn?.();
return;
}
let timer: ReturnType<typeof setTimeout> | undefined;
const stopped = instance
.stop()
.catch(() => {
// Swallow shutdown errors — the data dir is reclaimed regardless.
})
.finally(() => {
try {
cleanupFn?.();
} catch {
// Best-effort reclaim; ignore removal errors.
}
});
try {
await Promise.race([
stopped,
new Promise<void>((resolve) => {
timer = setTimeout(resolve, EMBEDDED_POSTGRES_STOP_TIMEOUT_MS);
timer.unref?.();
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
await instance?.stop();
await cleanupFn?.();
}
// Upper bound on start attempts. `getAvailablePort` uses a check-then-use probe:
@@ -251,6 +218,7 @@ async function startEmbeddedPostgresWithRetry(tempDirPrefix: string): Promise<{
await stopEmbeddedPostgresBounded(created.instance, () =>
cleanupEmbeddedPostgresTestDirs(created.dataDir),
);
if (!/address already in use/i.test(lastError.message)) throw lastError;
}
}
@@ -306,10 +274,6 @@ const TEMPLATE_ROOT_DIR_NAME = "paperclip-embedded-postgres-templates";
// Templates whose key no longer matches are pruned once they have not been
// used for this long. Every reuse refreshes the template's timestamp.
const TEMPLATE_PRUNE_AGE_MS = 7 * 24 * 60 * 60 * 1000;
// Publishing a template requires the builder cluster to have exited for real,
// so its data files are quiescent before the rename. Give that stop more room
// than the per-suite bounded stop; a template build happens once per host.
const TEMPLATE_BUILD_STOP_TIMEOUT_MS = 30_000;
export type EmbeddedPostgresStartMode = "template" | "fresh";
@@ -482,18 +446,10 @@ function pruneStaleTemplates(rootDir: string, keepDir: string): void {
// Waits for the raw `stop()` so the builder cluster has really exited before
// its data directory is published. Returns false if the stop overran.
async function stopForTemplatePublish(instance: EmbeddedPostgresInstance): Promise<boolean> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
instance.stop().then(() => true, () => false),
new Promise<boolean>((resolve) => {
timer = setTimeout(() => resolve(false), TEMPLATE_BUILD_STOP_TIMEOUT_MS);
timer.unref?.();
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
await instance.stopGracefully();
return true;
} catch { return false; }
}
// Builds a migrated cluster and publishes its data directory at `templateDir`.
@@ -533,12 +489,6 @@ async function buildEmbeddedPostgresTemplate(templateDir: string): Promise<boole
let templateDirPromise: Promise<string | null> | null = null;
// Resolves the template data directory for this host, building it on first
// use. Resolves null when the cache is disabled, the cache root is not private
// to this user, or the template could not be built or published; every caller
// then takes the fresh path, which reports the real error itself (a cluster
// that cannot start, a migration that cannot apply). Never rejects, so a
// broken cache can never turn into `supported: false` and skip a suite.
async function getEmbeddedPostgresTemplateDir(): Promise<string | null> {
if (isTemplateCacheDisabled()) return null;
if (!templateDirPromise) {
@@ -554,6 +504,7 @@ async function getEmbeddedPostgresTemplateDir(): Promise<string | null> {
const built = await buildEmbeddedPostgresTemplate(templateDir);
return built && isUsableTemplate(templateDir) ? templateDir : null;
} catch (error) {
if (error instanceof TestPostgresShutdownError) throw error;
warnTemplateOnce(
"build",
`template build failed, using fresh clusters: ${(error as Error).message}`,
@@ -604,6 +555,7 @@ async function startEmbeddedPostgresFromTemplateWithRetry(
await stopEmbeddedPostgresBounded(created?.instance ?? null, () =>
cleanupEmbeddedPostgresTestDirs(rootDir),
);
if (!/address already in use/i.test(lastError.message)) throw lastError;
}
}
@@ -636,12 +588,6 @@ export function __resetEmbeddedPostgresTemplateStateForTests(): void {
}
async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSupport> {
// A published template proves this host can run the cluster: it was built
// here. Reusing it makes the probe free for every file after the first.
// Building it costs one cluster start, the same as the throwaway probe did.
// When the template cannot be built or used, the throwaway probe below
// decides, so only a cluster that cannot start reports `supported: false`;
// a migration that cannot apply fails the suite loudly on the fresh path.
if ((await getEmbeddedPostgresTemplateDir()) !== null) return { supported: true };
let started: { dataDir: string; instance: EmbeddedPostgresInstance } | null = null;
@@ -650,12 +596,13 @@ async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSuppo
started = await startEmbeddedPostgresWithRetry("paperclip-embedded-postgres-probe-");
return { supported: true };
} catch (error) {
return {
supported: false,
reason: formatEmbeddedPostgresError(error, {
fallbackMessage: "embedded Postgres startup failed",
}).message,
};
const failure = formatEmbeddedPostgresError(error, {
fallbackMessage: "embedded Postgres startup failed",
});
if (/Unsupported (?:arch|platform) "/.test(failure.message)) {
return { supported: false, reason: failure.message };
}
throw failure;
} finally {
if (started) {
const { dataDir, instance } = started;
@@ -665,6 +612,11 @@ async function probeEmbeddedPostgresSupport(): Promise<EmbeddedPostgresTestSuppo
}
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
const adminUrl = testPostgresAdminUrl();
if (adminUrl) {
await checkTestPostgresAdmin(adminUrl);
return { supported: true };
}
if (!embeddedPostgresSupportPromise) {
embeddedPostgresSupportPromise = probeEmbeddedPostgresSupport();
}
@@ -684,6 +636,7 @@ async function startEmbeddedPostgresClusterForSuite(
const started = await startEmbeddedPostgresFromTemplateWithRetry(templateDir, tempDirPrefix);
return { ...started, mode: "template" };
} catch (error) {
if (error instanceof TestPostgresShutdownError) throw error;
// The next caller in this process re-resolves the template: if it was
// pruned it is rebuilt, instead of retrying a path that is gone.
resetTemplateDirCache();
@@ -705,6 +658,8 @@ async function startEmbeddedPostgresClusterForSuite(
export async function startEmbeddedPostgresTestDatabase(
tempDirPrefix: string,
): Promise<EmbeddedPostgresTestDatabase> {
const adminUrl = testPostgresAdminUrl();
if (adminUrl) return startTestPostgresDatabase(adminUrl);
const { port, dataDir, instance, mode } = await startEmbeddedPostgresClusterForSuite(tempDirPrefix);
lastStartMode = mode;
@@ -723,8 +678,10 @@ export async function startEmbeddedPostgresTestDatabase(
// stops can crash the process: the stop kills the backend socket,
// but a queued write on that connection still fires later and finds
// a null socket.
await closeRegisteredClients(connectionString);
await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir));
try { await closeRegisteredClients(connectionString); }
finally {
await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir));
}
},
};
} catch (error) {
packages/db/src/test-postgres-admin.test.ts
+86
View File
@@ -0,0 +1,86 @@
import { randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { createDb } from "./client.js";
import {
__setEmbeddedPostgresCtorProviderForTests,
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestDatabase,
} from "./test-embedded-postgres.js";
const previousAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
const previousTemplate = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
let server: EmbeddedPostgresTestDatabase;
const databases: EmbeddedPostgresTestDatabase[] = [];
const support = await getEmbeddedPostgresTestSupport();
describe.skipIf(!support.supported)("shared PostgreSQL fixtures", () => {
beforeAll(async () => {
server = await startEmbeddedPostgresTestDatabase("paperclip-admin-regression-");
process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = server.connectionString;
}, 90_000);
afterEach(async () => {
__setEmbeddedPostgresCtorProviderForTests(null);
process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = server.connectionString;
if (previousTemplate === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = previousTemplate;
while (databases.length) await databases.pop()!.cleanup();
});
afterAll(async () => {
await server?.cleanup();
if (previousAdmin === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
else process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = previousAdmin;
if (previousTemplate === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = previousTemplate;
});
it("uses one server for isolated databases and closes clients before dropping them", async () => {
__setEmbeddedPostgresCtorProviderForTests(async () => { throw new Error("Must not start another PostgreSQL server"); });
expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true });
const first = await startEmbeddedPostgresTestDatabase("ignored-"); databases.push(first);
const second = await startEmbeddedPostgresTestDatabase("ignored-"); databases.push(second);
expect(first.connectionString).not.toBe(second.connectionString);
const a = createDb(first.connectionString), b = createDb(second.connectionString);
await a.$client`CREATE TABLE isolated_probe (id integer)`;
expect((await b.$client`SELECT to_regclass('isolated_probe') AS name`)[0]?.name).toBeNull();
await first.cleanup();
await first.cleanup();
const control = createDb(server.connectionString);
try {
const name = new URL(first.connectionString).pathname.slice(1);
expect(await control.$client`SELECT datname FROM pg_database WHERE datname = ${name}`).toHaveLength(0);
expect((await b.$client`SELECT 1 AS value`)[0]?.value).toBe(1);
} finally { await control.$client.end(); }
}, 90_000);
it("rejects unavailable configured servers rather than reporting support", async () => {
process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = "postgres://paperclip:paperclip@127.0.0.1:1/postgres";
await expect(getEmbeddedPostgresTestSupport()).rejects.toThrow();
});
it("rejects unsafe templates and treats zero as a disabled template", async () => {
process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = 'postgres"; DROP DATABASE postgres; --';
await expect(startEmbeddedPostgresTestDatabase("ignored-")).rejects.toThrow("Invalid disposable test template name");
process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0";
databases.push(await startEmbeddedPostgresTestDatabase("ignored-"));
}, 90_000);
it("applies pending migrations when a supplied database template is not current", async () => {
const template = `paperclip_test_template_${randomUUID().replaceAll("-", "")}`;
const control = createDb(server.connectionString);
await control.$client.unsafe(`CREATE DATABASE "${template}"`);
try {
process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = template;
const database = await startEmbeddedPostgresTestDatabase("ignored-");
databases.push(database);
const db = createDb(database.connectionString);
expect((await db.$client`SELECT to_regclass('companies') AS name`)[0]?.name).toBe("companies");
} finally {
await control.$client.unsafe(`DROP DATABASE "${template}"`);
await control.$client.end();
}
}, 90_000);
});
packages/db/src/test-postgres-admin.ts
+50
View File
@@ -0,0 +1,50 @@
import { randomUUID } from "node:crypto";
import postgres from "postgres";
import { applyPendingMigrations, closeRegisteredClients } from "./client.js";
export function testPostgresAdminUrl(): string | undefined {
const value = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL?.trim();
if (!value) return undefined;
const url = new URL(value);
if (url.protocol !== "postgres:" && url.protocol !== "postgresql:") {
throw new Error("PAPERCLIP_TEST_POSTGRES_ADMIN_URL must be a PostgreSQL URL");
}
return value;
}
export async function checkTestPostgresAdmin(url: string): Promise<void> {
const sql = postgres(url, { max: 1, connect_timeout: 5, onnotice() {} });
try { await sql`SELECT 1`; }
finally { await sql.end({ timeout: 1 }); }
}
export async function startTestPostgresDatabase(adminUrl: string): Promise<{
connectionString: string;
cleanup(): Promise<void>;
}> {
const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`;
const setting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
const template = setting === "0" ? undefined : setting;
if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) {
throw new Error("Invalid disposable test template name");
}
const admin = postgres(adminUrl, { max: 1, connect_timeout: 5, onnotice() {} });
const url = new URL(adminUrl);
url.pathname = `/${name}`;
const connectionString = url.toString();
let cleaning: Promise<void> | null = null;
const cleanup = () => cleaning ??= (async () => {
try {
await closeRegisteredClients(connectionString, { scope: "database" });
await admin.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
} finally { await admin.end({ timeout: 1 }); }
})();
try {
await admin.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`);
await applyPendingMigrations(connectionString);
return { connectionString, cleanup };
} catch (error) {
await cleanup();
throw error;
}
}
packages/db/src/test-postgres-command.test.ts
+57
View File
@@ -0,0 +1,57 @@
import { execFile, spawn } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { setTimeout } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport } from "./test-embedded-postgres.js";
const previousAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
let supported = false;
try { supported = process.platform !== "win32" && (await getEmbeddedPostgresTestSupport()).supported; }
finally {
if (previousAdmin !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = previousAdmin;
}
const execute = promisify(execFile);
it.skipIf(!supported).each(["SIGTERM", "SIGINT"] as const)("drains the test command's descendants before PostgreSQL on %s", async (signal) => {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-command-cancel-"));
const ready = path.join(directory, "ready"), done = path.join(directory, "done");
const env: NodeJS.ProcessEnv = {
...process.env, TEST_COMMAND_TREE_READY: ready, TEST_COMMAND_TREE_DONE: done,
TMPDIR: directory, TEMP: directory, TMP: directory,
};
delete env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
delete env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
const wrapper = spawn(process.execPath, [
"--import", fileURLToPath(new URL("../node_modules/tsx/dist/loader.mjs", import.meta.url)),
fileURLToPath(new URL("./test-postgres-command.ts", import.meta.url)), "--", process.execPath,
fileURLToPath(new URL("./__fixtures__/test-command-tree.mjs", import.meta.url)), "root",
], { env, stdio: ["ignore", "pipe", "pipe"] });
let output = "";
wrapper.stdout.on("data", (data) => { output = (output + String(data)).slice(-8000); });
wrapper.stderr.on("data", (data) => { output = (output + String(data)).slice(-8000); });
const exited = new Promise<number | null>((resolve, reject) => {
wrapper.once("error", reject);
wrapper.once("exit", (code) => resolve(code));
});
try {
for (let i = 0; i < 1200 && !fs.existsSync(ready) && wrapper.exitCode === null; i++) await setTimeout(25);
expect(fs.existsSync(ready), output).toBe(true);
const descendants = JSON.parse(fs.readFileSync(ready, "utf8")) as { root: number; grandchild: number };
wrapper.kill(signal);
expect(await exited, output).toBe(signal === "SIGINT" ? 130 : 143);
expect(fs.readFileSync(done, "utf8")).toBe("database available during child shutdown");
expect(fs.readdirSync(directory).filter((name) => name.startsWith("paperclip-test-command-"))).toEqual([]);
for (const pid of Object.values(descendants)) {
const result = await execute("/bin/ps", ["-p", String(pid), "-o", "stat="]).catch(() => ({ stdout: "" }));
expect(result.stdout.trim() === "" || result.stdout.trim().startsWith("Z"), `descendant ${pid} still running`).toBe(true);
}
} finally {
if (wrapper.exitCode === null && wrapper.signalCode === null) { wrapper.kill("SIGTERM"); await exited; }
fs.rmSync(directory, { recursive: true, force: true });
}
}, 90_000);
packages/db/src/test-postgres-command.ts
+46
View File
@@ -0,0 +1,46 @@
import path from "node:path";
import { createDb } from "./client.js";
import { computeEmbeddedPostgresTemplateKey, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
import { checkTestPostgresAdmin, testPostgresAdminUrl } from "./test-postgres-admin.js";
import { beforeTestPostgresShutdown } from "./test-postgres-owner.js";
import { startTestProcessTree } from "./test-process-tree.js";
const args = process.argv.slice(2);
if (args[0] === "--") args.shift();
const command = args.shift() ?? process.execPath;
if (command === process.execPath && args.length === 0) args.push(path.resolve("scripts/run-vitest-stable.mjs"));
let server: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
try {
const env = { ...process.env };
let adminUrl = testPostgresAdminUrl();
if (adminUrl) await checkTestPostgresAdmin(adminUrl);
else {
server = await startEmbeddedPostgresTestDatabase("paperclip-test-command-");
const url = new URL(server.connectionString);
url.pathname = "/postgres";
adminUrl = url.toString();
if (env.PAPERCLIP_TEST_POSTGRES_TEMPLATE !== "0") {
const template = `paperclip_test_template_${computeEmbeddedPostgresTemplateKey()}`;
const admin = createDb(adminUrl, { maxConnections: 1 });
try { await admin.$client.unsafe(`CREATE DATABASE "${template}" TEMPLATE "paperclip"`); }
finally { await admin.$client.end({ timeout: 1 }); }
env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = template;
}
}
env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = adminUrl;
console.log("[test:postgres] Each suite gets a disposable database on one test PostgreSQL server");
const child = startTestProcessTree(command, args, env);
const removeShutdownTask = beforeTestPostgresShutdown({
...child,
afterStop: async () => { await server?.cleanup(); },
});
try {
process.exitCode = await child.wait();
} finally {
try { await child.stop(); }
finally { removeShutdownTask(); }
}
} finally {
await server?.cleanup();
}
packages/db/src/test-postgres-owner-process.test.ts
+54
View File
@@ -0,0 +1,54 @@
import { fork } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { setTimeout } from "node:timers/promises";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport } from "./test-embedded-postgres.js";
const configuredAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
let embeddedSupported = false;
try { embeddedSupported = process.platform !== "win32" && (await getEmbeddedPostgresTestSupport()).supported; }
finally {
if (configuredAdmin !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = configuredAdmin;
}
function exists(pid: number): boolean {
try { process.kill(pid, 0); return true; }
catch (error) { return (error as NodeJS.ErrnoException).code !== "ESRCH"; }
}
it.skipIf(!embeddedSupported).each(["exit", "SIGTERM"])("stops the owned PostgreSQL process when its worker receives %s", async (mode) => {
const child = fork(fileURLToPath(new URL("./__fixtures__/test-postgres-exit.mjs", import.meta.url)), [mode], {
execArgv: ["--import", fileURLToPath(new URL("../node_modules/tsx/dist/loader.mjs", import.meta.url))],
stdio: ["ignore", "pipe", "pipe", "ipc"],
});
let output = "";
child.stdout?.on("data", (data) => { output = (output + String(data)).slice(-8000); });
child.stderr?.on("data", (data) => { output = (output + String(data)).slice(-8000); });
const exited = new Promise<number | null>((resolve, reject) => {
child.once("error", reject);
child.once("exit", (code) => resolve(code));
});
let server: { pid: number; dataDir: string } | undefined;
try {
const ready = await new Promise<{ pid: number; dataDir: string }>((resolve, reject) => {
child.once("message", (value) => resolve(value as { pid: number; dataDir: string }));
child.once("exit", () => reject(new Error(`Worker exited before readiness: ${output}`)));
});
server = ready;
if (mode === "SIGTERM") child.kill("SIGTERM");
expect(await exited, output).toBe(mode === "exit" ? 0 : 143);
for (let i = 0; i < 200 && exists(ready.pid); i++) await setTimeout(25);
expect(exists(ready.pid), output).toBe(false);
} finally {
if (child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await exited; }
if (server && exists(server.pid)) {
const contents = fs.readFileSync(path.join(server.dataDir, "postmaster.pid"), "utf8");
if (Number(contents.split("\n")[0]) === server.pid) process.kill(server.pid, "SIGINT");
for (let i = 0; i < 200 && exists(server.pid); i++) await setTimeout(25);
}
if (server && !exists(server.pid)) fs.rmSync(server.dataDir, { recursive: true, force: true });
}
}, 30_000);
packages/db/src/test-postgres-owner.test.ts
+105
View File
@@ -0,0 +1,105 @@
import { EventEmitter } from "node:events";
import type { ChildProcess } from "node:child_process";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ownTestPostgres } from "./test-postgres-owner.js";
function fixture() {
const child = Object.assign(new EventEmitter(), {
exitCode: null as number | null,
signalCode: null as NodeJS.Signals | null,
kill: vi.fn(() => true),
});
const raw = {
initialise: vi.fn(async () => {}),
start: vi.fn(async () => {}),
stop: vi.fn(async () => {}),
process: child as unknown as ChildProcess,
};
const owner = ownTestPostgres(raw);
function exit() { child.exitCode = 0; child.emit("exit", 0, null); }
return { child, raw, owner, exit };
}
afterEach(() => { vi.useRealTimers(); });
describe.skipIf(process.platform === "win32")("test PostgreSQL ownership", () => {
it("waits for fast shutdown and makes concurrent cleanup idempotent", async () => {
const before = process.listenerCount("exit");
const f = fixture();
const stop = f.owner.stop();
expect(f.owner.stop()).toBe(stop);
expect(f.child.kill).toHaveBeenCalledWith("SIGINT");
f.exit();
await stop;
await f.owner.stop();
expect(f.child.kill).toHaveBeenCalledTimes(1);
expect(process.listenerCount("exit")).toBe(before);
});
it("does not call the dependency's stop after the process already exited", async () => {
const f = fixture();
f.exit();
await f.owner.stop();
expect(f.raw.stop).not.toHaveBeenCalled();
expect(f.child.kill).not.toHaveBeenCalled();
});
it("uses immediate PostgreSQL shutdown when fast shutdown times out", async () => {
vi.useFakeTimers();
const f = fixture();
const stop = f.owner.stop();
await vi.advanceTimersByTimeAsync(5_000);
expect(f.child.kill.mock.calls).toEqual([["SIGINT"], ["SIGQUIT"]]);
f.exit();
await stop;
expect(f.child.kill).not.toHaveBeenCalledWith("SIGKILL");
});
it("reports a shutdown failure instead of declaring the process stopped", async () => {
vi.useFakeTimers();
const f = fixture();
const failure = expect(f.owner.stop()).rejects.toThrow("data directory is retained");
await vi.advanceTimersByTimeAsync(10_000);
await failure;
f.exit();
await f.owner.stop();
});
it("keeps one exit hook while two servers have different owners", async () => {
const before = process.listenerCount("exit");
const a = fixture(), b = fixture();
expect(process.listenerCount("exit")).toBe(before + 1);
a.exit(); await a.owner.stop();
expect(process.listenerCount("exit")).toBe(before + 1);
b.exit(); await b.owner.stop();
expect(process.listenerCount("exit")).toBe(before);
});
it("never uses immediate shutdown when preparing a clean template", async () => {
vi.useFakeTimers();
const f = fixture();
const clean = expect(f.owner.stopGracefully()).rejects.toThrow("template cannot be published");
await vi.advanceTimersByTimeAsync(30_000);
await clean;
expect(f.child.kill.mock.calls).toEqual([["SIGINT"]]);
f.exit();
await f.owner.stop();
});
it("rejects a crashed process as a publishable template", async () => {
const f = fixture();
f.child.exitCode = 1;
await expect(f.owner.stopGracefully()).rejects.toThrow("without clean shutdown");
await f.owner.stop();
});
it("does not reuse an earlier immediate stop as a clean template shutdown", async () => {
vi.useFakeTimers();
const f = fixture();
const stopped = f.owner.stop();
await vi.advanceTimersByTimeAsync(5_000);
f.exit();
await stopped;
await expect(f.owner.stopGracefully()).rejects.toThrow("without clean shutdown");
});
});
packages/db/src/test-postgres-owner.ts
+193
View File
@@ -0,0 +1,193 @@
import type { ChildProcess } from "node:child_process";
import { execFile, spawnSync } from "node:child_process";
import path from "node:path";
import { promisify } from "node:util";
const execute = promisify(execFile);
export type TestPostgresInstance = {
initialise(): Promise<void>;
start(): Promise<void>;
stop(): Promise<void>;
process?: ChildProcess;
};
type Owner = { stop(clean?: boolean): Promise<void>; interrupt(): void };
export class TestPostgresShutdownError extends Error {}
const owners = new Set<Owner>();
type ShutdownTask = { stop(signal: NodeJS.Signals): Promise<void>; interrupt(): void; afterStop?(): Promise<void> };
const shutdownTasks = new Set<ShutdownTask>();
let shuttingDown = false;
let hooksInstalled = false;
function interruptOwnedServers(): void {
for (const task of shutdownTasks) task.interrupt();
for (const owner of owners) owner.interrupt();
}
export function beforeTestPostgresShutdown(task: ShutdownTask): () => void {
shutdownTasks.add(task);
updateHooks();
return () => { shutdownTasks.delete(task); updateHooks(); };
}
export async function drainTestPostgresOwners(): Promise<void> {
const results = await Promise.allSettled([...owners].map((owner) => owner.stop()));
const failures = results.filter((result) => result.status === "rejected");
if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Test PostgreSQL shutdown failed");
}
async function shutdown(signal: "SIGINT" | "SIGTERM" | "SIGHUP"): Promise<void> {
if (shuttingDown) { interruptOwnedServers(); return; }
shuttingDown = true;
const registered = [...shutdownTasks];
const tasks = await Promise.allSettled(registered.map((task) => task.stop(signal)));
for (const result of tasks) {
if (result.status === "rejected") console.error(result.reason);
}
try { await drainTestPostgresOwners(); }
catch (error) { console.error(error); }
const cleanups = await Promise.allSettled(registered.map((task) => task.afterStop?.()));
for (const result of cleanups) {
if (result.status === "rejected") console.error(result.reason);
}
process.exit(128 + { SIGINT: 2, SIGTERM: 15, SIGHUP: 1 }[signal]);
}
const onInterrupt = () => { void shutdown("SIGINT"); };
const onTerminate = () => { void shutdown("SIGTERM"); };
const onHangup = () => { void shutdown("SIGHUP"); };
function updateHooks(): void {
if ((owners.size > 0 || shutdownTasks.size > 0) && !hooksInstalled) {
hooksInstalled = true;
process.on("exit", interruptOwnedServers);
process.on("SIGINT", onInterrupt);
process.on("SIGTERM", onTerminate);
process.on("SIGHUP", onHangup);
} else if (owners.size === 0 && shutdownTasks.size === 0) {
hooksInstalled = false;
process.removeListener("exit", interruptOwnedServers);
process.removeListener("SIGINT", onInterrupt);
process.removeListener("SIGTERM", onTerminate);
process.removeListener("SIGHUP", onHangup);
}
}
async function settlesWithin(promise: Promise<unknown>, timeoutMs: number): Promise<boolean> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise.then(() => true),
new Promise<false>((resolve) => { timer = setTimeout(() => resolve(false), timeoutMs); }),
]);
} finally {
if (timer) clearTimeout(timer);
}
}
export function ownTestPostgres(instance: TestPostgresInstance, dataDir?: string): Pick<TestPostgresInstance, "initialise" | "start" | "stop"> & {
stopGracefully(): Promise<void>;
} {
let pending: Promise<void> | null = null;
let stopPromise: Promise<void> | null = null;
let stopped = false;
let usedImmediateShutdown = false;
let stoppedCleanly = false;
function controlPath(child: ChildProcess): string | undefined {
if (!dataDir || !child.spawnfile) return undefined;
return path.join(path.dirname(child.spawnfile), `pg_ctl${path.extname(child.spawnfile) === ".exe" ? ".exe" : ""}`);
}
async function requestStop(child: ChildProcess, mode: "fast" | "immediate", clean: boolean): Promise<void> {
const control = controlPath(child);
if (control && (clean || process.platform === "win32")) {
try {
await execute(control, ["-D", dataDir!, "stop", "-m", mode, "-W"], {
timeout: 2000, maxBuffer: 64 * 1024, windowsHide: true,
});
} catch (error) {
if (child.exitCode === null && child.signalCode === null) throw error;
}
} else child.kill(mode === "fast" ? "SIGINT" : "SIGQUIT");
}
async function run(operation: () => Promise<void>): Promise<void> {
if (stopped || stopPromise) throw new Error("Test PostgreSQL is shutting down");
const running = operation();
pending = running;
try { await running; }
finally { if (pending === running) pending = null; }
}
const owner: Owner = {
interrupt() {
const child = instance.process;
if (child && child.exitCode === null && child.signalCode === null) {
usedImmediateShutdown = true;
const control = controlPath(child);
if (process.platform === "win32" && control) {
spawnSync(control, ["-D", dataDir!, "stop", "-m", "immediate", "-W"], { timeout: 2000, windowsHide: true });
} else {
try { child.kill("SIGQUIT"); } catch {}
}
}
},
stop(clean = false) {
if (stopped) return Promise.resolve();
if (stopPromise) return stopPromise;
stopPromise = (async () => {
if (pending && !await settlesWithin(pending.catch(() => {}), 30_000)) {
throw new TestPostgresShutdownError("Test PostgreSQL startup did not settle before shutdown; its data directory is retained");
}
const child = instance.process;
if (!child) {
if (!await settlesWithin(instance.stop(), 10_000)) {
throw new TestPostgresShutdownError("Test PostgreSQL shutdown timed out; its data directory is retained");
}
} else if (child.exitCode === null && child.signalCode === null) {
let onExit: () => void;
const exited = new Promise<void>((resolve) => { onExit = resolve; child.once("exit", onExit); });
try {
await requestStop(child, "fast", clean);
if (!await settlesWithin(exited, clean ? 30_000 : 5_000)) {
if (clean) throw new TestPostgresShutdownError("Test PostgreSQL did not complete clean shutdown; the template cannot be published");
usedImmediateShutdown = true;
await requestStop(child, "immediate", false);
if (!await settlesWithin(exited, 5_000)) {
throw new TestPostgresShutdownError("Test PostgreSQL did not exit after fast and immediate shutdown; its data directory is retained");
}
}
} finally {
child.removeListener("exit", onExit!);
}
}
stoppedCleanly = !usedImmediateShutdown && (!child || (child.exitCode === 0 && child.signalCode === null));
if (clean && !stoppedCleanly) {
throw new TestPostgresShutdownError("Test PostgreSQL exited without clean shutdown; the template cannot be published");
}
instance.process = undefined;
stopped = true;
owners.delete(owner);
updateHooks();
})().catch((error) => {
stopPromise = null;
if (error instanceof TestPostgresShutdownError) throw error;
throw new TestPostgresShutdownError("Test PostgreSQL shutdown failed; its data directory is retained", { cause: error });
});
return stopPromise;
},
};
owners.add(owner);
if (owners.size === 1) updateHooks();
return {
initialise: () => run(() => instance.initialise()),
start: () => run(() => instance.start()),
stop: () => owner.stop(),
stopGracefully: async () => {
await owner.stop(true);
if (!stoppedCleanly) throw new TestPostgresShutdownError("Test PostgreSQL exited without clean shutdown; the template cannot be published");
},
};
}
packages/db/src/test-process-tree.ts
+78
View File
@@ -0,0 +1,78 @@
import { execFile, spawn } from "node:child_process";
import { constants } from "node:os";
import { setTimeout } from "node:timers/promises";
import { promisify } from "node:util";
const execute = promisify(execFile);
export function startTestProcessTree(command: string, args: string[], env: NodeJS.ProcessEnv) {
const posix = process.platform !== "win32";
const child = spawn(command, args, { stdio: "inherit", env, detached: posix });
let exited = false;
let stopping: Promise<void> | null = null;
let retired = false;
const exit = new Promise<number>((resolve, reject) => {
child.once("error", (error) => { exited = true; reject(error); });
child.once("exit", (code, signal) => {
exited = true;
resolve(code ?? (signal ? 128 + constants.signals[signal] : 1));
});
});
async function alive(): Promise<boolean> {
if (!child.pid) return false;
if (!posix) return !exited;
const { stdout } = await execute("/bin/ps", ["-axo", "pgid=,stat="], { timeout: 2000, maxBuffer: 1024 * 1024 });
return stdout.split("\n").some((line) => {
const [group, state] = line.trim().split(/\s+/);
return Number(group) === child.pid && state && !state.startsWith("Z");
});
}
function signal(signal: NodeJS.Signals): void {
if (!child.pid || retired) return;
try {
if (posix) process.kill(-child.pid, signal);
else child.kill(signal);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error;
}
}
async function waitUntilStopped(milliseconds: number): Promise<boolean> {
const deadline = Date.now() + milliseconds;
do {
if (!await alive()) return true;
await setTimeout(50);
} while (Date.now() < deadline);
return false;
}
function stop(requested: NodeJS.Signals = "SIGTERM"): Promise<void> {
if (stopping) return stopping;
stopping = (async () => {
if (posix) {
signal(requested);
if (!await waitUntilStopped(15_000)) {
signal("SIGKILL");
if (!await waitUntilStopped(5_000)) throw new Error("Test command process group did not stop");
}
} else if (child.pid && !exited) {
await execute("taskkill", ["/PID", String(child.pid), "/T", "/F"], { timeout: 15_000 });
}
await exit;
retired = true;
})();
return stopping;
}
return {
stop,
interrupt: () => signal("SIGTERM"),
async wait(): Promise<number> {
const status = await exit;
await stop();
return status;
},
};
}
packages/db/vitest.config.ts
+2
View File
@@ -3,6 +3,8 @@ import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
environment: "node",
hookTimeout: 30_000,
teardownTimeout: 30_000,
include: ["src/**/*.test.ts"],
},
});
packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts
+273
View File
@@ -0,0 +1,273 @@
import { createInterface } from "node:readline";
import { PassThrough } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { NOOP_PLUGIN_TRACER, startWorkerRpcHost, type PluginInvocationContext, type PluginTracer } from "@paperclipai/plugin-sdk";
import plugin, { __resetDaytonaSandboxHandleCacheForTest, __setDaytonaPluginContextForTest } from "./plugin.js";
import manifest from "./manifest.js";
import { leaseStateAttributes, withLeaseSpan } from "./lease-tracing.js";
const { get, create } = vi.hoisted(() => ({ get: vi.fn(), create: vi.fn() }));
vi.mock("@daytonaio/sdk", () => ({
Daytona: class { get = get; create = create; },
DaytonaNotFoundError: class extends Error {},
DaytonaTimeoutError: class extends Error {},
}));
type Receipt = {
name: string;
startTimeMs: number;
endTimeMs: number;
attributes?: Record<string, unknown>;
status?: { code: number };
children: Receipt[];
};
function sandbox(id = "fixture-sandbox", state = "archived") {
return {
id, state, target: "us", recoverable: true, errorReason: null as string | null,
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
refreshData: vi.fn().mockResolvedValue(undefined),
recover: vi.fn().mockResolvedValue(undefined),
delete: vi.fn().mockResolvedValue(undefined),
getWorkDir: vi.fn().mockResolvedValue("/home/daytona"),
fs: { createFolder: vi.fn().mockResolvedValue(undefined), uploadFile: vi.fn().mockResolvedValue(undefined) },
process: { executeCommand: vi.fn(async (command: string) => ({
exitCode: 0, result: command.startsWith("cat ") ? '{"token":"private-token"}' : "bash",
})) },
};
}
const params = {
driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment",
providerLeaseId: "fixture-sandbox", config: { apiKey: "private-api-key", timeoutMs: 300000, reuseLease: true },
leaseMetadata: { workspaceSentinel: { token: "private-token", result: "written" } },
};
function workerHarness() {
const stdin = new PassThrough(), stdout = new PassThrough();
const lines = createInterface({ input: stdout });
const pending = new Map<string, { resolve(value: unknown): void; reject(error: Error): void }>();
const receipts: Array<{ invocationId: string; span: Receipt }> = [];
let sequence = 0;
lines.on("line", line => {
const message = JSON.parse(line);
if (message.method) {
if (message.method === "span.record") receipts.push({ invocationId: message.paperclipInvocationId, span: message.params });
if (message.id !== undefined) stdin.write(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: null }) + "\n");
} else {
const waiter = pending.get(message.id);
pending.delete(message.id);
if (message.error) waiter?.reject(new Error(message.error.message));
else waiter?.resolve(message.result);
}
});
const worker = startWorkerRpcHost({ plugin, stdin, stdout });
function call(method: string, input: unknown, invocation?: PluginInvocationContext): Promise<unknown> {
const id = String(++sequence);
return new Promise((resolve, reject) => {
pending.set(id, { resolve, reject });
stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: input,
...(invocation ? { paperclipInvocation: invocation } : {}) }) + "\n");
});
}
return {
call, receipts,
initialize: () => call("initialize", { manifest, config: {}, apiVersion: 1,
instanceInfo: { instanceId: "fixture", hostVersion: "0.0.0" } }),
close() { worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); },
};
}
function invocation(id: string, enabled = true): PluginInvocationContext {
return { id, scope: { companyId: "fixture-company" },
...(enabled ? { traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } : {}) };
}
describe("Daytona lease worker tracing", () => {
beforeEach(() => { get.mockReset(); create.mockReset(); __resetDaytonaSandboxHandleCacheForTest(); });
afterEach(() => { __setDaytonaPluginContextForTest(null); __resetDaytonaSandboxHandleCacheForTest(); });
it.each([true, false])("records the archived start and nested verification only with trace context: %s", async enabled => {
const resource = sandbox(); get.mockResolvedValue(resource);
const worker = workerHarness();
try {
await worker.initialize();
expect(await worker.call("environmentResumeLease", params, invocation("resume", enabled)))
.toMatchObject({ providerLeaseId: resource.id, metadata: { resumedFromState: "archived" } });
expect(resource.start).toHaveBeenCalledExactlyOnceWith(300);
expect(get).toHaveBeenCalledExactlyOnceWith(resource.id);
if (!enabled) { expect(worker.receipts).toEqual([]); return; }
expect(worker.receipts).toHaveLength(1);
const root = worker.receipts[0].span;
expect(root.name).toBe("plugin.worker");
expect(root.children.map(span => span.name)).toEqual(["lease.lookup", "lease.channel_cleanup", "lease.start", "lease.verify"]);
expect(root.children[2].attributes).toMatchObject({ "paperclip.sandbox.startup.state": "archived" });
expect(root.children[3].children.map(span => span.name)).toEqual(["lease.workspace", "lease.sentinel", "lease.shell"]);
function checkBounds(parent: Receipt) {
for (const child of parent.children) {
expect(child.startTimeMs).toBeGreaterThanOrEqual(parent.startTimeMs);
expect(child.endTimeMs).toBeGreaterThanOrEqual(child.startTimeMs);
expect(child.endTimeMs).toBeLessThanOrEqual(parent.endTimeMs);
checkBounds(child);
}
}
checkBounds(root);
expect(JSON.stringify(root)).not.toMatch(/private-|fixture-sandbox|fixture-company|\/home\//);
} finally { worker.close(); }
});
it("keeps simultaneous resume receipts separate and closes failed starts", async () => {
const slow = sandbox(), failed = sandbox("failed-sandbox", "stopped");
let release!: () => void, started!: () => void;
const waiting = new Promise<void>(resolve => { release = resolve; });
const entered = new Promise<void>(resolve => { started = resolve; });
slow.start.mockImplementation(async () => { started(); await waiting; });
failed.start.mockRejectedValue(new Error("private-provider-error"));
get.mockImplementation(async id => id === slow.id ? slow : failed);
const worker = workerHarness();
try {
await worker.initialize();
const first = worker.call("environmentResumeLease", params, invocation("slow"));
await entered;
await expect(worker.call("environmentResumeLease", { ...params, providerLeaseId: failed.id }, invocation("failed")))
.rejects.toThrow("private-provider-error");
release(); await first;
expect(worker.receipts.map(record => record.invocationId)).toEqual(["failed", "slow"]);
const failure = worker.receipts[0].span;
expect(failure.status).toEqual({ code: 2 });
expect(failure.children.find(span => span.name === "lease.start")?.status).toEqual({ code: 2 });
expect(failure.children.some(span => span.name === "lease.verify")).toBe(false);
expect(JSON.stringify(worker.receipts)).not.toContain("private-provider-error");
expect(slow.start).toHaveBeenCalledTimes(1);
expect(failed.start).toHaveBeenCalledTimes(1);
} finally { release(); worker.close(); }
});
it("waits for a pending teardown and records the wait under the resume worker", async () => {
const resource = sandbox("fixture-sandbox", "started"); get.mockResolvedValue(resource);
let release!: () => void, stopping!: () => void;
const waiting = new Promise<void>(resolve => { release = resolve; });
const entered = new Promise<void>(resolve => { stopping = resolve; });
resource.stop.mockImplementation(async () => { stopping(); await waiting; resource.state = "stopped"; });
const worker = workerHarness();
try {
await worker.initialize();
const teardown = worker.call("environmentReleaseLease", params, invocation("release"));
await entered;
let settled = false;
const resume = worker.call("environmentResumeLease", params, invocation("resume"))
.then(result => { settled = true; return result; });
// Process the resume request while the actual provider stop is held open.
await new Promise<void>(resolve => setImmediate(resolve));
expect(settled).toBe(false);
expect(get).toHaveBeenCalledTimes(1);
expect(resource.start).not.toHaveBeenCalled();
release();
await teardown;
expect(await resume).toMatchObject({ providerLeaseId: resource.id });
const root = worker.receipts.find(receipt => receipt.invocationId === "resume")!.span;
expect(root.name).toBe("plugin.worker");
expect(root.children.filter(span => span.name === "lease.wait_for_teardown")).toHaveLength(1);
expect(root.children[0].name).toBe("lease.wait_for_teardown");
expect(root.children[0].endTimeMs).toBeLessThanOrEqual(root.children[1].startTimeMs);
expect(get).toHaveBeenCalledTimes(2);
expect(resource.start).toHaveBeenCalledTimes(1);
} finally { release(); worker.close(); }
});
it.each([false, true])("records missing-container confirmation and preserves refresh failures: %s", async fails => {
const resource = sandbox("fixture-sandbox", "error");
resource.recoverable = false;
resource.errorReason = `not found: failed to inspect sandbox container ${resource.id}: Error response from daemon: No such container: ${resource.id}`;
get.mockResolvedValue(resource);
resource.refreshData.mockImplementation(async () => {
if (fails) throw new Error("private-refresh-failure");
resource.state = "started";
});
const worker = workerHarness();
try {
await worker.initialize();
const result = worker.call("environmentResumeLease", params, invocation("resume"));
if (fails) await expect(result).rejects.toThrow("private-refresh-failure");
else expect(await result).toMatchObject({ providerLeaseId: resource.id });
expect(resource.refreshData).toHaveBeenCalledTimes(1);
const root = worker.receipts[0].span;
expect(root.children.map(span => span.name)).toEqual(fails
? ["lease.lookup", "lease.refresh"] : ["lease.lookup", "lease.refresh", "lease.verify"]);
expect(root.children[1].status).toEqual(fails ? { code: 2 } : undefined);
expect(resource.start).not.toHaveBeenCalled();
expect(resource.delete).not.toHaveBeenCalled();
expect(create).not.toHaveBeenCalled();
expect(JSON.stringify(worker.receipts)).not.toContain("private-refresh-failure");
} finally { worker.close(); }
});
it.each(["started", "error"])("traces a %s sandbox without adding a provider start", async state => {
const resource = sandbox("fixture-sandbox", state); get.mockResolvedValue(resource);
const worker = workerHarness();
try {
await worker.initialize();
await worker.call("environmentResumeLease", params, invocation("resume"));
expect(resource.start).not.toHaveBeenCalled();
expect(resource.recover).toHaveBeenCalledTimes(state === "error" ? 1 : 0);
const phases = worker.receipts[0].span.children;
expect(phases.some(span => span.name === "lease.start")).toBe(false);
expect(phases.some(span => span.name === "lease.recover")).toBe(state === "error");
} finally { worker.close(); }
});
it("closes the verification group and failing workspace child", async () => {
const resource = sandbox(); get.mockResolvedValue(resource);
resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error"));
const worker = workerHarness();
try {
await worker.initialize();
await expect(worker.call("environmentResumeLease", params, invocation("resume")))
.rejects.toThrow("private-workspace-error");
const verify = worker.receipts[0].span.children.find(span => span.name === "lease.verify")!;
expect(verify.status).toEqual({ code: 2 });
expect(verify.children.map(span => span.name)).toEqual(["lease.workspace"]);
expect(verify.children[0].status).toEqual({ code: 2 });
expect(resource.delete).not.toHaveBeenCalled();
} finally { worker.close(); }
});
it("records setup failure and compensating cleanup as siblings", async () => {
const resource = sandbox(); create.mockResolvedValue(resource);
resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error"));
const worker = workerHarness();
try {
await worker.initialize();
await expect(worker.call("environmentAcquireLease", { ...params, runId: "fixture-run" }, invocation("acquire")))
.rejects.toThrow("private-workspace-error");
expect(resource.delete).toHaveBeenCalledTimes(1);
const root = worker.receipts[0].span;
expect(root.children.map(span => span.name)).toEqual(["lease.create", "lease.workspace", "lease.cleanup"]);
expect(root.children[1].status).toEqual({ code: 2 });
expect(root.children[2].status).toBeUndefined();
} finally { worker.close(); }
});
});
describe("lease tracing cannot change provider work", () => {
it.each(["missing", "before", "after", "during"])("preserves the result and error with a %s tracer failure", async mode => {
const tracer: PluginTracer = { ...NOOP_PLUGIN_TRACER,
...(mode === "missing" ? {} : { async withSpan<T>(_name: string, work: () => T | Promise<T>): Promise<T> {
if (mode === "before") throw new Error("trace failure");
const result = work();
if (mode === "during") throw new Error("trace failure");
await result;
throw new Error("trace failure");
} }),
};
const success = vi.fn(async () => 42), error = new Error("provider failure"), failure = vi.fn(async () => { throw error; });
expect(await withLeaseSpan(tracer, "lease.lookup", success)).toBe(42);
await expect(withLeaseSpan(tracer, "lease.lookup", failure)).rejects.toBe(error);
expect(success).toHaveBeenCalledTimes(1); expect(failure).toHaveBeenCalledTimes(1);
});
it("normalizes unknown provider states before recording", () => {
expect(leaseStateAttributes("provider-private-state")).toEqual({ "paperclip.sandbox.startup.state": "other" });
});
});
packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts
+49
View File
@@ -0,0 +1,49 @@
import type { PluginTracer } from "@paperclipai/plugin-sdk";
export type LeaseSpanName =
| "lease.wait_for_teardown"
| "lease.lookup"
| "lease.refresh"
| "lease.channel_cleanup"
| "lease.start"
| "lease.recover"
| "lease.verify"
| "lease.workspace"
| "lease.sentinel"
| "lease.shell"
| "lease.create"
| "lease.expiry"
| "lease.cleanup";
/** Keep provider state useful without exporting a new SDK value as free text. */
export function leaseStateAttributes(state: unknown): Record<string, string> {
return {
"paperclip.sandbox.startup.state":
state === "started" || state === "stopped" || state === "archived" || state === "error"
? state : "other",
};
}
/** Use the invocation's async parent, while keeping tracer faults out of work. */
export async function withLeaseSpan<T>(
tracer: PluginTracer,
name: LeaseSpanName,
work: () => Promise<T>,
attributes?: Record<string, string | number | boolean>,
): Promise<T> {
let result: Promise<T> | undefined;
// A tracer can fail before or after invoking work. Preserve its exact result
// or error and never repeat provider operations in either case.
const runOnce = () => result ??= Promise.resolve().then(work);
try {
if (tracer.withSpan) {
await tracer.withSpan(name, runOnce, {
"paperclip.sandbox.startup.provider": "daytona",
...attributes,
});
}
} catch {
// The operation's error is still held by result and is rethrown below.
}
return runOnce();
}
packages/plugins/sandbox-providers/daytona/src/plugin.ts
+65 -52
View File
@@ -50,6 +50,7 @@ import type {
} from "@paperclipai/plugin-sdk";
import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js";
import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js";
import { withLeaseSpan, leaseStateAttributes, type LeaseSpanName } from "./lease-tracing.js";
import { observeSessionStreamExit } from "./session-stream-exit.js";
import { DaytonaIdleActivity, trackDaytonaHooks, type DaytonaIdleProof } from "./idle-activity.js";
@@ -126,6 +127,10 @@ export function getPluginTracer(): PluginTracer {
return pluginContext?.tracer ?? NOOP_PLUGIN_TRACER;
}
function traceLeaseStep<T>(name: LeaseSpanName, work: () => Promise<T>, attributes?: Record<string, string | number | boolean>): Promise<T> {
return withLeaseSpan(getPluginTracer(), name, work, attributes);
}
/**
* Test seam: set the module-level plugin context, and return a restore function.
* `plugin.test.ts` uses it to inject a recording tracer without running `setup`.
@@ -528,12 +533,16 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P
const startBoundMs = timeoutSeconds * 1_000 + LIVENESS_START_TIMEOUT_MARGIN_MS;
if (sandbox.state === "error") {
if (sandbox.recoverable) {
await withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds));
await traceLeaseStep("lease.recover", () =>
withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)),
leaseStateAttributes(sandbox.state));
return;
}
throw new Error(`Daytona sandbox ${sandbox.id} is in an unrecoverable error state: ${sandbox.errorReason ?? "unknown error"}`);
}
await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds));
await traceLeaseStep("lease.start", () =>
withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)),
leaseStateAttributes(sandbox.state));
}
function hasMissingSandboxContainer(sandbox: Sandbox): boolean {
@@ -1362,6 +1371,7 @@ const sandboxHandleActivityGates = (() => {
type SandboxLeaseAdmissionOptions = {
allowClosed?: boolean;
traceLeaseWait?: boolean;
};
const sandboxHandleLeaseAdmissionStates = (() => {
@@ -1402,7 +1412,8 @@ async function withSandboxActivityGate<T>(
const teardownGate = sandboxHandleTeardownGates.current(scope);
if (teardownGate) {
await teardownGate.promise;
if (options.traceLeaseWait) await traceLeaseStep("lease.wait_for_teardown", () => teardownGate.promise);
else await teardownGate.promise;
if (!options.allowClosed && sandboxHandleLeaseAdmissionStates.isClosed(scope)) {
throw new Error(`Daytona sandbox lease ${scope.providerLeaseId} is no longer active.`);
}
@@ -2682,39 +2693,39 @@ const plugin = definePlugin(trackDaytonaHooks({
return timeoutFailure();
};
const acquire = async (): Promise<PluginEnvironmentLease> => {
const sandbox = await createSandbox(params, config, {
const sandbox = await traceLeaseStep("lease.create", () => createSandbox(params, config, {
onCreateAttempt: (attempt) => { assertActive(); cleanup = attempt; },
onCreateFailure: observeFailure,
onSnapshotIdentity: identity => { snapshotIdentity = identity; },
});
}));
try {
assertActive();
if (cleanup) cleanup.observedProviderLeaseId = sandbox.id;
phase = "workspace";
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox));
assertActive();
phase = "shell";
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now())));
const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now()))));
assertActive();
// Configure a provider-side destroy time at or before a caller deadline, so
// an abandoned sandbox self-destroys even if Paperclip is down. The lease
// carries the real provider expiry (or none) as evidence of the bound.
phase = "expiry";
const expiresAt = await configureSandboxExpiry({
const expiresAt = await traceLeaseStep("lease.expiry", () => configureSandboxExpiry({
sandbox,
requestedExpiresAt: params.requestedExpiresAt,
nowMs: Date.now(),
preserveCreationExpiry: config.ttlMinutes != null,
});
}));
assertActive();
phase = "sentinel";
const workspaceSentinel = await writeWorkspaceSentinel({
const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => writeWorkspaceSentinel({
sandbox,
remoteCwd,
params,
config,
timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())),
});
}));
assertActive();
sandboxHandleLeaseAdmissionStates.open({
driverKey: params.driverKey,
@@ -2757,7 +2768,7 @@ const plugin = definePlugin(trackDaytonaHooks({
cleanupInProgress = true;
try {
const proof = idleActivity.proof(providerActivityKey(params, config, sandbox.id));
await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true);
await traceLeaseStep("lease.cleanup", () => sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true));
idleActivity.confirm(proof);
} catch (cleanupError) {
if (cleanup) {
@@ -2821,7 +2832,7 @@ const plugin = definePlugin(trackDaytonaHooks({
throw new Error("The stopped Daytona sandbox is still settling cancelled work. Retry shortly.");
}
return await withSandboxActivityGate(scope, async () => {
const sandbox = await getSandboxOrNull(scope, { bypassTeardownGate: true });
const sandbox = await traceLeaseStep("lease.lookup", () => getSandboxOrNull(scope, { bypassTeardownGate: true }));
if (!sandbox) {
return { providerLeaseId: null, metadata: { expired: true } };
}
@@ -2831,7 +2842,7 @@ const plugin = definePlugin(trackDaytonaHooks({
// backup-guarded replacement path. Unknown errors preserve the lease.
if (hasMissingSandboxContainer(sandbox)) {
try {
await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData());
await traceLeaseStep("lease.refresh", () => withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()));
assertHandleMatchesLease(sandbox, params.providerLeaseId);
} catch (error) {
evictSandboxHandle(scope);
@@ -2859,48 +2870,50 @@ const plugin = definePlugin(trackDaytonaHooks({
// A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
// is dead after a real restart. Close and drop every channel on this lease
// before the restart, so no stale channel id survives the resume.
await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
await traceLeaseStep("lease.channel_cleanup", () => closeDaytonaDuplexChannelsForLease(params.providerLeaseId));
}
const resumedFromState = sandbox.state ?? null;
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox, false);
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await verifyWorkspaceSentinel({
sandbox,
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
});
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return {
providerLeaseId: null,
metadata: { expired: true, workspaceSentinel },
};
}
const shellCommand = await detectSandboxShellCommand(
sandbox,
toTimeoutSeconds(config.timeoutMs),
);
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
return await traceLeaseStep("lease.verify", async () => {
const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox, false));
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => verifyWorkspaceSentinel({
sandbox,
shellCommand,
remoteCwd: requestedCwd ?? remoteCwd,
resumedLease: true,
resumedFromState,
sandboxState: "started",
workspaceSentinel,
}),
};
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
}));
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return {
providerLeaseId: null,
metadata: { expired: true, workspaceSentinel },
};
}
const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(
sandbox,
toTimeoutSeconds(config.timeoutMs),
));
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
sandbox,
shellCommand,
remoteCwd: requestedCwd ?? remoteCwd,
resumedLease: true,
resumedFromState,
sandboxState: "started",
workspaceSentinel,
}),
};
});
} catch (error) {
evictSandboxHandle(scope);
// A timeout, rate limit, or provider 5xx does not prove this sandbox is
@@ -2910,7 +2923,7 @@ const plugin = definePlugin(trackDaytonaHooks({
throw error;
}
},
{ allowClosed: true },
{ allowClosed: true, traceLeaseWait: true },
);
},
packages/shared/src/ai-connections.ts
+11
View File
@@ -190,6 +190,17 @@ export interface AiManagedConnectionSummary {
unavailableReason?: string;
usageProbeSupported?: boolean;
}
/** Safe provider account details for one authorized saved credential. */
export interface AiConnectionAccountIdentity {
connectionId: string;
grantId: string;
status: "available" | "unavailable" | "error";
checkedAt: string;
accountLabel: string | null;
organizationLabel: string | null;
errorCode?: "unsupported" | "connection_unavailable" | "authentication_required" | "permission_denied" | "rate_limited" | "provider_unavailable" | "invalid_response";
message?: string;
}
export interface AiConnectionList {
currentUserId: string;
pools?: import("./ai-connection-router.js").AiConnectionPool[];
packages/shared/src/constants.ts
+6
View File
@@ -198,6 +198,12 @@ export const ISSUE_STATUSES = [
] as const;
export type IssueStatus = (typeof ISSUE_STATUSES)[number];
/** Read-only task presentation. Monitoring never grants execution authority. */
export const ISSUE_DISPLAY_STATUSES = [
"backlog", "todo", "in_progress", "monitoring", "in_review", "done", "blocked", "cancelled",
] as const;
export type IssueDisplayStatus = (typeof ISSUE_DISPLAY_STATUSES)[number];
export const INBOX_MINE_ISSUE_STATUSES = [
"backlog",
"todo",
packages/shared/src/index.ts
+2
View File
@@ -364,6 +364,7 @@ export {
AGENT_ICON_NAMES,
PROJECT_ICON_NAMES,
ISSUE_STATUSES,
ISSUE_DISPLAY_STATUSES,
ISSUE_SUMMARIES_MAX_IDS,
INBOX_MINE_ISSUE_STATUSES,
INBOX_MINE_ISSUE_STATUS_FILTER,
@@ -562,6 +563,7 @@ export {
type AgentIconName,
type ProjectIconName,
type IssueStatus,
type IssueDisplayStatus,
type IssuePriority,
type IssueReviewPolicy,
type IssueWorkMode,
packages/shared/src/types/issue.ts
+6 -1
View File
@@ -20,6 +20,7 @@ import type {
IssueHarnessKind,
IssueOriginKind,
IssuePriority,
IssueDisplayStatus,
IssueReviewPolicy,
IssueRecoveryActionKind,
IssueRecoveryActionOutcome,
@@ -270,6 +271,7 @@ export interface AcceptedPlanDecompositionSummary extends AcceptedPlanDecomposit
}
export interface IssueRelationIssueSummary {
displayStatus?: IssueDisplayStatus;
id: string;
identifier: string | null;
title: string;
@@ -850,12 +852,14 @@ export type IssueChanges = Record<string, IssueChangeReceiptEntry>;
export type IssueSummary = Pick<
Issue,
"id" | "companyId" | "title" | "identifier" | "status" | "updatedAt"
"id" | "companyId" | "title" | "identifier" | "status" | "displayStatus" | "updatedAt"
> & {
externalConversationState: "active" | "waiting" | null;
};
export interface Issue {
/** Read-only status for task presentation; never use it for execution admission. */
displayStatus?: IssueDisplayStatus;
/** Collection projections omit task bodies and cannot seed the detail cache. */
listProjection?: "index" | "rows";
statusVersion?: number;
@@ -985,6 +989,7 @@ export type CompactIssue = Pick<
| "privacyRootIssueId"
| "title"
| "description"
| "displayStatus"
| "status"
| "workMode"
| "priority"
packages/shared/src/types/search.ts
+3 -1
View File
@@ -1,4 +1,4 @@
import type { IssuePriority, IssueStatus } from "../constants.js";
import type { IssueDisplayStatus, IssuePriority, IssueStatus } from "../constants.js";
export const COMPANY_SEARCH_SCOPES = ["all", "issues", "comments", "documents", "artifacts", "agents", "projects"] as const;
export type CompanySearchScope = (typeof COMPANY_SEARCH_SCOPES)[number];
@@ -38,6 +38,7 @@ export interface CompanySearchIssueSummary {
identifier: string | null;
title: string;
status: IssueStatus;
displayStatus?: IssueDisplayStatus;
externalConversationState?: "active" | "waiting" | null;
priority: IssuePriority;
assigneeAgentId: string | null;
@@ -76,6 +77,7 @@ export interface CompanySearchResult {
export interface CompanySearchFilterOptionCounts {
status: Partial<Record<IssueStatus, number>>;
displayStatus?: Partial<Record<IssueDisplayStatus, number>>;
priority: Partial<Record<IssuePriority, number>>;
assigneeAgentId: Record<string, number>;
assigneeUserId: Record<string, number>;
packages/shared/src/types/user-profile.ts
+2 -1
View File
@@ -1,4 +1,4 @@
import type { IssuePriority, IssueStatus } from "../constants.js";
import type { IssuePriority, IssueStatus, IssueDisplayStatus } from "../constants.js";
export interface UserProfileIdentity {
id: string;
@@ -42,6 +42,7 @@ export interface UserProfileIssueSummary {
identifier: string | null;
title: string;
status: IssueStatus;
displayStatus?: IssueDisplayStatus;
priority: IssuePriority;
assigneeAgentId: string | null;
assigneeUserId: string | null;
packages/shared/src/validators/search.ts
+4 -1
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
import { ISSUE_DISPLAY_STATUSES, ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
import { isUuidLike } from "../agent-url-key.js";
import {
COMPANY_SEARCH_EXTRACT_KINDS,
@@ -156,6 +156,9 @@ export const companySearchQuerySchema = z.object({
status: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)),
displayStatus: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "displayStatus", ISSUE_DISPLAY_STATUSES)),
priority: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "priority", ISSUE_PRIORITIES)),
scripts/__tests__/release-verify-workflow.test.mjs
+75 -85
View File
@@ -579,9 +579,8 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
readWorkflow("chat-feature-testops.yml"),
readWorkflow("chat-completion-testops.yml"),
readWorkflow("chat-catalog-testops.yml"),
readWorkflow("chat-catalog-testops-case.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -621,9 +620,8 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
"chat-feature-testops.yml",
"chat-completion-testops.yml",
"chat-catalog-testops.yml",
"chat-catalog-testops-case.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -642,6 +640,79 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
for (const name of paidWorkflowNames) {
const workflow = readWorkflow(name);
const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n"));
if (name === "chat-catalog-testops.yml") {
assert.match(triggerHeader, /^ workflow_dispatch:/m);
assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_call|workflow_run):/m);
const callerBlocks = ["chat", "chat_external"].map(id => workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0]);
assert.doesNotMatch(callerBlocks.reduce((rest, block) => rest.replace(block, ""), workflow), providerSecretReference);
assert.match(workflow, /group: chat-catalog-testops-streaming-v1/);
for (const id of ["chat", "chat_external"]) {
const block = workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0];
const assertCaller = block => {
assert.match(block, /if: needs\.authorize\.outputs\.operation == 'evaluate'/);
assert.ok(block.includes(`needs.select.outputs.${id === "chat" ? "native" : "external"}_count != '0'`));
assert.match(block, /needs: \[authorize, select, build\]/);
assert.match(block, /uses: \.\/\.github\/workflows\/chat-catalog-testops-case\.yml/);
assert.match(block, /protocol: streaming-v1/);
const mapping = block.split("\n secrets:\n");
assert.equal(mapping.length, 2);
assert.deepEqual(mapping[1].trim().split("\n").map(line => line.trim()), [
"OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}",
"ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}",
"ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}",
]);
assert.doesNotMatch(block, /secrets: inherit/);
assert.match(block, /fail-fast: false/);
};
assertCaller(block);
const lane = id === "chat" ? "native" : "external";
for (const mutant of [block.replace("needs.authorize.outputs.operation == 'evaluate' && ", ""),
block.replace(`needs.select.outputs.${lane}_count != '0'`, 'true'),
block.replace('${{ secrets.OPENROUTER_API_KEY }}', "''"),
block + '\n EXTRA: ${{ secrets.EXTRA }}',
block.replace(/\n secrets:[\s\S]*/, '\n secrets: inherit')]) assert.throws(() => assertCaller(mutant));
}
const final = workflow.split(" verify:\n")[1];
assert.match(final, /chat-completion-testops-cli\.mjs verify-stream/);
assert.doesNotMatch(final, /allurectl|cli\.mjs upload(?: |$)/);
continue;
}
if (name === "chat-catalog-testops-case.yml") {
assert.match(triggerHeader, /^ workflow_call:/m);
assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_dispatch|workflow_run):/m);
const chat = workflow.split(" chat:\n")[1].split("\n report:\n")[0];
const report = workflow.split(" report:\n")[1].split("\n upload:\n")[0];
const upload = workflow.split(" upload:\n")[1];
const boundary = value => {
assert.match(value, /environment: runner-e2e-private-paid/);
assert.match(value, /CHAT_TESTOPS_PROTOCOL: \$\{\{ inputs\.protocol \}\}/);
assert.match(value, /test "\$CHAT_TESTOPS_PROTOCOL" = streaming-v1/);
assert.match(value, /chat-completion-testops-cli\.mjs guard/);
assert.match(value, /chat-completion-testops-cli\.mjs verify-selection/);
assert.match(value, /--max-parallel 1 --max-automatic-retries 0/);
assert.match(value, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/);
assert.doesNotMatch(value, /ALLURE_TOKEN/);
assert.equal([...value.matchAll(providerSecretReference)].length, 2);
assert.ok(value.indexOf("chat-completion-testops-cli.mjs verify-selection") < value.search(providerSecretReference));
};
boundary(chat);
for (const mutant of [chat.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
chat.replace("cli.mjs verify-selection", "cli.mjs unverified-selection"), chat + "\n ALLURE_TOKEN: leaked"]) {
assert.throws(() => boundary(mutant));
}
assert.match(report, /needs: chat/); assert.match(report, /always\(\) && !cancelled\(\)/);
assert.match(report, /environment: runner-e2e-private-history/);
assert.doesNotMatch(report, /secrets\./);
assert.match(report, /manual-private-transfer\.ts download-cell/);
assert.match(report, /cli\.mjs prepare-case-upload/);
assert.match(upload, /needs: report/); assert.match(upload, /always\(\) && !cancelled\(\)/);
assert.match(upload, /environment: runner-e2e-testops-reporting/);
assert.doesNotMatch(upload, /configure-aws-credentials|manual-private-transfer|secrets\.(?:OPENROUTER|ANTHROPIC)/);
assert.match(upload, /ALLURE_TOKEN: \$\{\{ secrets\.ALLURE_TOKEN \}\}/);
assert.match(upload, /cli\.mjs upload-case/);
assert.doesNotMatch(upload, /allurectl/);
continue;
}
assert.doesNotMatch(
triggerHeader,
/^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
@@ -656,87 +727,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
(block) => [...block.matchAll(providerSecretReference)].length > 0,
);
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
if (name === "chat-completion-testops.yml" || name === "chat-catalog-testops.yml") {
assert.equal(providerJobs.length, 1);
const expectedKeys = name === "chat-catalog-testops.yml" ? 2 : 1;
const assertBoundary = (block) => {
assert.match(block, /^ chat:\n/);
assert.match(block, /\n needs: \[authorize, select, build\]\n/);
if (expectedKeys === 2) assert.ok(block.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.native_count != '0'"));
else assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
assert.match(block, /\n environment: runner-e2e-private-paid\n/);
assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
assert.match(block, /chat-completion-testops-cli\.mjs verify-selection/);
assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
assert.match(block, /--id "\$\{\{ matrix\.executionId \}\}" --max-parallel 1 --max-automatic-retries 0/);
assert.match(block, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/);
if (expectedKeys === 2) assert.ok(block.includes('max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}'));
else assert.match(block, /max-parallel: 1/);
assert.doesNotMatch(block, /ALLURE_TOKEN/);
assert.equal([...block.matchAll(providerSecretReference)].length, expectedKeys);
if (expectedKeys === 2) assert.ok(block.includes("ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}"));
assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
assert.ok(block.indexOf("chat-completion-testops-cli.mjs verify-selection") < block.search(providerSecretReference));
};
const block = providerJobs[0]; assertBoundary(block);
assert.equal([...workflow.matchAll(providerSecretReference)].length, expectedKeys);
for (const mutant of [
block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
block.replace(" - *guard\n", ""),
block.replace("chat-completion-testops-cli.mjs verify-selection", "chat-completion-testops-cli.mjs unverified-selection"),
block.replace('--id "${{ matrix.executionId }}"', "--all"),
block.replace("PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1", "PAPERCLIP_CHAT_COMPLETION_TESTOPS: unknown"),
]) assert.throws(() => assertBoundary(mutant));
if (expectedKeys === 2) assert.throws(() => assertBoundary(block.replace("contains(matrix.required_credentials, 'ANTHROPIC_API_KEY')", "true")));
if (expectedKeys === 2) {
const external = jobBlocks.find(value => value.startsWith(' chat_external:\n'));
const assertExternal = value => {
assert.match(value, /needs: \[authorize, select, build\]/);
assert.ok(value.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.external_count != '0'"));
assert.match(value, /environment: runner-e2e-private-paid/);
assert.match(value, /max-parallel: 1/);
assert.ok(value.includes('matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}'));
assert.match(value, /env: \*retention/);
assert.match(value, /steps: \*catalog_worker_steps/);
};
assertExternal(external);
for (const mutant of [external.replace('*catalog_worker_steps', '*unguarded'), external.replace('max-parallel: 1', 'max-parallel: 8'), external.replace('environment: runner-e2e-private-paid', 'environment: unprotected')]) assert.throws(() => assertExternal(mutant));
}
continue;
}
if (name === "chat-feature-testops.yml") {
assert.equal(providerJobs.length, 1);
const assertBoundary = (block) => {
assert.match(block, /^ chat:\n/);
assert.match(block, /\n needs: \[authorize, select, build\]\n/);
assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
assert.match(block, /\n environment: runner-e2e-private-paid\n/);
assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/);
assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/);
assert.doesNotMatch(block, /ALLURE_TOKEN/);
assert.equal([...block.matchAll(providerSecretReference)].length, 1);
assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference));
};
const block = providerJobs[0];
assertBoundary(block);
assert.equal([...workflow.matchAll(providerSecretReference)].length, 1);
assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/);
assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/);
assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/);
for (const mutant of [
block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
block.replace(" - *guard\n", ""),
block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"),
block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"),
block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"),
]) assert.throws(() => assertBoundary(mutant));
continue;
}
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
scripts/benchmark-connection-install-capabilities.ts
+4 -2
View File
@@ -15,6 +15,8 @@ import type { AuthorizationActor } from "../server/src/services/authorization.js
// Run explicitly; this benchmark never reads DATABASE_URL or contacts a hosted DB.
// node --import ./server/node_modules/tsx/dist/esm/index.mjs scripts/benchmark-connection-install-capabilities.ts --baseline-ref=<pre-fix SHA>
const connectionPurpose = process.argv.find(arg => arg.startsWith("--purpose="))?.slice(10) ?? "tool";
assert.ok(connectionPurpose === "tool" || connectionPurpose === "ai", "Use --purpose=tool or --purpose=ai");
const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15);
assert.ok(baselineRef, "Supply an explicit pre-fix --baseline-ref");
const repository = fileURLToPath(new URL("../", import.meta.url));
@@ -101,7 +103,7 @@ try {
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } });
await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Synthetic benchmark", type: "mcp_http" });
await db.insert(toolConnections).values({ id: connectionId, applicationId, companyId, uid: connectionId,
name: "Synthetic benchmark", transport: "mcp_remote", authKind: "none", config: { provider: "fixture" } });
name: "Synthetic benchmark", connectionPurpose, transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", config: { provider: "fixture" } });
await db.insert(toolConnectionInstalls).values(installs === "company"
? [{ companyId, connectionId, targetType: "company", targetId: companyId }]
: candidateIds.slice(0, Math.max(1, Math.floor(n / 4))).map(targetId => ({ companyId, connectionId, targetType: "agent" as const, targetId })));
@@ -151,7 +153,7 @@ try {
const expectedRetained = installs === "company" ? n : Math.max(1, Math.floor(n / 4));
assert.equal(targetLookups(captured.baseline), n * expectedRetained + n - expectedRetained);
assert.equal(targetLookups(captured.treatment), n);
const result = { n, installs, principal: "cloud noncreator member with target-scoped grant", completePositionalEquality: true,
const result = { n, installs, connectionPurpose, principal: "cloud noncreator member with target-scoped grant", completePositionalEquality: true,
baseline: { ...before, ...counters.baseline, targetLookups: targetLookups(captured.baseline) },
treatment: { ...after, ...counters.treatment, targetLookups: targetLookups(captured.treatment) },
speedup: before.medianMs! / after.medianMs!, statements: captured, treatmentPlans: plans };
scripts/benchmark-github-webhook-ingress.ts
+3 -3
View File
@@ -5,7 +5,7 @@ import { promises as fs, readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { applyPendingMigrations, chatActions, createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../packages/db/src/response-capture.js";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../server/src/__tests__/helpers/full-database-response-capture.js";
import { createGitHubWebhookIngress } from "../server/src/services/chat-channels/github/webhook-ingress.js";
import { captureIngressQueries, cleanupIngressFixture, githubIngressFixture, type CapturedQuery } from "../server/src/__tests__/helpers/github-webhook-ingress.js";
import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js";
@@ -60,7 +60,7 @@ try {
queries.length = 0;
f.calls.length = 0;
const events: Array<Record<string, unknown>> = [], timestamp = Date.now();
const capture = iteration === 11 ? createDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined;
const capture = iteration === 11 ? createFullDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, { chatActions }, capture);
const start = performance.now();
const response = await service.processGitHubWebhookIngress(action.id);
@@ -80,7 +80,7 @@ try {
const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${candidate.sql}`, candidate.params as never[]);
const positionalRows = await db.$client.unsafe(candidate.sql, candidate.params as never[]).values();
measurements[name] = { queries: executed, initialQueryPlan: plan["QUERY PLAN"], initialReadJsonBytes: Buffer.byteLength(JSON.stringify(positionalRows)),
responseValueBytes: events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes,
responseValueBytes: events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes,
captureEvents: events };
}
}
server/scripts/benchmark-native-backup-publication.mjs
+196
View File
@@ -0,0 +1,196 @@
import assert from "node:assert/strict";
import { fork, execFileSync } from "node:child_process";
import { createServer, get, Agent } from "node:http";
import { promises as fs } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
import { setTimeout as delay } from "node:timers/promises";
const filename = fileURLToPath(import.meta.url);
const checkout = path.resolve(path.dirname(filename), "../..");
const arg = (name, fallback) => {
const position = process.argv.indexOf(`--${name}`);
return position < 0 ? fallback : process.argv[position + 1];
};
const percentile = (values, fraction) => {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? null;
};
if (process.argv.includes("--worker")) {
const { publishNativeHarnessBackup } = await import(pathToFileURL(arg("module")));
let publishing = false;
const server = createServer((_request, response) => {
response.setHeader("x-publication-active", publishing ? "1" : "0");
response.end("ok");
});
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
process.send({ kind: "ready", port: server.address().port });
process.on("message", async message => {
if (message.kind === "close") {
server.closeAllConnections();
server.close(() => process.disconnect());
return;
}
if (message.kind !== "publish") return;
const histogram = monitorEventLoopDelay({ resolution: 10 });
histogram.enable();
await delay(20);
const cpu = process.cpuUsage();
const started = performance.now();
publishing = true;
try {
await publishNativeHarnessBackup({
db: {}, companyId: "benchmark", scopeDigest: "fixture",
backupRoot: message.backupRoot, pendingRoot: message.pendingRoot,
stamp: async () => {},
});
const durationMs = performance.now() - started;
const cpuUsed = process.cpuUsage(cpu);
publishing = false;
await delay(20);
histogram.disable();
assert.equal(await fs.readFile(path.join(message.backupRoot, "current/manifest.json"), "utf8"), "replacement");
assert.deepEqual((await fs.readdir(message.backupRoot)).sort(), ["current"]);
assert.equal(await fs.readFile(path.join(message.liveRoot, "untouched.txt"), "utf8"), "live");
assert.equal((await fs.stat(message.liveRoot)).mode & 0o777, 0o555);
process.send({ kind: "result", durationMs, cpuMs: (cpuUsed.user + cpuUsed.system) / 1000,
eventLoopMaxMs: histogram.max / 1e6, workCompleted: true });
} catch (error) {
publishing = false;
process.send({ kind: "error", message: error.stack });
}
});
} else {
const baselineRef = arg("baseline-ref");
assert(baselineRef, "--baseline-ref is required");
const pairs = Number(arg("pairs", "3"));
const directories = Number(arg("directories", "100"));
const filesPerDirectory = Number(arg("files-per-directory", "100"));
const intervalMs = Number(arg("interval-ms", "5"));
for (const value of [pairs, directories, filesPerDirectory, intervalMs]) assert(Number.isSafeInteger(value) && value > 0);
const evidence = await fs.mkdtemp(path.join(tmpdir(), "native-backup-publication-proof-"));
const { build } = await import("esbuild");
const buildVariant = async variant => {
const outfile = path.join(evidence, `${variant}.mjs`);
await build({ absWorkingDir: checkout, entryPoints: ["server/src/services/native-runtime/native-harness-backup-publication.ts"],
outfile, bundle: true, platform: "node", format: "esm", target: "node24", sourcemap: true,
alias: { "@paperclipai/db/work-signals": path.join(checkout, "packages/db/src/work-signals.ts") },
plugins: variant === "baseline" ? [{ name: "pinned-source", setup(builder) {
builder.onLoad({ filter: /\.ts$/ }, args => {
const relative = path.relative(checkout, args.path);
if (relative.startsWith("..") || relative.includes("node_modules")) return;
return { contents: execFileSync("git", ["show", `${baselineRef}:${relative}`], { cwd: checkout, encoding: "utf8" }),
loader: "ts", resolveDir: path.dirname(args.path) };
});
} }] : [],
});
return outfile;
};
const modules = { baseline: await buildVariant("baseline"), treatment: await buildVariant("treatment") };
const makeFixture = async label => {
const root = await fs.mkdtemp(path.join(tmpdir(), `backup-publication-${label}-`));
const backupRoot = path.join(root, "backups");
const liveRoot = path.join(root, "live");
const pendingRoot = path.join(backupRoot, "pending");
await fs.mkdir(liveRoot);
await fs.writeFile(path.join(liveRoot, "untouched.txt"), "live");
await fs.chmod(liveRoot, 0o555);
for (const version of ["current", "previous"]) {
const versionRoot = path.join(backupRoot, version);
for (let directory = 0; directory < directories; directory++) {
const target = path.join(versionRoot, `skill-${directory}`);
await fs.mkdir(target, { recursive: true });
for (let file = 0; file < filesPerDirectory; file++) {
await fs.writeFile(path.join(target, `file-${file}.txt`), "retired instructions\n".repeat(4), { mode: 0o444 });
}
await fs.chmod(target, 0o555);
}
await fs.symlink(liveRoot, path.join(versionRoot, "outside"));
await fs.chmod(versionRoot, 0o555);
}
await fs.mkdir(pendingRoot);
await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement");
return { root, backupRoot, liveRoot, pendingRoot };
};
const run = async variant => {
const fixture = await makeFixture(variant);
const worker = fork(filename, ["--worker", "--module", modules[variant]], { stdio: ["ignore", "inherit", "inherit", "ipc"] });
const exited = new Promise(resolve => worker.once("exit", resolve));
const nextMessage = kind => new Promise((resolve, reject) => {
const onMessage = message => {
if (message.kind !== kind && message.kind !== "error") return;
worker.off("message", onMessage); worker.off("exit", onExit);
if (message.kind === "error") reject(new Error(message.message)); else resolve(message);
};
const onExit = code => reject(new Error(`worker exited before ${kind}: ${code}`));
worker.on("message", onMessage); worker.once("exit", onExit);
});
const { port } = await nextMessage("ready");
const agent = new Agent({ keepAlive: true, maxSockets: 512 });
const requests = [];
let sent = 0;
const epoch = performance.now();
const dispatch = intended => {
const dispatched = performance.now();
requests.push(new Promise(resolve => {
const request = get({ hostname: "127.0.0.1", port, agent }, response => {
response.resume();
response.on("end", () => resolve({ intended, dispatchLateMs: dispatched - intended,
latencyMs: performance.now() - intended, active: response.headers["x-publication-active"] === "1", error: false }));
});
request.setTimeout(10_000, () => request.destroy(new Error("request timeout")));
request.on("error", () => resolve({ intended, dispatchLateMs: dispatched - intended,
latencyMs: performance.now() - intended, active: false, error: true }));
}));
};
let ticker;
const pump = () => {
const now = performance.now();
while (epoch + (sent + 1) * intervalMs <= now) dispatch(epoch + (++sent * intervalMs));
ticker = setTimeout(pump, Math.max(0, epoch + (sent + 1) * intervalMs - performance.now()));
};
ticker = setTimeout(pump, intervalMs);
try {
await delay(250);
const pending = nextMessage("result");
const start = performance.now();
worker.send({ kind: "publish", ...fixture });
const publication = await pending;
const end = performance.now();
await delay(250);
clearTimeout(ticker);
const completed = await Promise.all(requests);
const during = completed.filter(request => request.intended >= start && request.intended <= end);
assert(during.length > 0);
const result = { variant, ...publication, sent, completed: completed.length,
errors: completed.filter(request => request.error).length, duringRequests: during.length,
requestsServedDuringPublication: completed.filter(request => request.active).length,
foregroundP95Ms: percentile(during.map(request => request.latencyMs), 0.95),
foregroundMaxMs: Math.max(...during.map(request => request.latencyMs)),
generatorMaxLateMs: Math.max(...completed.map(request => request.dispatchLateMs)) };
console.log(JSON.stringify(result));
return result;
} finally {
clearTimeout(ticker); agent.destroy();
if (worker.connected) worker.send({ kind: "close" });
await exited;
await fs.chmod(fixture.liveRoot, 0o700);
if ((await fs.readdir(fixture.backupRoot)).join() === "current") await fs.rm(fixture.root, { recursive: true });
}
};
const measurements = [];
for (let pair = 0; pair < pairs; pair++) {
for (const variant of pair % 2 ? ["treatment", "baseline"] : ["baseline", "treatment"]) measurements.push(await run(variant));
}
const report = { baselineRef: execFileSync("git", ["rev-parse", baselineRef], { cwd: checkout, encoding: "utf8" }).trim(),
treatmentHead: execFileSync("git", ["rev-parse", "HEAD"], { cwd: checkout, encoding: "utf8" }).trim(),
treatmentDiff: execFileSync("git", ["diff", "HEAD", "--", "server/src/services/native-runtime/native-harness-backup-cleanup.ts", "server/src/services/native-runtime/native-harness-backup-publication.ts"], { cwd: checkout, encoding: "utf8" }),
node: process.version, platform: `${process.platform}/${process.arch}`, pairs, directories, filesPerDirectory,
filesRemovedPerPublication: 2 * directories * filesPerDirectory, intervalMs, evidence, measurements };
const output = arg("output", path.join(evidence, "result.json"));
await fs.writeFile(output, JSON.stringify(report, null, 2));
console.log(`Evidence: ${output}`);
}
server/scripts/benchmark-profile-review-projection.ts
+54
View File
@@ -0,0 +1,54 @@
import assert from "node:assert/strict";
import { isDeepStrictEqual } from "node:util";
import { pathToFileURL } from "node:url";
import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db";
import { createToolProfileService } from "../src/services/tool-access/profiles/service.js";
import { createToolAppAttention } from "../src/services/tool-access/catalog/attention.js";
import { createProfileProjectionFixture } from "../src/__tests__/helpers/tool-profile-projection-fixture.js";
const option = (name: string, fallback: string) => { const i = process.argv.indexOf(name); return i < 0 ? fallback : process.argv[i + 1] ?? fallback; };
const baselineProfiles = option("--baseline-profiles", ""), baselineAttention = option("--baseline-attention", "");
if (!baselineProfiles || !baselineAttention) throw new Error("--baseline-profiles and --baseline-attention modules required");
const beforeProfiles = (await import(pathToFileURL(baselineProfiles).href)).createToolProfileService as typeof createToolProfileService;
const beforeAttention = (await import(pathToFileURL(baselineAttention).href)).createToolAppAttention as typeof createToolAppAttention;
const profiles = Number(option("--profiles", "200")), repeats = Number(option("--repeat", "5"));
if (!Number.isInteger(profiles) || profiles < 2 || !Number.isInteger(repeats) || repeats < 1) throw new Error("Positive fixture counts required");
const database = await startEmbeddedPostgresTestDatabase("profile-review-projection-");
const db = createDb(database.connectionString, { maxConnections: 1 });
let queries = 0, rows = 0;
const measuredDb = new Proxy(db, { get(target, key) {
if (key !== "select") return Reflect.get(target, key);
return (...fields: Parameters<Db["select"]>) => {
const builder = target.select(...fields), from = builder.from.bind(builder);
builder.from = ((...args: Parameters<typeof from>) => {
const selection = from(...args), execute = selection.execute.bind(selection);
selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; };
return selection;
}) as typeof builder.from;
return builder;
};
} });
try {
const fixture = await createProfileProjectionFixture(db, { profiles });
const now = () => new Date("2026-02-01T00:00:00Z");
const implementations = {
before: { profiles: beforeProfiles(measuredDb, now), attention: beforeAttention(measuredDb, now) },
after: { profiles: createToolProfileService(measuredDb, now), attention: createToolAppAttention(measuredDb, now) },
};
const invoke = async (label: keyof typeof implementations) => {
const service = implementations[label];
return { profiles: await service.profiles.listProfiles(fixture.company.id), attention: await service.attention(fixture.company.id) };
};
const baseline = await invoke("before"), treatment = await invoke("after");
assert.ok(isDeepStrictEqual(baseline, treatment), "Complete profile and attention results must match baseline");
console.log(JSON.stringify({ fixture: { profiles, tools: 1900, meanRulesPerProfile: 380, ruleCounts: [65, 695], removedTools: 695, agents: 12, staleReferenceProfiles: Math.floor(profiles / 2) }, equality: true, profilePendingCounts: [...new Set(treatment.profiles.map(p => p.newToolsPendingCount))] }));
for (let round = 0; round < repeats; round++) {
for (const label of round % 2 ? ["after", "before"] as const : ["before", "after"] as const) {
global.gc?.(); queries = 0; rows = 0;
const start = performance.now(), cpu = process.cpuUsage();
const result = await invoke(label);
const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu);
assert.equal(result.profiles.length, profiles);
console.log(JSON.stringify({ round, label, elapsedMs, cpuMs: (used.user + used.system) / 1000, queries, rows, profiles, forcedGc: Boolean(global.gc) }));
}
}
} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); }
server/scripts/benchmark-skill-list.ts
+3 -3
View File
@@ -16,7 +16,7 @@ import { authUsers, companies, companyMemberships, companySkills, companySkillSo
projectSkillSources, projects, applyPendingMigrations, closeRegisteredClients, createDb,
startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js";
import * as schema from "../../packages/db/src/schema/index.js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../../packages/db/src/response-capture.js";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../src/__tests__/helpers/full-database-response-capture.js";
import { withRunExecutionTiming } from "../src/services/execution-preparation-timing.js";
const argument = (name: string) => process.argv.find(value => value.startsWith(`${name}=`))?.slice(name.length + 1);
@@ -195,7 +195,7 @@ try {
};
const first = await request(); assert.ok(first.etag);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = captureValues ? createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined;
const capture = captureValues ? createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, schema, capture);
const samples = [];
try {
@@ -213,7 +213,7 @@ try {
} finally { capture?.flush(); }
assert.equal(await sourceHash(), measuredSourceHash, "Source changed during measurement");
const timings = samples.map(sample => sample.elapsedMs).sort((a, b) => a - b);
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1");
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1");
if (capture) assert.ok(summary && summary.inFlightAtEnd === 0 && summary.overflowCalls === 0);
const rowCount = (await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).length;
console.log(JSON.stringify({ schema: "paperclip.skill-list-benchmark.v1", sourceRevision, measuredSourceHash,
server/scripts/benchmark-tool-profile-matching.ts
+51
View File
@@ -0,0 +1,51 @@
import assert from "node:assert/strict";
import { pathToFileURL } from "node:url";
import { and, eq } from "drizzle-orm";
import { agents, createDb, type Db } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js";
import { createToolProfileMatchingFixture } from "../src/__tests__/helpers/tool-profile-matching-fixture.js";
import { createToolProfileResolver } from "../src/services/tool-access/profiles/resolution.js";
const baselinePath = process.argv[process.argv.indexOf("--baseline") + 1];
if (!process.argv.includes("--baseline") || !baselinePath) throw new Error("--baseline module required");
const baselineFactory = (await import(pathToFileURL(baselinePath).href)).createToolProfileResolver as typeof createToolProfileResolver;
const database = await startEmbeddedPostgresTestDatabase("profile-matching-benchmark-");
const db = createDb(database.connectionString, { maxConnections: 1 });
let queries = 0, rows = 0;
const observedDb = new Proxy(db, { get(target, key) {
if (key !== "select") return Reflect.get(target, key);
return (...fields: Parameters<Db["select"]>) => {
const builder = target.select(...fields), from = builder.from.bind(builder);
builder.from = ((...args: Parameters<typeof from>) => {
const selection = from(...args), execute = selection.execute.bind(selection);
selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; };
return selection;
}) as typeof builder.from;
return builder;
};
} });
try {
const { company, agent } = await createToolProfileMatchingFixture(db);
const dependencies = {
assertOptionalAgent: async (companyId: string, agentId: string | null | undefined) => {
const valid = await observedDb.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId!)));
assert.equal(valid.length, 1);
},
resolveInstalledConnectionsForAgent: async () => [],
toCatalogEntry: () => { throw new Error("Full catalog definitions are not used by this assignment benchmark"); },
};
const before = baselineFactory(observedDb, dependencies);
const after = createToolProfileResolver(observedDb, dependencies);
const invoke = (resolver: typeof after) => resolver.getEffectiveToolAssignmentsForAgent(company.id, agent.id);
assert.deepEqual(await invoke(after), await invoke(before));
for (let i = 0; i < 2; i++) { await invoke(before); await invoke(after); }
for (let round = 0; round < 7; round++) {
for (const [label, resolver] of round % 2 ? [["after", after], ["before", before]] as const : [["before", before], ["after", after]] as const) {
queries = 0; rows = 0;
const cpu = process.cpuUsage(), start = performance.now();
for (let i = 0; i < 10; i++) await invoke(resolver);
const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu);
console.log(JSON.stringify({ round, label, operations: 10, tools: 1890, rules: 1575, agents: 12, queries, rows, elapsedMs, cpuMs: (used.user + used.system) / 1000 }));
}
}
} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); }
server/scripts/bundle-runtime.test.mjs
+36
View File
@@ -130,6 +130,42 @@ describe("bundleRuntime", () => {
}
});
it("attributes asynchronous database operations in a plain runtime bundle", async () => {
const captureModule = fileURLToPath(new URL("../../packages/db/src/response-capture.ts", import.meta.url));
write(path.join(root, "server", "dist", "services", "database-owner.js"), [
`import { withDatabaseOperation } from ${JSON.stringify(captureModule)};`,
'export const run = (owner, db) => withDatabaseOperation(owner, async () => {',
' await Promise.resolve();',
' return await db.$client.unsafe("select $1", ["private-parameter"]);',
'});',
].join("\n"));
write(path.join(root, "server", "dist", "index.js"), [
`import { createDatabaseResponseCapture, installDatabaseResponseCapture } from ${JSON.stringify(captureModule)};`,
'import { run } from "./services/database-owner.js";',
'const events = [];',
'const now = Date.now();',
'const capture = createDatabaseResponseCapture({ start: now, end: now + 60000 }, event => events.push(event));',
'const db = { $client: { unsafe: () => ({ then(resolve, reject) { return Promise.resolve(Object.assign([["private-result"]], { columns: [{ name: "value" }] })).then(resolve, reject); } }) } };',
'installDatabaseResponseCapture(db, {}, capture);',
'await Promise.all([run("fixture.first", db), run("fixture.second", db)]);',
'capture.flush();',
'console.log(JSON.stringify(events));',
].join("\n"));
const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir: path.join(root, "server", "dist-runtime") });
const env = { ...process.env };
delete env.NODE_OPTIONS;
delete env.PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW;
const { stdout } = await execFileAsync(process.execPath, [summary.outfile], { cwd: path.join(root, "server"), env, timeout: 30_000 });
const events = JSON.parse(stdout);
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
expect(entries.map(entry => entry.owner).sort()).toEqual(["fixture.first", "fixture.second"]);
for (const entry of entries) expect(entry).toMatchObject({ calls: 1, rows: 1, callers: [] });
expect(events.at(-1)).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2 });
expect(stdout).not.toContain("private-parameter");
expect(stdout).not.toContain("private-result");
expect(stdout).not.toContain(root);
});
it("emits a server module that loads without starting the server", async () => {
const outdir = path.join(root, "server", "dist-runtime");
const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir });
server/src/__tests__/agent-auth-identity-transfer.test.ts
+3 -3
View File
@@ -4,7 +4,7 @@ import request from "supertest";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
import { actorMiddleware } from "../middleware/auth.js";
@@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport();
it("bounds actual middleware responses and preserves the complete stored run", async () => {
const fixture = await seed();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
@@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport();
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
server/src/__tests__/agent-directory-working-copies.test.ts
+87 -1
View File
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js";
@@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j
import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js";
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js";
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { heartbeatRunEvents } from "@paperclipai/db";
@@ -123,6 +124,44 @@ describe("persistent agent directories", () => {
for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true);
} finally { await worker.stop(); }
});
it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => {
const copy = await run();
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm");
try {
await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready;
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(100);
expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull();
});
await expect.poll(() => idleWorkSnapshot().active).toBe(0);
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(120_000);
expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled();
expect(worker.nextWakeAt()).toBeNull();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { await worker.stop(); }
});
it("honors a cleanup obligation published before stale-deadline settlement", async () => {
const copy = await run(), dueAt = new Date(Date.now() + 30_000);
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const transact = db.transaction.bind(db);
const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
return transact(callback, config);
});
try {
await copies.release(companyId, copy.runId);
expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null });
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { race.mockRestore(); }
});
it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => {
const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory");
clock(); const worker = coordinator();
@@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => {
return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId };
}
it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)(
"recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => {
const { copy, lease } = await unavailableRemote();
const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID();
const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000);
await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" });
await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment",
issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } })
.where(eq(environmentLeases.id, lease.id)).returning();
const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt,
releasedAt: new Date(), metadata: { agentId } };
if (scenario === "foreign_company") {
const foreignCompanyId = randomUUID();
await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) });
successor.companyId = foreignCompanyId;
}
const handoff = createReusableLeaseHandoff(previous!, successor);
const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId,
state: scenario === "stopped" ? "stopped" : "destroyed" });
await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } })
.where(eq(environmentLeases.id, lease.id));
await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination,
reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } });
if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" });
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute"));
try {
await copies.recoverStopped();
const recovered = (await copies.get(companyId, copy.runId))!;
if (scenario === "destroyed") {
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE",
receipt: { cleanupPending: false }, nextAttemptAt: null });
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
} else {
expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null });
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
}
expect(shell).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination");
} finally { shell.mockRestore(); }
});
it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])(
"defers unavailable cleanup without exact destruction proof (%s)", async proof => {
const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed",
server/src/__tests__/agent-record-read-projections.test.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => {
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
try {
const result = await run(observed);
server/src/__tests__/ai-connections.test.ts
+40
View File
@@ -1496,6 +1496,46 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello"
} finally { fetchSpy.mockRestore(); }
});
it("reads provider identity only for an authorized credential and preserves connected state when profile access is denied", async () => {
const owner = "identity-owner";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const account = await service.save(companyId, owner, { provider: "anthropic", method: "subscription", ownership: "personal", name: "Identity account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "identity-selected-secret");
const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ account: { email: "avery@example.test", uuid: "private-account-id" }, organization: { name: "Example team", uuid: "private-org-id" } }));
const app = express();
app.use((req, _res, next) => {
req.actor = req.header("x-test-anonymous") ? { type: "none" }
: req.header("x-agent") ? { type: "agent", agentId, companyId, companyIds: [companyId], onBehalfOfUserId: owner }
: { type: "board", source: "session", userId: req.header("x-test-user") ?? owner, companyIds: [companyId] };
next();
});
app.use("/api", aiConnectionRoutes(db));
app.use((error: { status?: number; message: string }, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); });
const url = `/api/companies/${companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
try {
const result = await request(app).get(url);
expect(result.status).toBe(200);
expect(result.headers["cache-control"]).toBe("no-store");
expect(result.body).toMatchObject({ ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" });
expect(fetchSpy).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ headers: expect.objectContaining({ Authorization: "Bearer identity-selected-secret" }) }));
for (const value of ["identity-selected-secret", "private-account-id", "private-org-id"]) expect(JSON.stringify(result.body)).not.toContain(value);
fetchSpy.mockReset();
expect((await request(app).get(url).set("x-test-user", "bob")).status).toBe(404);
expect((await request(app).get(url.replace(companyId, otherCompanyId))).status).toBe(403);
expect((await request(app).get(url.replace(account.grantId, randomUUID()))).status).toBe(404);
expect((await request(app).get(url).set("x-test-anonymous", "yes")).status).toBe(401);
expect((await request(app).get(url).set("x-agent", "yes")).status).toBe(403);
expect((await request(app).get(url.replace(account.connectionId, "invalid"))).status).toBe(404);
expect(fetchSpy).not.toHaveBeenCalled();
fetchSpy.mockResolvedValue(Response.json({ error: "private-provider-error" }, { status: 403 }));
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "permission_denied", accountLabel: null });
expect((await service.list(companyId, owner)).find(item => item.id === account.connectionId)?.status).toBe("connected");
fetchSpy.mockClear();
await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "connection_unavailable" });
expect(fetchSpy).not.toHaveBeenCalled();
} finally { fetchSpy.mockRestore(); }
});
it("uses the same usage probe for shared accounts across legacy and native runner selections", async () => {
const account = await service.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "shared", name: "Shared usage account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "shared-usage-secret");
const selected = { provider: "anthropic", method: "subscription", mode: "shared", ...account } as const;
server/src/__tests__/attention-service.test.ts
+7 -7
View File
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
agents,
@@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => {
})));
}
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
@@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => {
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, {
@@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
.toEqual([...keptIds].sort());
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
@@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => {
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
try {
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
@@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(index.totalCount).toBe(feed.totalCount);
} finally { summaryCapture.flush(); }
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
expect(cardReads).toEqual([]);
server/src/__tests__/chat-channels.integration.test.ts
+235 -11
View File
@@ -35,11 +35,13 @@ import { budgetService } from "../services/budgets.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js";
import { createSystemRecovery } from "../services/system-recovery-state.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
import type { Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
import * as activityLogModule from "../services/activity-log.js";
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
@@ -114,6 +116,7 @@ import {
isNotNull,
isNull,
like,
ne,
notInArray,
or,
sql,
@@ -2891,7 +2894,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
companyMemberships, chatActions, heartbeatRuns }, capture);
@@ -2920,7 +2923,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
} };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
let projected: { event: Record<string, unknown> };
try {
@@ -7164,6 +7167,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0);
expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0);
});
it("offers Retry for a saved automatic PR event after a successful older review", async () => {
const f = await reviewBotFixture();
const payload = {
action: "opened", installation: { id: 2468 },
repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } },
sender: { id: 77, login: "maintainer" },
pull_request: { number: 81, title: "Review successive heads", body: "", draft: false,
base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) },
user: { id: 42, login: "octocat", type: "User" }, labels: [] },
};
const send = async (event: typeof payload) => {
await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({
event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret,
}));
await f.service.processPendingDeliveries();
};
await send(payload);
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await readChatConversationsByEndpointId(f.endpoint.id)).length;
}, { timeout: 10_000 }).toBe(1);
const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id);
const [task] = await readIssuesById(conversation.issueId!);
const [original] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")));
await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1);
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId,
issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id,
contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] },
}).returning();
await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id));
await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } });
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length;
}).toBe(2);
const [pending] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id)));
await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1);
const [receipt] = await readAgentWakeupRequestsById(pending.id);
await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null,
payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]),
requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id));
expect(receipt.requestedByActorType).toBe("user");
await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id));
expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" }))
.toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id });
await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } })
.where(eq(agents.id, f.assignedAgentId));
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() });
f.wakeup.mockImplementation(heartbeatService(db).wakeup);
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user",
actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry });
await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId });
const [resumed] = await readAgentWakeupRequestsById(pending.id);
expect(resumed.runId).toBeTruthy();
expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } });
expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId });
});
it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => {
const f = await reviewBotFixture();
const { buildSkillMentionHref } = await import("@paperclipai/shared");
@@ -62894,7 +62959,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const commentsBeforeDisabledLifecycle = await db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId));
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id);
const priorLifecycleIds = new Set(
(
await db
@@ -62974,7 +63040,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId)),
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id),
).resolves.toEqual(commentsBeforeDisabledLifecycle);
await service.shutdown();
});
@@ -64168,6 +64235,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
return { ...f, pending, originalDelivery };
}
async function successfulGitHubQueuedInputFixture() {
const f = await currentGitHubTaskRetryFixture(true);
const companyId = f.fixture.companyId, issueId = f.issue.id;
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload,
[String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id));
const contractId = randomUUID(), resultId = randomUUID();
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" });
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed",
completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId,
completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId,
resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning();
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id));
const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant",
observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry };
return { ...f, input, result };
}
it("retries saved GitHub input after a successful run without replaying that run", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const { input } = f;
const companyId = f.fixture.companyId;
const before = (await readHeartbeatRunsById(f.runId))[0];
const [request, duplicate] = await Promise.all([requestTaskRetry(input),
requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]);
expect(request.accepted).toBe(true);
expect(duplicate.requestId).toBe(request.requestId);
await reconcileTaskChatRetryReceipts(db, { companyId });
const [source] = await readAgentWakeupRequestsById(f.pending!.id);
expect(source.runId).toBeTruthy();
const [successor] = await readHeartbeatRunsById(source.runId!);
expect(successor).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(request.requestId!))[0])
.toMatchObject({ status: "coalesced", runId: successor.id });
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)(
"keeps saved-input Retry behind current admission gates: %s", async gate => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const request = await requestTaskRetry(f.input);
const before = (await readHeartbeatRunsById(f.runId))[0];
if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id));
if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } })
.where(eq(chatActions.id, f.pending!.id));
if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id));
if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId));
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
if (gate === "cleanup_pending") {
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
}
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
if (gate === "cleanup_pending") {
expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution");
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
}
} finally { await f.service.shutdown(); }
});
it.each(["issue.comment.reopen", "boardMessageRetry"] as const)(
"resumes saved GitHub input after a successful board turn: %s", async origin => {
const f = await successfulGitHubQueuedInputFixture();
try {
const [previous] = await readHeartbeatRunsById(f.runId);
await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: {
...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }),
} }).where(eq(heartbeatRuns.id, f.runId));
const before = (await readHeartbeatRunsById(f.runId))[0];
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null,
contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
} finally { await f.service.shutdown(); }
});
it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id,
authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning();
const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId,
source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution",
requestedByActorType: "user", requestedByActorId: f.input.actorId,
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id,
_paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" },
}, [comment.id]),
}).returning();
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued",
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board);
} finally { await f.service.shutdown(); }
});
it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => {
const f = await successfulGitHubQueuedInputFixture();
const { issueRoutes } = await import("../routes/issues.js");
const app = express();
app.use(express.json());
app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); });
app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service }));
app.use(errorHandler);
try {
const body = { observedRunId: f.runId, idempotencyKey: randomUUID() };
const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(first.status, JSON.stringify(first.body)).toBe(202);
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(second.status).toBe(202);
expect(second.body.requestId).toBe(first.body.requestId);
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
const detail = await request(app).get(`/api/issues/${f.issue.id}`);
expect(detail.status).toBe(200);
expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false });
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
@@ -64297,7 +64521,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
status: "active", membershipRole: "operator" });
}
const queryEvents: Record<string, unknown>[] = [];
const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1,
const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1,
end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments,
agentWakeupRequests, chatEndpoints, chatConversations }, capture);
@@ -64317,7 +64541,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]);
if (capture) {
capture.flush();
expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries))
expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries))
.toBeLessThan(180);
}
expect(two.requestId).toBe(one.requestId);
@@ -67517,7 +67741,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.issue.id };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -67534,7 +67758,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
@@ -77242,7 +77466,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -77259,7 +77483,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
server/src/__tests__/chat-completion-delivery.test.ts
+72 -1
View File
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import { and, asc, eq } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs,
companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
companies, createDb, documents, environmentLeases, executionWorkspaces, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, workspaceOperations } from "@paperclipai/db";
import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { getExecutionBlocker } from "../services/execution-blocker.js";
@@ -15,6 +15,8 @@ import { documentService } from "../services/documents.js";
import { instanceSettingsService } from "../services/instance-settings.js";
import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js";
import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js";
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
(support.supported ? describe : describe.skip)("chat completion delivery", () => {
let db: ReturnType<typeof createDb>;
@@ -68,6 +70,75 @@ import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../ser
})]);
expect(JSON.stringify(run.contextSnapshot?.chatCompletionUpdates)).not.toMatch(/Injected|credentials|Plan accepted|Agent approved/);
});
it("retains a distinct completion receipt behind a real dependency gate", async () => {
const f = await seed();
const blocker = await f.create();
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [blocker.id] });
await f.finish();
const heartbeat = heartbeatService(db);
// The generic dependency signal is replaceable. The outbox's exact wake
// receipt is not: it needs that identity to reconcile a refused admission.
await heartbeat.wakeup(f.agentId, {
source: "automation", triggerDetail: "system", reason: "issue_continuation_needed",
requestedByActorType: "system", requestedByActorId: "dependency-test",
payload: { issueId: f.sourceId }, contextSnapshot: { issueId: f.sourceId },
});
const [delivery] = await f.rows();
const service = chatCompletionDeliveryService(db, heartbeat);
await service.deliver(delivery.id);
const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId));
expect(receipts).toEqual(expect.arrayContaining([
expect.objectContaining({ idempotencyKey: `chat-completion:${delivery.id}:0`,
reason: "issue_dependencies_blocked", status: "skipped", runId: null,
payload: expect.objectContaining({ chatCompletionDeliveryIds: [delivery.id] }) }),
]));
expect(receipts).toHaveLength(2);
expect(receipts.find(receipt => receipt.idempotencyKey?.startsWith("execution-wait:")))
.toMatchObject({ coalescedCount: 0 });
expect((await f.rows())[0]).toMatchObject({ attempts: 0, status: "pending", targetRunId: null, error: null });
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
expect(await issueService(db).getDependencyReadiness(f.sourceId)).toMatchObject({
isDependencyReady: false, unresolvedBlockerIssueIds: [blocker.id],
});
});
it("admits a saved worker completion after its owning finalize encloses the snapshot", async () => {
const f = await seed(), workspaceId = randomUUID(), workerRunId = randomUUID();
await db.update(issues).set({ responsibleUserId: "operator" }).where(eq(issues.id, f.sourceId));
await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId,
mode: "isolated_workspace", strategyType: "managed_remote", name: "Saved worker", providerType: "daytona" });
await db.insert(heartbeatRuns).values({ id: workerRunId, companyId: f.companyId, agentId: f.agentId,
status: "succeeded", contextSnapshot: { issueId: f.task.id } });
await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.task.id));
await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [f.task.id] });
await db.insert(workspaceOperations).values([
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
phase: "workspace_finalize", status: "succeeded", startedAt: new Date(10_000), finishedAt: new Date(12_000) },
{ companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId,
phase: "remote_workspace_snapshot", status: "succeeded", startedAt: new Date(10_100), finishedAt: new Date(11_900) },
]);
await f.finish();
const deps = createHeartbeatQueueTestDependencies(db), issueSvc = issueService(db);
deps.getAgent.mockImplementation(id => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null));
deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental());
deps.issuesSvc.getDependencyReadiness.mockImplementation(issueSvc.getDependencyReadiness);
deps.issuesSvc.listDependencyReadiness.mockImplementation(issueSvc.listDependencyReadiness);
// Admit the real durable run, then hold provider dispatch at the normal drain boundary.
deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" });
const queue = createHeartbeatQueue(db, deps);
const service = chatCompletionDeliveryService(db, { wakeup: queue.trackWakeup });
const [delivery] = await f.rows();
await service.deliver(delivery.id);
const [queued] = await f.rows();
expect(queued).toMatchObject({ status: "queued", attempts: 0, error: null, targetRunId: expect.any(String) });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.targetRunId!));
expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId: f.sourceId,
wakeReason: "chat_task_completed", chatCompletionDeliveryIds: [delivery.id] } });
const prepared = await prepareChatCompletionTurn(db, run);
expect(prepared.contextSnapshot?.chatCompletionUpdates).toEqual([
expect.objectContaining({ id: f.task.id, status: "done", completedAt: expect.any(Date), url: `/issues/${f.task.identifier ?? f.task.id}` }),
]);
expect(deps.executeRun).not.toHaveBeenCalled();
});
it("dispatches a committed completion without activity publication and recovers after restart", async () => {
const f = await seed();
const run = vi.fn(() => f.service.sweepPending());
server/src/__tests__/company-skills-service.test.ts
+10 -10
View File
@@ -17,7 +17,7 @@ import {
projectWorkspaces,
} from "@paperclipai/db";
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => {
]);
await svc.list(companyId);
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { agents, costEvents }, capture);
let listed: Awaited<ReturnType<typeof svc.list>>;
try { listed = await svc.list(companyId); } finally { capture.flush(); }
expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2);
expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1);
expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0);
const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents");
const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents");
expect(agentReads).toHaveLength(1);
expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000);
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId));
const refreshed = await svc.list(companyId, { q: "First" });
expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1);
@@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => {
const before = await svc.list(companyId);
expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core");
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkills }, capture);
let after: Awaited<ReturnType<typeof svc.list>>;
try { after = await svc.list(companyId); } finally { capture.flush(); }
expect(after).toEqual(before);
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills");
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills");
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000);
const stored = await svc.getById(companyId, inserted[0]!.id);
expect(stored?.markdown).toBe(savedMarkdown);
@@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
.orderBy(companySkillVersions.id);
expect(before.length).toBeGreaterThan(0);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "company_skill_versions" && event.operation === "select");
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
@@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
? { type, companyId: otherCompanyId, agentId: randomUUID() }
: { type, companyIds: [otherCompanyId] };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
finally { capture.flush(); }
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0);
});
it("restores a deleted release after warm validation", async () => {
server/src/__tests__/connection-management.integration.test.ts
+97 -5
View File
@@ -6,9 +6,10 @@ import { join } from "node:path";
import { createServer } from "node:http";
import { and, eq, sql } from "drizzle-orm";
import { beforeAll, afterAll, describe, expect, it, vi } from "vitest";
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db";
import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js";
import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js";
import { aiConnectionService } from "../services/ai-connections.js";
import { redactRunnerApiPayloads } from "../services/native-runtime/runner-api-log-redaction.js";
import * as remoteEndpointGuard from "../services/remote-http-endpoint-guard.js";
import * as remoteHttpFetch from "../services/remote-http-fetch.js";
@@ -87,13 +88,14 @@ describe("connection management with production default agents", () => {
authority: new PaperclipRunnerToolAuthority(server.db, { companyId: base.companyId, agentId: agent.id, issueId: base.issueId, runId: base.runId, apiUrl: server.apiUrl, storage: server.storage }) };
}
it("matches ordinary install capabilities on real PostgreSQL for scoped, retained and fallback resources", async () => {
it.each(["tool", "ai"] as const)("matches ordinary %s install capabilities on real PostgreSQL for scoped, retained and fallback resources", async (connectionPurpose) => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Capability target ${index}`, adapterType: "process" })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Capability differential", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Capability differential", transport: "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
uid: randomUUID(), name: "Capability differential", connectionPurpose,
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning();
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds.slice(0, 2) } });
const service = connectionManagementService(server.db);
@@ -177,7 +179,7 @@ describe("connection management with production default agents", () => {
expect(await faulting.describeAgentInstallCapabilities(localActor, resource, [candidateIds[1]])).toEqual([denied]);
});
it("preserves the complete grants GET across human roles and denies a real install mutation after revocation", async () => {
it.each(["tool", "ai"] as const)("preserves the complete %s grants GET across human roles and denies a real install mutation after revocation", async (connectionPurpose) => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, randomUUID(), randomUUID()];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Grant target ${index}`, adapterType: "process" })));
@@ -198,7 +200,8 @@ describe("connection management with production default agents", () => {
scope: permissionKey === "tools:manage_profiles" ? { targetAgentIds: candidateIds.slice(0, 2) } : null })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Grants response parity", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Grants response parity", transport: "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
uid: randomUUID(), name: "Grants response parity", connectionPurpose,
transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning();
const grants = await server.db.insert(connectionGrants).values([
{ companyId: f.companyId, connectionId: connection.id, kind: "organization", createdByUserId: users.creator, isDefault: true },
...[users.creator, users.manager, users.member].map((subjectUserId, index) => ({ companyId: f.companyId, connectionId: connection.id, kind: "user" as const,
@@ -253,6 +256,65 @@ describe("connection management with production default agents", () => {
expect(savedInstalls).toHaveLength(2);
});
it("checks company-wide personal AI installs once per target and preserves pool member authorization", async () => {
const f = await server.fixture({ connectionScenario: "fresh" });
const candidateIds = [f.agentId, ...Array.from({ length: 19 }, () => randomUUID())];
await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `AI target ${index}`, adapterType: "process" })));
const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "AI capability checks", type: "mcp_http" }).returning();
const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Personal AI account", transport: "runtime_auth", connectionPurpose: "ai", authKind: "api_key",
credentialPolicy: "per_user", createdByUserId: f.responsibleUserId }).returning();
const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id,
kind: "user", subjectUserId: f.responsibleUserId, createdByUserId: f.responsibleUserId }).returning();
await server.db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection.id, targetType: "company", targetId: f.companyId });
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } });
let targetReads = 0;
const observedDb = new Proxy(server.db, { get(target, key, receiver) {
if (key !== "select") return Reflect.get(target, key, receiver);
return (...fields: Parameters<typeof target.select>) => {
const builder = target.select(...fields), from = builder.from.bind(builder);
builder.from = ((...args: Parameters<typeof from>) => {
const query = from(...args), execute = query.execute.bind(query);
query.execute = (async (...executeArgs: Parameters<typeof execute>) => {
if (query.toSQL().sql.includes('"agents"."id" =')) targetReads++;
return execute(...executeArgs);
}) as typeof execute;
return query;
}) as typeof builder.from;
return builder;
};
} });
const service = connectionManagementService(observedDb);
const actor: AuthorizationActor = { type: "board", userId: f.responsibleUserId, source: "cloud_tenant" };
const resource = { type: "connection_management" as const, companyId: f.companyId, connectionId: connection.id, operation: "configure", readOnly: false };
const baseline = [];
for (const targetAgentId of candidateIds) baseline.push(await service.capability(actor, "connections:profiles", { ...resource, targetAgentId }));
expect(baseline.every(result => result.allowed)).toBe(true);
expect(targetReads).toBe(candidateIds.length ** 2);
targetReads = 0;
expect(await service.describeAgentInstallCapabilities(actor, resource, candidateIds)).toEqual(baseline);
expect(targetReads).toBe(candidateIds.length);
const [pool] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
uid: randomUUID(), name: "Pool with private member", transport: "runtime_auth", connectionPurpose: "ai", authKind: "none" }).returning();
await server.db.insert(aiConnectionPools).values({ id: pool.id, companyId: f.companyId, pluginKey: "fixture-pool", config: {
name: "Pool", enabled: true, mode: "round_robin", thresholdPercent: 90,
members: [{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", connectionId: connection.id, grantId: grant.id }, profile: { provider: "codex", model: "fixture-model" } }],
} });
await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, f.companyId), eq(principalPermissionGrants.principalId, f.responsibleUserId!)));
await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!,
permissionKey: "tools:manage_profiles", scope: { connectionIds: [pool.id] } });
const poolResource = { ...resource, connectionId: pool.id };
expect((await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).every(result => result.allowed)).toBe(true);
// The pool grant cannot replace the member account's own authority.
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id));
await server.db.update(toolConnections).set({ createdByUserId: "different-owner" }).where(eq(toolConnections.id, connection.id));
const ordinary = await service.capability(actor, "connections:profiles", { ...poolResource, targetAgentId: candidateIds[0] });
expect(ordinary.allowed).toBe(false);
expect(await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).toEqual([ordinary, ordinary]);
});
it("creates and configures tool connections and profiles with fresh grants and actual key/JWT authentication", async () => {
const f = await fixture();
const grants = await server.db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.agentId));
@@ -1642,6 +1704,36 @@ describe("connection management with production default agents", () => {
} finally { provider = null; unregisterServerAdapter("paperclip_runner"); }
});
it("reads only the authorized Anthropic account identity through actual agent authentication", async () => {
const f = await fixture();
const account = await aiConnectionService(server.db).save(f.companyId, f.userId, {
provider: "anthropic", method: "subscription", ownership: "personal", name: "Account identity fixture",
loginSessionId: "fixture", agentIds: [f.agentId], allAgents: false,
}, "selected-identity-credential");
let calls = 0;
provider = async (url, init) => {
calls++;
expect(url).toBe("https://api.anthropic.com/api/oauth/profile");
expect(new Headers(init?.headers).get("authorization")).toBe("Bearer selected-identity-credential");
return Response.json({ account: { uuid: "private-seat", email: "avery@example.test" }, organization: { uuid: "private-org", name: "Example team" } });
};
const path = `/companies/${f.companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`;
try {
for (const token of [f.key.token, f.jwt]) {
const response = await f.request("GET", path, undefined, token);
expect(response).toMatchObject({ status: 200, body: { ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" } });
for (const secret of ["selected-identity-credential", "private-seat", "private-org"]) expect(JSON.stringify(response)).not.toContain(secret);
}
expect(calls).toBe(2);
expect((await f.request("GET", path.replace(f.companyId, f.foreignCompanyId))).status).toBe(403);
expect((await f.request("GET", path.replace(account.grantId, randomUUID()))).status).toBe(404);
expect(calls).toBe(2);
await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId));
expect(await f.request("GET", path)).toMatchObject({ status: 200, body: { status: "unavailable", errorCode: "connection_unavailable", accountLabel: null } });
expect(calls).toBe(2);
} finally { provider = null; }
});
it("filters AI account and pool listings by the agent's resolved connection scope", async () => {
const f = await fixture();
provider = async url => {
server/src/__tests__/conversation-settlement-transfer.test.ts
+3 -3
View File
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
@@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport();
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
const input = await seed(reply), before = await sourceDigests(input);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
&& (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
server/src/__tests__/database-operation-attribution.test.ts
+137
View File
@@ -0,0 +1,137 @@
import express from "express";
import request from "supertest";
import { QueryPromise } from "drizzle-orm/query-promise";
import { afterEach, expect, it, vi } from "vitest";
import { withDatabaseOperation } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals";
import { databaseOperationMiddleware } from "../middleware/database-operation.js";
import { withBudgetSpan } from "../services/budget-tracing.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
import {
createRunPreparationTiming,
measureRunPhase,
withRunExecutionTiming,
withRunResponsePresentationTiming,
withRunWorkspaceFinalizationTiming,
} from "../services/execution-preparation-timing.js";
const { owners } = await vi.hoisted(async () => {
const { AsyncLocalStorage } = await import("node:async_hooks");
return { owners: new AsyncLocalStorage<string>() };
});
vi.mock("@paperclipai/db", async importOriginal => ({
...await importOriginal<typeof import("@paperclipai/db")>(),
withDatabaseOperation: <T>(owner: string, action: () => T): T => owners.run(owner, action),
}));
class OwnerQuery extends QueryPromise<string | undefined> {
async execute() { return owners.getStore(); }
}
afterEach(() => {
vi.unstubAllEnvs();
vi.useRealTimers();
});
it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
await withDatabaseOperation("http.request", async () => {
const query = new OwnerQuery();
await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt");
await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization");
await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation");
await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read");
await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup");
await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory");
await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction");
await expect(query).resolves.toBe("http.request");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
const failure = new Error("private accounting failure");
await withRunExecutionTiming("private-run", async () => {
for (let i = 0; i < 18; i++) {
await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory");
}
await expect(measureRunPhase("execution.budget_reservation", async () => {
await expect(withBudgetSpan("budget.admission.reserve", async () => {
await Promise.resolve();
expect(owners.getStore()).toBe("budget.admission.reserve");
throw failure;
})).rejects.toBe(failure);
return new OwnerQuery();
})).resolves.toBe("execution.budget_reservation");
await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
const app = express();
app.use(databaseOperationMiddleware);
app.use(express.json());
let arrived = 0;
let release!: () => void;
const both = new Promise<void>(resolve => { release = resolve; });
app.all("/:privateId", async (req, res) => {
const before = await new OwnerQuery();
const work = async () => {
if (++arrived === 2) release();
await both;
return new OwnerQuery();
};
const during = req.params.privateId === "first"
? await withBudgetSpan("budget.admission.reserve", work)
: await measureRunPhase("skills.inventory", work);
res.json({ before, during, after: await new OwnerQuery() });
});
const [first, second] = await Promise.all([
request(app).get("/first?secret=private-first").expect(200),
request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200),
]);
expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" });
expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" });
expect(owners.getStore()).toBeUndefined();
});
it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => {
vi.useFakeTimers();
const execute = vi.fn(async () => [{ xid: "42" }]);
const database = installDatabaseWorkSignals({ execute, async transaction<T>(action: (tx: object) => Promise<T>): Promise<T> {
return action({ execute });
} });
const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = [];
const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true,
onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); },
});
let fail = true;
try {
const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue =>
coordinator.register(queue, { retryMs: 5000,
run: async () => {
await Promise.resolve();
seen.push({ queue, phase: "run", owner: await new OwnerQuery() });
if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); }
},
hasPending: async () => {
seen.push({ queue, phase: "pending", owner: await new OwnerQuery() });
return false;
},
})));
await Promise.all(workers.map(worker => worker.ready));
await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx =>
notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback)));
await vi.advanceTimersByTimeAsync(1);
expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]);
for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`);
expect(owners.getStore()).toBeUndefined();
expect(coordinator.nextWakeAt()).toBeNull();
} finally {
await coordinator.stop();
}
});
server/src/__tests__/environment-cleanup-work.test.ts
+12 -6
View File
@@ -1,5 +1,5 @@
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
import { createOwnedSandbox } from "../services/sandbox-creation-operations.js";
import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js";
import { mkdtemp, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -32,8 +32,8 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
const errors: unknown[] = [], disposers: Array<() => void> = [];
beforeAll(async () => {
if (!external) database = await startEmbeddedPostgresTestDatabase("environment-cleanup-work-");
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`)));
}, 90_000);
beforeEach(async () => {
await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`);
@@ -48,6 +48,9 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
afterEach(async () => {
worker?.stop(); await coordinator?.stop(); coordinator = undefined; worker = undefined;
for (const stop of disposers.splice(0)) stop();
// Creation ownership intentionally survives company deletion; company
// cascades therefore cannot isolate this case from the next test suite.
await db.delete(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId));
vi.useRealTimers(); vi.restoreAllMocks();
expect(errors).toEqual([]);
});
@@ -116,15 +119,18 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
]));
await quiet();
});
it("settles a deleted-company creation as manual history across restart without idle polling", async () => {
it.each([false, true])("retains deleted-company creation uncertainty across a quiet worker restart (contextual=%s)", async contextual => {
await start(); await quiet();
await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} },
const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {},
metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} },
async () => ({ providerLeaseId: "fixture-deleted-company" }));
await db.delete(companies).where(eq(companies.id, companyId));
await settle(31 * 60_000);
await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([
expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }),
]));
if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId,
providerLeaseId: created.providerLeaseId });
await quiet();
worker!.stop(); await coordinator!.stop();
await start(); await quiet();
@@ -132,7 +138,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => {
const ownerId = randomUUID();
const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 };
expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none"))
.toEqual({ version: 1, backgroundWork: "none" });
.toEqual({ version: 1, backgroundWork: "present" });
await quiet();
});
it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => {
server/src/__tests__/environment-task-authority.integration.test.ts
+105 -2
View File
@@ -1,11 +1,12 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db";
import { resolvePluginTaskResume } from "../services/environment-task-runtime.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
@@ -20,6 +21,108 @@ databaseSuite("environment task lease authority", () => {
});
afterAll(async () => { await stopDb?.(); });
it("replaces a stopped warm candidate within its tracked startup context", async () => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
const calls: string[] = [];
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void) => {
calls.push(method === "environmentTask" ? params.operation.kind : method);
try {
if (method === "environmentAcquireLease") return { providerLeaseId: "replacement-task" };
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
return { kind: "accepted", taskId: params.taskId };
} finally { settled?.(); }
},
} as unknown as PluginWorkerManager;
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
try {
await expect(environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
operation: { kind: "describe" }, executionContext: context })).resolves.toMatchObject({ kind: "description", taskId: "replacement-task" });
expect(calls).toEqual(["status", "status", "complete", "environmentAcquireLease", "describe"]);
expect(context.pending).toBe(false);
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
expect(lease.providerLeaseId).toBe("replacement-task");
expect(lease.metadata?.runnerTaskReused).toBe(false);
expect(JSON.stringify(lease.metadata)).not.toContain(context.id);
} finally { context.finish(); }
});
it.each(["late", "post-result", "retiring"])("queues a replacement instead of attaching after %s startup cancellation", async mode => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" });
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready",
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never });
await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin",
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId,
provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment",
metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } });
const calls: string[] = [];
let reply!: () => Promise<void>, acquiring!: () => void;
const entered = new Promise<void>(resolve => { acquiring = resolve; });
const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }),
call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void, _contextId?: string, observe?: (result: any) => Promise<void>) => {
calls.push(method === "environmentTask" ? params.operation.kind : method);
try {
if (method === "environmentAcquireLease") return await new Promise(resolve => {
reply = async () => { const result = { providerLeaseId: "replacement-task", metadata: { binding: "fresh" } };
await observe?.(result);
if (mode === "retiring") await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskRetiring: true } }).where(eq(environmentLeases.id, leaseId));
resolve(result); };
acquiring();
});
if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true };
if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId,
runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 };
return { kind: "accepted", taskId: params.taskId };
} finally { settled?.(); }
},
} as unknown as PluginWorkerManager;
const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000);
if (mode === "post-result") {
const wait = context.wait.bind(context);
vi.spyOn(context, "wait").mockImplementation(operation => wait(async () => {
const value = await operation(); context.abort(new Error("Cancelled after recording")); return value;
}));
}
try {
const request = environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId,
operation: { kind: "describe" }, executionContext: context });
const rejected = expect(request).rejects.toThrow("operation unavailable");
await entered;
if (mode === "late") {
context.abort(new Error("Startup cancelled"));
await rejected;
expect(context.pending).toBe(true);
}
await reply(); await rejected;
expect(context.pending).toBe(false);
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId));
expect(lease.providerLeaseId).toBe("stopped-task");
const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.runId, runId));
expect(creation.state).toBe("cleanup_pending");
expect(creation.metadata.runnerTaskCreationContextId).toBe(context.id);
const [cleanup] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, creation.leaseId!));
expect(cleanup).toMatchObject({ heartbeatRunId: runId, providerLeaseId: "replacement-task", status: "pending_cleanup" });
expect(cleanup.metadata).toMatchObject({ runnerTaskLease: true, providerMetadata: { binding: "fresh" } });
expect(calls).not.toContain("describe");
} finally { context.finish(); }
});
it("authorizes a new task lease only after scoped prior execution has stopped", async () => {
const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID();
const oldRunId = randomUUID(), nextRunId = randomUUID(), oldLeaseId = randomUUID(), nextLeaseId = randomUUID();
server/src/__tests__/environment-task-runtime.test.ts
+62 -4
View File
@@ -2,14 +2,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest";
import { PgDialect } from "drizzle-orm/pg-core";
import { executionWorkspaces, environmentLeases, type Db } from "@paperclipai/db";
import { environmentTaskOperationSchema, parseEnvironmentTaskResult } from "@paperclipai/plugin-sdk";
import { executeEnvironmentTask } from "../services/environment-task-runtime.js";
import { executeEnvironmentTask, settleDispatchedEnvironmentTask } from "../services/environment-task-runtime.js";
import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js";
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn() }));
const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn(), identity: vi.fn(), config: vi.fn() }));
vi.mock("../services/provider-fresh-workspace.js", () => ({
stageProviderFreshWorkspace: async (input: { existing: unknown }) => input.existing,
releaseProviderRecoveryTransfers: state.release,
}));
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin) }) }));
vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin), getConfig: state.config }) }));
vi.mock("../services/plugin-agent-runtime-state.js", () => ({ pluginAgentRuntimeIdentity: state.identity }));
const leaseId = "10000000-0000-4000-8000-000000000001";
const projectId = "10000000-0000-4000-8000-000000000002";
const secondProjectId = "10000000-0000-4000-8000-000000000003";
@@ -25,7 +27,7 @@ function database(value: ReturnType<typeof row> | null = row(), projectRows = [{
const query: any = { from: (table: unknown) => { selectedRows = table === executionWorkspaces ? workspaceRows : table === environmentLeases ? ownerRows : projectRows; return query; }, leftJoin: () => query, where: vi.fn(() => query), limit: () => Promise.resolve(results.shift()), then: (resolve: (rows: typeof projectRows) => unknown) => Promise.resolve(selectedRows).then(resolve as (rows: unknown[]) => unknown) };
const sets: Record<string, unknown>[] = [];
const update: any = { set: (value: Record<string, unknown>) => { sets.push(value); return update; }, where: () => update,
returning: async () => [{ id: leaseId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
returning: async () => [{ id: leaseId, metadata: value?.lease.metadata, heartbeatRunId: value?.lease.heartbeatRunId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) };
const db = { select: () => query, update: () => update,
transaction: vi.fn(async (write: (tx: Db) => Promise<unknown>) => write(db as unknown as Db)) };
return { db: db as unknown as Db, query, sets, transaction: db.transaction };
@@ -37,9 +39,65 @@ const submit = { kind: "submit" as const, preparedLaunchId: "configuration", pro
beforeEach(() => {
state.release.mockReset();
state.identity.mockReset().mockResolvedValue({ instanceSecrets: { launcher: "synthetic-transient-binding" } });
state.config.mockReset().mockResolvedValue({ configJson: { endpoint: "synthetic" } });
state.plugin = { id: "original", pluginKey: "test.provider", status: "ready", manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } };
});
describe("plugin-provided Runner execution", () => {
it.each([false, true])("retires an undispatched warm task only with required preparation proof (%s)", async preparationStopped => {
const value = row();
Object.assign(value.lease.metadata, { runnerTaskDispatched: false,
runnerTaskStartupCleanup: { contextId: "startup", requiresPreparationStop: true } });
const fixture = database(value), workers = worker();
fixture.db.select = ((...args: unknown[]) => (database(value).db.select as (...values: unknown[]) => unknown)(...args)) as Db["select"];
let release!: () => void;
const drain = new Promise<void>(resolve => { release = resolve; });
const drainStartup = vi.fn(() => drain);
Object.assign(workers, { drainStartup });
const operations: string[] = [];
workers.call.mockImplementation(async (...args: any[]) => {
const operation = args[2].operation.kind; operations.push(operation);
return operation === "status" ? { kind: "status", taskId: "attempt-1", phase: "cancelled", executionStopped: true, preparationStopped }
: { kind: "accepted", taskId: "attempt-1" };
});
const retiring = settleDispatchedEnvironmentTask(fixture.db, workers as never, { companyId: "company", id: leaseId });
const checked = retiring.then(value => ({ value, error: null }), error => ({ value: false, error }));
await vi.waitFor(() => expect(operations).toContain("stop"));
await vi.waitFor(() => expect(operations).toContain("status"));
expect(drainStartup).toHaveBeenCalledWith("startup");
expect(operations).not.toContain("complete");
release();
const result = await checked;
if (preparationStopped) { expect(result).toEqual({ value: true, error: null }); expect(operations).toEqual(["stop", "status", "complete"]); }
else { expect(result.error?.message).toContain("termination is unconfirmed"); expect(fixture.sets).toContainEqual(expect.objectContaining({ status: "pending_cleanup", cleanupStatus: "failed" })); }
});
it("keeps legacy payloads passive without capability or transport activation", async () => {
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: "run" } } as never, 1_000);
const workers = worker();
workers.call.mockImplementation(async (...args: any[]) => { args[5]?.(); return { kind: "description", taskId: "attempt-1", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; });
try {
for (const executionContext of [undefined, context]) {
await executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId,
operation: { kind: "describe" }, ...(executionContext ? { executionContext } : {}) });
const sent = workers.call.mock.calls.at(-1) as unknown as any[];
expect(sent[2]).not.toHaveProperty("executionContextSupported");
expect(sent[2]).not.toHaveProperty("executionContext");
if (!executionContext) expect(sent).toHaveLength(4);
}
expect(state.identity).not.toHaveBeenCalled();
expect(state.config).not.toHaveBeenCalled();
expect(context.pending).toBe(false);
} finally { context.finish(); }
});
it.each(["json", "wrong-run"])("rejects a %s context before the worker", async mode => {
const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: mode === "wrong-run" ? "other" : "run" } } as never, 1_000);
const workers = worker();
try {
await expect(executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, operation: { kind: "describe" },
executionContext: mode === "json" ? JSON.parse(JSON.stringify(context)) : context })).rejects.toThrow("native_execution_context");
expect(workers.call).not.toHaveBeenCalled();
} finally { context.finish(); }
});
it("forwards a retained fresh directory intent on later launch calls and checks provider support", async () => {
const value = row();
value.lease.metadata.runnerTaskProjectAuthValidation.projectIds = [];
server/src/__tests__/github-operation-credentials.test.ts
+3 -3
View File
@@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
import { errorHandler } from "../middleware/index.js";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import {
agents,
@@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
});
await db.insert(toolConnections).values(irrelevant);
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { toolConnections }, capture);
try {
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
@@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
}).where(eq(heartbeatRuns.id, input.runId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
server/src/__tests__/heartbeat-dependency-scheduling.test.ts
+3 -8
View File
@@ -689,14 +689,9 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () =
});
expect(interactionWake).not.toBeNull();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, interactionWake!.id))
.then((rows) => rows[0] ?? null);
return run?.status === "succeeded";
});
// This checks dependency interaction semantics, not provider startup speed.
// Join the admitted execution before inspecting its persisted outcome.
await heartbeat.drainActiveRunExecutions();
const interactionRun = await db
.select({
server/src/__tests__/heartbeat-native-cleanup-admission.test.ts
+111 -2
View File
@@ -1,10 +1,15 @@
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db";
import { agents, authUsers, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { heartbeatService } from "../services/heartbeat.js";
import { NativeStartupCleanupPendingError, readNativeStartupCleanup } from "../services/native-runtime/native-startup-cleanup.js";
import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js";
import { environmentService } from "../services/environments.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createIsolatedTestHome } from "./helpers/isolated-test-home.js";
const execute = vi.hoisted(() => vi.fn());
vi.mock("../adapters/index.js", async () => {
@@ -70,4 +75,108 @@ suite("native terminal-run cleanup admission", () => {
expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded");
expect(execute).toHaveBeenCalledTimes(3);
}, 30_000);
it("retries a failed startup intent through timer failure and drain before verified cancellation", async () => {
const home = createIsolatedTestHome(), originalHome = process.env.PAPERCLIP_HOME;
process.env.PAPERCLIP_HOME = home.home;
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID();
const contextId = randomUUID(), leaseOwner = randomUUID(), taskId = randomUUID(), deadlineAtMs = Date.now() - 1;
const originalSetTimeout = globalThis.setTimeout;
const retries: Array<{ timer: ReturnType<typeof setTimeout>; fire(): void }> = [];
const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => {
if (typeof delay === "number" && delay <= 30_000 && new Error().stack?.includes("scheduleNativeSessionResumeDispatch")) {
const timer = originalSetTimeout(() => {}, 3_600_000);
retries.push({ timer, fire() { clearTimeout(timer); callback(...args); } });
return timer;
}
return originalSetTimeout(callback, delay, ...args);
}) as typeof setTimeout);
let failSuppression = false, runId = "";
const runtimeEnv = new Proxy<Record<string, string | undefined>>({}, { get(_target, key) {
if (key === "PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS" && failSuppression) {
failSuppression = false;
throw new Error("Scheduling dependency unavailable");
}
return undefined;
} });
const heartbeat = heartbeatService(db, { runtimeEnv });
try {
await db.insert(companies).values({ id: companyId, name: "Intent retry", issuePrefix: "CIR", defaultResponsibleUserId: "owner" });
await db.insert(authUsers).values({ id: "owner", name: "Fixture user", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner", status: "active", membershipRole: "member" });
await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process",
adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } });
await db.insert(issues).values({ id: issueId, companyId, title: "Recover startup", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" });
const [environment] = await db.insert(environments).values({ companyId, name: "Controlled provider", driver: "plugin", config: {} }).returning();
await db.execute(sql`create table test_cleanup_intent_fault (enabled boolean not null)`);
await db.execute(sql`insert into test_cleanup_intent_fault values (true)`);
await db.execute(sql`create sequence test_cleanup_intent_attempts`);
await db.execute(sql.raw(`create function test_reject_cleanup_intent() returns trigger language plpgsql as $$ begin
if new.company_id = '${companyId}'::uuid and new.error_code = 'native_startup_cleanup_pending'
and exists (select 1 from test_cleanup_intent_fault where enabled) then
perform nextval('test_cleanup_intent_attempts'); raise exception 'Cleanup intent write unavailable';
end if; return new; end $$`));
await db.execute(sql`create trigger test_cleanup_intent_fault before update on heartbeat_runs for each row execute function test_reject_cleanup_intent()`);
execute.mockReset();
execute.mockImplementation(async (context: { runId: string }) => {
runId = context.runId;
const [run] = await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: null }).where(eq(heartbeatRuns.id, runId)).returning();
await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Recover startup", description: null }, environmentLeaseId: leaseId });
await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId));
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId));
await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1,
controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) });
await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId,
issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", metadata: { runnerTaskLease: true } });
await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({
nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId,
contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 },
})}::jsonb` }).where(eq(heartbeatRuns.id, runId));
throw new NativeStartupCleanupPendingError(contextId, deadlineAtMs, leaseOwner, false, leaseId);
});
await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual");
await heartbeat.drainActiveRunExecutions();
const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!);
const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!);
expect((await db.execute<{ last_value: string }>(sql`select last_value from test_cleanup_intent_attempts`))[0].last_value).toBe("2");
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "running", errorCode: null });
expect((await owner()).leaseOwner).toBe(leaseOwner);
expect(await lease()).toMatchObject({ status: "active", cleanupStatus: null });
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)).toBeNull();
const drainedRetries = retries.length;
expect(drainedRetries).toBeGreaterThan(0);
await heartbeat.resumeQueuedRunsAfterDrain();
expect(retries.length).toBeGreaterThan(drainedRetries);
const resumedRetries = retries.length;
failSuppression = true;
retries.at(-1)!.fire();
await vi.waitFor(() => expect(retries.length).toBeGreaterThan(resumedRetries));
expect((await owner()).leaseOwner).toBe(leaseOwner);
expect((await heartbeat.getRun(runId))?.status).toBe("running");
await db.execute(sql`update test_cleanup_intent_fault set enabled = false`);
retries.at(-1)!.fire();
await vi.waitFor(async () => expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("pending"));
await vi.waitFor(async () => expect((await heartbeat.listEvents(runId)).some(event => event.eventType === "native.recovery.transition")).toBe(true));
const current = await lease();
await environmentService(db).releaseLease(leaseId, "released", { cleanupStatus: "success",
remoteExecutionTermination: remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }) });
await heartbeat.cancelRun(runId, "Startup cleanup retry completed");
expect(await heartbeat.getRun(runId)).toMatchObject({ status: "cancelled" });
expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("complete");
expect(await owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null });
expect(execute).toHaveBeenCalledOnce();
} finally {
await heartbeat.drainActiveRunExecutions();
timeout.mockRestore();
for (const retry of retries) clearTimeout(retry.timer);
await db.execute(sql`drop trigger if exists test_cleanup_intent_fault on heartbeat_runs`);
await db.execute(sql`drop function if exists test_reject_cleanup_intent()`);
await db.execute(sql`drop table if exists test_cleanup_intent_fault`);
await db.execute(sql`drop sequence if exists test_cleanup_intent_attempts`);
if (originalHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = originalHome;
home.cleanup();
}
}, 30_000);
});
server/src/__tests__/heartbeat-run-event-sequencing.test.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
agents,
companies,
@@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
server/src/__tests__/heartbeat-run-log.test.ts
+18
View File
@@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => {
expect(compacted).not.toContain("paperclip-flag-secret");
});
});
describe("compactRunLogChunk with long base64 commands", () => {
it("redacts a tool call that writes base64 of source text without stalling", () => {
const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n";
const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024);
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`;
const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`;
const startedAt = performance.now();
const compacted = compactRunLogChunk(chunk);
expect(performance.now() - startedAt).toBeLessThan(1_000);
expect(compacted).toContain(blob);
expect(compacted).not.toContain("tool-call-secret");
expect(compacted).toContain("***REDACTED***");
});
});
server/src/__tests__/heartbeat-run-privacy-routes.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import express from "express";
import request from "supertest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
}).where(eq(heartbeatRuns.id, f.privateRunId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const app = createApp(f.companyId, f.grantedAgentId, observed);
server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts
+7 -39
View File
@@ -791,19 +791,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
`));
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const [run, wakeup] = await Promise.all([
db.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null),
db.select({ status: agentWakeupRequests.status })
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, wakeupRequestId))
.then((rows) => rows[0] ?? null),
]);
return run?.status === "cancelled" && wakeup?.status === "skipped";
});
await heartbeat.drainActiveRunExecutions();
const [run, wakeup, issue] = await Promise.all([
db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode })
@@ -970,16 +958,10 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(ordering).toEqual(["validated", "parked"]);
};
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
return run?.status === "cancelled";
});
await heartbeat.drainActiveRunExecutions();
await parkPromise;
expect((await heartbeat.getRun(runId))?.status).toBe("cancelled");
const issue = await db
.select({ status: issues.status })
.from(issues)
@@ -1108,7 +1090,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
expect(run).not.toBeNull();
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
await heartbeat.drainActiveRunExecutions();
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
@@ -1126,7 +1108,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
expect(run).not.toBeNull();
await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0);
await heartbeat.drainActiveRunExecutions();
expect(countExecuteCallsForRun(run!.id)).toBe(1);
});
@@ -1718,13 +1700,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
.where(eq(agentWakeupRequests.id, wakeupRequestId));
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const [deferred] = await db
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.id, deferredWakeupId));
return Boolean(deferred?.runId) && deferred?.status !== "deferred_issue_execution";
});
await heartbeat.drainActiveRunExecutions();
const [deferred] = await db
.select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId })
@@ -1788,15 +1764,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
await heartbeat.resumeQueuedRuns();
await waitForCondition(async () => {
const run = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, runId))
.then((rows) => rows[0] ?? null);
return run?.status === "cancelled";
});
await heartbeat.drainActiveRunExecutions();
const [run, wakeup, issue] = await Promise.all([
db
server/src/__tests__/helpers/embedded-postgres.ts
+4 -32
View File
@@ -1,35 +1,7 @@
import { randomUUID } from "node:crypto";
import {
applyPendingMigrations,
createDb,
getEmbeddedPostgresTestSupport as embeddedSupport,
startEmbeddedPostgresTestDatabase as startEmbedded,
export {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
type EmbeddedPostgresTestDatabase,
type EmbeddedPostgresTestSupport,
} from "@paperclipai/db";
export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport };
export async function getEmbeddedPostgresTestSupport(): Promise<EmbeddedPostgresTestSupport> {
return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport();
}
export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise<EmbeddedPostgresTestDatabase> {
const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
if (!adminUrl) return startEmbedded(prefix);
const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`;
const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE;
if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name");
const admin = createDb(adminUrl, { maxConnections: 1 });
const url = new URL(adminUrl);
url.pathname = `/${name}`;
const cleanup = async () => {
try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); }
finally { await admin.$client.end({ timeout: 1 }); }
};
try {
await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`);
if (!template) await applyPendingMigrations(url.toString());
return { connectionString: url.toString(), cleanup };
} catch (error) { await cleanup(); throw error; }
}
Loaded 100 of 314 files, more files were not shown because too many files have changed in this diff. Show more