Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-02 16:58:52 +01:00
Code Issues Packages Projects Releases Wiki Activity

Integrate main while preserving qualified question journey

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-10-30 15:37:39 -05:00
2 parents 7f7de8be28 fd008979f5
commit e795866d1e
197 files changed
+14764 -817
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/chat-feature-testops.yml
+448
View File
@@ -0,0 +1,448 @@
name: Chat feature TestOps
run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Fixed evaluation, zero-result registration, or one TestOps plan launch
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps supplies its job-run identity; no test/source selector is accepted
ALLURE_USERNAME:
type: string
required: false
description: Reserved TestOps service metadata; never used as authorization
permissions:
contents: read
concurrency:
group: chat-feature-testops
cancel-in-progress: false
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.guard.outputs.sha }}
operation: ${{ steps.guard.outputs.operation }}
triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify fixed default-source dispatch and both actor identities
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
NODE_OPTIONS: ''
NODE_PATH: ''
run: node tests/runner-e2e/testops-chat-dispatch.ts
select:
name: Validate TestOps selection or register zero-result setup
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
sha256: ${{ steps.selection.outputs.sha256 }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register the public job without an evaluation or a synthetic result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap
- name: Validate the one real TestOps selection before any build or provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection
- name: Start exactly the validated TestOps plan and exit without waiting for its child
if: needs.authorize.outputs.operation == 'launch-plan'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}
CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
GH_TOKEN: ${{ github.token }}
run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan
- name: Retain only closed metadata from zero-result setup
if: always() && needs.authorize.outputs.operation == 'bootstrap'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json
${{ runner.temp }}/chat-testops-bootstrap-state/once.json
${{ runner.temp }}/chat-testops-bootstrap-state/plan.json
${{ runner.temp }}/chat-testops-bootstrap-state/job.json
${{ runner.temp }}/chat-testops-bootstrap-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain the one plan-launch receipt and stop state
if: always() && needs.authorize.outputs.operation == 'launch-plan'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-plan-launch-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-launch-plan-state/once.json
${{ runner.temp }}/chat-testops-launch-plan-state/launch.json
${{ runner.temp }}/chat-testops-launch-plan-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain only the token-free exact selection receipt
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-selection-${{ github.run_id }}-1
path: testops-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable shipped native candidate without model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- name: Install only the source-reviewed graph without lifecycle scripts
run: |
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
pnpm install --frozen-lockfile --ignore-scripts
- name: Build the existing TypeScript and native runner owners
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package source-bound outputs and resolved lock
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1
path: |
chat-testops-build.tar.gz
chat-testops-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: One real native OpenCode backlog request on AWS
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
id-token: write
env: &retention
PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1
TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TARGET_SHA: ${{ needs.authorize.outputs.sha }}
MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Check out the same admitted source for trusted retention
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-testops-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- name: Verify immutable outputs and token-free TestOps selection
env:
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
(cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256)
tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE"
node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json
cmp chat-testops-lock.json chat-testops-worker-lock.json
cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store")
- name: Materialize and qualify pinned OpenCode runtime and browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Prepare private evidence with shipped-main authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only the exact private cell storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify strict private custody before the model key
env: &cell_session
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the fixed original request with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain exact original grade and private evidence, including failures
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and export only reviewed fixture results
needs: [authorize, select, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies only
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store"
- name: Prepare exact private report and cell-read policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact private cell reader
id: read_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the exact private original and verify all inventories
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: |
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing normalized campaign without exposing original logs
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately even when its grade failed
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only the bound normalized result, JUnit and marked PNGs
id: export
if: always() && steps.private_report.outcome == 'success'
env:
TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts
- name: Upload only the sanitized TestOps projection
if: always() && steps.export.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Report the original outcome to its exact TestOps case
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- *allure
- name: Verify and upload only the bound safe case outcome
id: allure_upload
env: &testops_reporter
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results
run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops
- name: Verify the actual TestOps result after upload processing
id: verify_upload
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *testops_reporter
run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops
- name: Retain only closed actual TestOps verification metadata
if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-verify-upload-state/verification.json
${{ runner.temp }}/chat-testops-verify-upload-state/failure.json
retention-days: 30
if-no-files-found: error
DESIGN.md
+2
View File
@@ -84,6 +84,8 @@ Background refreshes and history expansion keep the conversation and composer
mounted. A failed history request shows an inline retry without hiding readable
content. Full-history and classic views retain the bounded 15-second initial wait.
Standing agent chats open with the latest 20 contiguous messages and hydrate only their runs. Upward scrolling requests earlier pages; a fixed-height status slot says “Loading earlier messages…” while the page is pending. Preserve the logical row anchor and mounted composer through prepends and background refresh. Retry failed history inline without automatic retry loops. Answer receipts and created-task history use the same scrolling affordance.
Only the viewer's own human comments sit on the right in blue bubbles. Agents
and other humans sit on the left on the page surface, with their own name and
avatar above the message. Human identity comes from the company directory;
doc/DATABASE.md
+9
View File
@@ -1545,6 +1545,15 @@ named `review_wait_benchmark...` can be supplied through
`PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version.
DataRow bytes exclude response metadata and transport overhead.
## Object-backed run evidence
Optional segmented run logs and raw provider traces store their payloads and
range indexes directly in S3-compatible storage. Existing `heartbeat_runs` and
`provider_trace_records` contain backend/reference and lifecycle metadata;
there is no new payload table or per-segment SQL publication. Object manifests
use conditional writes to fence publication. Existing structured events retain
their database append contract. See [run evidence storage](run-evidence-storage.md).
## Cloud runtime configuration singleton
The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly
doc/DEVELOPING.md
+10
View File
@@ -3883,6 +3883,16 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`,
also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those
URLs; the sleep suite truncates company data between tests.
## S3-backed run evidence
New logs and opted-in provider traces automatically use the instance's existing
S3 storage configuration, bucket, credentials and prefix, under `run-logs/` and
`provider-traces/`. No extra storage switches are needed. Local-disk instances
remain local. See [run evidence storage](run-evidence-storage.md) for required provider
capabilities, stable keys, operating limits, automatic history migration and qualification.
Do not remove a web volume until its complete retained evidence inventory and
the independent disposable-server requirements are satisfied.
## Cloud startup configuration
Cloud stores nonsecret startup choices in a private PostgreSQL singleton.
doc/connections/AI-CONNECTIONS.md
+27 -16
View File
@@ -580,22 +580,33 @@ transaction or an intentional freshness/authorization discard. These retained
homes are recovery evidence, not a background replay queue; persistent database
or lock failures still require operator intervention.
Quota OAuth replacement tokens are encrypted with the instance secrets master
key and fsynced under `<instance-root>/quota-credential-recovery/<company-id>/`
before vault, grant, or activity writes. Storage and encryption are checked before
exchanging a single-use refresh token. Failed saves retry without another OAuth
exchange; persistent failures keep the encrypted record for the next quota poll,
including after a restart. New OpenAI subscription runtimes serialize credential
reads with quota exchanges and recover a matching pending replacement before
materializing an auth home. Authentication-failure handling also recovers a
matching replacement before marking the grant invalid. A failed recovery save
defers these actions and leaves both the active grant and journal intact.
The journal is removed only after database commit (or
when an authorized newer credential makes it obsolete). Replay checks the grant,
connection, secret, and original credential fingerprint and cannot reactivate a
revoked grant or overwrite a reconnect. Back up this directory and the instance
secrets key with the instance data. Loss of durable storage while receiving a
provider token can still require reconnecting the account.
Server-owned quota refresh validates and locks the current grant and secret in
Postgres before exchanging a token. A failed database prerequisite prevents the
exchange. A returned replacement is saved directly into the existing secret
store. The same logical operation keeps it in memory for at most three save
attempts. A lost commit response is reconciled against the current saved value;
reconnect, revocation, secret-version and session-epoch changes win over a delayed
save. Save retries never repeat the OAuth exchange and no new recovery file is
written for this path, in cloud or local deployments.
If all save attempts fail, the request fails. If the server process then exits,
the unsaved replacement is lost. V1 accepts signing in again after the next failed
use; it does not report successful refresh or keep a second credential vault.
Remote Codex Runner tasks retain their private credential until secret writeback
and the lease's credential handoff commit. Provider completion follows that
handoff. Failed exports or database saves leave it pending for the existing
cleanup path, including a replacement web receiver. Its authority still checks
the company, grant, secret, version selector and session epoch.
Existing encrypted records in
`<instance-root>/quota-credential-recovery/<company-id>/` remain readable for
cutover settlement. Do not delete unresolved files or remove the old volume
before inventory and settlement. Legacy replay cannot reactivate a revoked grant
or overwrite a reconnect. Local Google Antigravity retirement still uses its
existing encrypted recovery record and private-home ownership; remote and warm
Google subscription sessions remain unsupported. This local path is not a cloud
fallback. Cloud cutover must settle any observed legacy credential files first.
## Advanced provider routing (2026-10-02)
doc/evals.md
+4 -1
View File
@@ -43,7 +43,10 @@ two natural-language task scenarios into local native Codex subscription and
OpenCode Product E2E cells. They
verify saving an unstarted task and delegating a saved announcement back to chat,
using ordinary production instructions, real browser messages and independent
state/provenance checks. The broader YAML catalog remains planning inventory.
state/provenance checks. A separately selected native OpenCode/DeepSeek1017
backlog cell measures TestOps integration acceptance with the same oracle and
original bounds. Its source registration is not an AWS or TestOps pass. The
broader YAML catalog remains planning inventory.
The explicit-only [Daytona restart continuity journey](../tests/runner-e2e/README.md)
extends the existing native restart oracle to a 125-second controller outage,
doc/observability.md
+13 -1
View File
@@ -254,6 +254,17 @@ It does not extend an earlier execution attempt or include time waiting for
the background worker to run. Retries have separate roots. A check that finds
no eligible work can end without a restore child.
`task.response_presentation` owns recovery of an already committed native reply.
It starts a separate linked root because deferred presentation can run after the
execution attempt and workspace finalization have ended. Worker discovery for
a specific run and worker checks include candidate reads under
`response.resolve`. Each discovery or check owns its root; direct recovery and
its worker check share one root when nested. `response.transaction`
includes the recovery transaction; `response.transaction.entry` measures entry
into its callback, including connection acquisition. These spans retain only the
same hashed run ID and never capture reply content. A successful task status is
separate from the time the reply is saved.
These spans exist before provider dispatch, including setup failures. Preparation phases have
only the existing `paperclip.task.run.run_id` attribute: the first 12 hexadecimal
characters of the run UUID's SHA-256 hash. Failed operations set error status
@@ -641,7 +652,8 @@ Without an OTLP endpoint, tracing remains silent and does not export data.
The run link in Honeycomb matches `task.provider_session` for native runs, `task.run` for legacy adapters,
`task.execution_attempt` for server preparation and settlement, and
`task.workspace_finalization` for deferred native copyback. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the four root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged.
`task.workspace_finalization` for deferred native copyback and `task.response_presentation`
for committed native reply recovery. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the five root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged.
The canonical lifecycle is:
doc/plans/2026-10-30-chat-feature-testops-smoke.md
+89
View File
@@ -0,0 +1,89 @@
# Chat feature TestOps smoke — 2026-10-30
## Finish line
Create one dedicated runnable TestOps plan and obtain one genuine passing
`tasks.create-backlog` result through TestOps → GitHub Actions/AWS → native
OpenCode/OpenRouter `deepseek/deepseek-v4.1-flash-1017`. The result must prove
the saved task, assignment, backlog-at-creation state, no worker execution,
agent attribution, rendered link and reload persistence. Confirm cleanup,
safe screenshot/report access and the final TestOps result link.
This is a new integration-acceptance measurement. It preserves the original
244-scenario campaign, attempts, grades and existing Codex measurements.
Delegation and the other catalog scenarios are deferred.
## Source and execution contract
- Public selector: `chat-features.runner-opencode-deepseek-1017.local.create-backlog`.
- Existing YAML query, fixture and independent backlog oracle stay unchanged.
- One human query and one accounted controller work run; 120-second turns,
420-second attempt, 300-second settled-state observation, zero retries.
- Existing 500-cent company/agent policies and unknown setup usage remain
explicit. Morgan is fixture context and must not execute this backlog task.
- Only the existing protected `OPENROUTER_API_KEY` is required. Agent
configuration uses the normal encrypted secret-reference path.
- OpenCode 1017 is selected only for this cell and remains a qualification
candidate until the real run. Other suites retain their existing models.
## Ownership and integration
Campaign owns public catalog/profile calibration and these progress notes.
Core owns the public AWS workflow/execution adapter, one coherent public PR,
and the sole actual operation. Coordinator supplies the public TestOps
selection/reporting helper and the separate private credential sealer.
The public workflow owns the dedicated plan. Source peers precede publication.
The current Cloud TestOps adapter is scoped to `qa-basics` Product journeys;
the new chat plan must have its own exact plan/case mapping. Reuse normalized
result/check/provenance and existing packaged evidence. Do not upload private
traces, provider sessions, raw logs or unreviewed media as TestOps attachments.
## Current progress
The public patch adds only the OpenCode backlog selection while preserving
the two Codex cells. A genuine parent selector control failed; all 68 focused
oracle/catalog controls and the owning type check now pass. The expected
suite-size assertion was updated for the single added cell. Independent source
review passed and the nine-file profile slice is committed as `0a996033`.
No local app, database, native SDK,
provider, model, install or build qualification was run.
The separate protected credential bootstrap seals the existing reporting token
for the source-pinned public reporting environment. Delivery is complete without
exporting plaintext. The public adapter has 53 focused authority/export/workflow
controls and 39 API/CLI/result controls passing, plus owning typechecking and
workflow lint. Unchanged controls are reused after narrow source corrections.
The independent composition source peer passed.
The public infrastructure [PR 18322](https://github.com/paperclipai/paperclip/pull/18322)
is open. Its first normal CI run rejected the new workflow in the existing paid
workflow allowlist and reported a separate heartbeat scheduling assertion; both
failures remain recorded. The policy correction preserves the older workflow
guards and rejects unsafe new-route variants. The reporting token is configured
through the reviewed encrypted bootstrap.
Review also added a one-time initial administrator launch fence. A protected
expected GitHub run number and actual own-run metadata must match before any
TestOps mutation. REST catalog reconciliation preserves exact case aliases;
job wiring uses only the reviewed existing project GitHub integration. These
provider-free helper controls pass. Actual registration IDs, tenant API behavior,
provider execution and the TestOps result remain unqualified. Dedicated plan,
AWS evaluation and final result links remain pending. Source checks alone do not
satisfy the finish line.
The final source consolidation adds actual admission-wrapper filesystem coverage,
the existing reviewed lock normalization with frozen installs and a matched
build/worker receipt, plus same-job server-result readback after Allure upload.
Focused checks cover 41 admission controls, 31 lock/owner/workflow controls and
44 API/reporter controls; unchanged profile/oracle evidence is reused. This is
source verification. The plan and actual model result are still not created.
Current-main composition preserves the four existing Codex and production-qualified
OpenCode cells introduced by #18319. The TestOps cell has its own
`runner-opencode-deepseek-1017` profile, so the existing OpenCode model remains
unchanged. It uses the shipped v2 grader, queue accounting and `final-state.png`
capture. The new profile keeps the original query, one work run, one human message,
120-second turn, 420-second attempt, 300-second observation bound and zero retries.
The existing private session and trace-asset owners admit only this additional
exact selector; the frozen 244-case candidate-approval registry remains unchanged.
Source checks do not establish a TestOps registration or model outcome.
doc/plans/2026-10-30-external-run-logs-and-provider-traces.md
+567
View File
@@ -0,0 +1,567 @@
# Durable run logs and provider traces for disposable web servers
Status: implementation and PR qualification in progress. Runtime storage,
history migration, and canonical native source replay are implemented using the
instance's existing storage selection. Production-provider qualification remains
a rollout gate.
Owner: this Codex chat, through the reviewable PR. Date: 2026-10-30.
Revision: object-storage-first, following Dotta's operational correction on 2026-10-30.
Source revision inspected: `0c70cedce9a803bdc201324528abb35e423e7cd0`.
Parent effort: near-zero-downtime Railway deployment of the main web/API server.
## Goal and boundary
After the main Paperclip web server and its local filesystem disappear, a
replacement connected to the same Postgres, object storage, and secrets can
read all acknowledged run-log output and retained provider traces, then continue
recording the same remote execution without gaps or duplicate records.
The ordinary journey is: open a task's run, inspect its output, replace the web
server while its remote agent is working, and continue reading the run. An
authorized administrator can still inspect, reveal, download, and delete a
captured provider trace. Existing consumers of run output continue working.
This is the **run logs and provider traces** slice of the parent plan. Railway
still documents that volume-backed deployments cannot overlap, which makes
removing durable filesystem dependencies necessary. This slice alone does not
remove every volume dependency or prove zero-downtime HTTP deploys.
[Railway volume limitations](https://docs.railway.com/volumes/reference#caveats).
Firm requirements:
- All retained historical logs and unexpired traces survive migration. Dotta
explicitly selected this on 2026-10-30. Do not shorten existing retention.
- A successful durable append survives abrupt process and disk loss. An orderly
shutdown flush is useful but insufficient evidence.
- Preserve company/task authorization, redaction, stable log byte offsets,
trace integrity checks, opt-in capture, existing limits, expiry, and deletion.
- Store log and raw trace payloads directly in S3-compatible object storage.
Postgres holds small ownership/lifecycle/reference records only; no new
per-token or per-frame SQL, payload tables, or DB-to-S3 payload compaction.
Local files may be caches or unacknowledged buffers. Preserve local mode.
- Keep task results, usage settlement, and optional diagnostics independent.
- Preserve bounded work and existing sleep behavior.
- Reuse existing instance storage settings and per-stack bucket credentials.
New logs and opted-in traces follow S3 file storage automatically, with their
own folders beneath the existing prefix. Add no required configuration.
Dotta approved implementation on 2026-10-30. Delivery ends at a reviewable PR
with relevant checks and review findings resolved. Deployment, volume removal,
and production history migration require the parent effort's target and cutover.
The acceptance matrix below remains the full storage-slice contract; an
implemented primitive does not establish that every row has passed.
Deferred to the parent effort: managed agent files, workspace recovery,
accounting/credential journals, workspace-operation logs, readiness/draining,
global live-event fanout, scheduler ownership, volume detach, and fleet rollout.
These remain blockers to claiming the entire web server disposable. No changes
to first-party Telemetry or operator OpenTelemetry export are proposed. Raw
provider protocol traces here are a separate, restricted diagnostic store.
## Current implementation and implications
| Owner | Current behavior | Required change |
| --- | --- | --- |
| `server/src/services/run-log-store.ts` | Only `local_file`; local NDJSON append; best-effort final/in-flight S3 mirroring; local-first reads | Add an explicit external store with shared durable append and reads |
| `server/src/services/heartbeat/run-output.ts` | Redacts/compacts before append; process-local output sequence; progress persisted on a 60-second cadence | Preserve stable source identities; batch output to S3; distinguish live delivery from durable receipts |
| `server/src/services/heartbeat-run-events.ts` | Structured events already use atomic DB sequence allocation and native source deduplication | Preserve this authority; do not turn its event table into a duplicate blob store |
| `server/src/services/provider-trace-store.ts` | DB metadata points to local native and `.rehydration` sidecars; 24-hour retention and 64 MiB capture limit | Remote payload storage, recoverable capture cursors, durable expiry/deletion obligations |
| Runner trace sinks and `runnerd-codex-transport.ts` | Rust/TypeScript capture and interpretation producers use paths; TypeScript capture is bounded and fire-and-forget | Introduce a durable capture delivery boundary and preserve channel/frame lineage |
| `doc/testing-native-remote-continuity.md` | Sparse checkpoints deliberately omit streaming output; acknowledged display output may be lost before app projection | Qualify replay/retention through web replacement without restoring per-token DB checkpoint writes |
| Run readers | Heartbeat, issues, feedback, and customer-success inspection assume `local_file` | Resolve each persisted backend, including historical records, through one reader |
Do not infer cloud durability from `RunLogStore.append()` succeeding today, or
from a trace's local `acknowledgedDebugSequence`. Neither proves remote commit.
Generic stdout/stderr and raw traces cannot be reconstructed from
`heartbeat_run_events`. The implementation investigation found a narrower
surviving source: native control ingestion already commits the exact validated
PRP event before ACK. Its canonical `paperclip.prp.event` log export can replay
that existing source with an S3 cursor, without adding payload writes. This
replaces the tentative second Runner spool below for canonical native output.
## Operational decision
The original proposal used Postgres for payload chunks and then copied them to
S3. That choice was premature. It adds payload writes, WAL/replication/backup
bytes, index churn, deletion/vacuum work, and extra database connections to the
same database serving the control plane. Batching would reduce transaction
count but would not remove the byte load. No workload measurement justified it.
Use **S3 as the durable payload store from the first persisted batch**. Local
append files remain useful for buffering and cache speed; the web server's disk
cannot be their only acknowledged durable home. Existing S3 mirroring is a useful
starting point, but repeatedly uploading the entire growing file amplifies
network and storage work. Upload each newly sealed segment once instead.
| Location | Responsibility |
| --- | --- |
| Browser live stream / bounded server memory | Immediate progress display; not a durable receipt |
| Existing native control event source / Runner outbox | Existing committed native records survive ACK; unacknowledged events remain replayable at the Runner |
| S3-compatible storage | Immutable log/trace segments and bounded manifest/index pages |
| Postgres | Existing company/run ownership, backend/reference, final status and expiry/deletion state; no per-batch writes |
| Disposable web disk | Optional bounded buffer/read cache; never the only acknowledged copy |
Existing structured `heartbeat_run_events`, task results, usage, and controller
checkpoints retain their contracts. This is not a rewrite of those data paths.
### Cost model and the first gate
Let `A` be actively producing streams, `r` output records/second/stream, `b`
bytes/second/stream, `T` maximum dirty-buffer age, and `S` target segment bytes.
For a steady stream, approximate segment rate is
`A × max(1/T, b/S)`; record rate is `A × r`. Empty streams emit no segments.
Bursts, final flushes, retries, indexes, and trace channels add work and must be
counted separately. A stream is not necessarily an agent: raw tracing can add
multiple streams to one run.
Illustrative arithmetic, **not a fleet measurement or chosen production setting**:
100 continuously active streams at 10 records/second give 1,000 records/second.
If five-second flushing dominates, they produce 20 segments/second, or 72,000
segments/hour. With one small metadata publication transaction per segment,
that is still 20 transactions/second. A 30-second interval gives about 3.3
segments/second but retains more pending output. S3 index PUTs, GETs, retries,
and transfer costs are additional. Neither option is free.
The first gate measures real output sizes/rates and concurrency, then compares
candidate size/age thresholds (initial experiment: 256 KiB–1 MiB and 5–30
seconds, not defaults). Include quiet text, bursty tool output, raw tracing,
long runs, slow S3, and the expected concurrent-run envelope. Record:
- Incremental Postgres transactions, row updates, WAL bytes, connections, and
application p95 latency; no log/trace payload bytes inserted into Postgres.
- S3 PUT/GET/HEAD/LIST counts, index overhead, object count, stored bytes,
compression CPU, transfer volume, and upload/read p95 latency.
- Pending bytes, upload lag, read recovery time, and behavior at queue limits.
- A daily/monthly cost projection using the **actual configured provider's**
rates and observed workload/duty cycle; do not substitute AWS prices for an
S3-compatible vendor or assume 24-hour production.
- Zero added recurring SQL/network work after buffers and obligations settle.
Record a concrete operating envelope, request-rate/backlog limits, and latency
budget from this evidence before enabling the feature. If the batched metadata
load is material, evaluate an object-store manifest head with conditional writes
and lifecycle-only SQL. Choose one publication scheme for the release; do not
ship two unqualified distributed state machines. Do not return to DB payload
buffering as an automatic workaround for S3 latency or failure.
## Design
### 1. External backend and batched publication
Keep local mode and add a versioned S3-backed store. Inject the existing object
provider plus narrow metadata/ownership operations. Readers dispatch from the
persisted backend/version, including historical records. Remove hardcoded
`local_file` checks in heartbeat, issues, feedback, and customer-success
inspection together. Preserve current `RUN_LOG_S3_*` mirror configuration for
legacy mode, without redirecting unrelated product file storage.
Reuse the existing instance storage configuration without additional switches.
Cloud already provisions a bucket and bucket-scoped credentials per stack.
Preserve any configured storage prefix and place journals in `run-logs/` and
`provider-traces/`, with company/run namespaces underneath. New journals must not
use a separately configured legacy mirror bucket. Configuration
errors fail explicitly; never select local-only persistence silently. Add only
small company/run-bound stream metadata, with unique identities and constraints.
There are **no proposed run-log payload-chunk or raw-trace payload tables**.
Selected publication protocol after the local operational experiment:
1. Redact/compact log output through the existing owner and enqueue exact NDJSON
bytes with stable source/attempt identities in a bounded buffer. A buffered
enqueue and a durable receipt are different results.
2. Seal on byte threshold, the first dirty buffer's actual deadline, explicit
flush, completion, or graceful handoff. Allocate sequence/byte offsets within
the current writer generation; do not write SQL per record. Seal channels
separately when their trace ordering requires it.
3. Upload a new immutable segment with digest, length, source range, and logical
byte range. Then upload the bounded immutable index/manifest update. Keep
manifest pages bounded; do not rewrite a growing full-history JSON document
every few seconds. Retain exact logical bytes even when compressing storage.
4. Publish one bounded head with S3 ETag `If-Match` (or `If-None-Match` for
creation). The head carries owner generation, committed cursor, and at most
53 immutable index roots. Check the admitted database execution authority
before claiming a new generation. No per-batch SQL or new stream table is
needed. Conditional head publication fences stale generations and tombstones.
5. Release durable receipts for the published contiguous range. Reconcile a
lost object response from the exact head before retrying. Same-source replay
must not duplicate bytes; conflicting replay fails explicitly.
Upload/index retries reuse immutable identities and verify ambiguous results.
Do not hold a DB transaction or connection during network upload. New immutable
index nodes combine completed binary subtrees; all normally published nodes
remain referenced, avoiding an obsolete index object for every flush. Metadata must
not grow with the number or size of individual text records: segment descriptors
and source indexes live in S3. Keep lifecycle/audit metadata bounded.
Use a company/agent/run prefix with a generation-fenced head. A stale owner may finish an upload,
but cannot publish it after takeover. A successor adopts the last published
manifest and replays its uncommitted tail. Per-stream prefix tracking plus scoped,
paginated enumeration must recover unpublished-object cleanup after a crash;
never expose bucket-wide listing through public APIs. Do not garbage-collect a
pending publication merely because one DB query did not find its result.
AWS supports conditional object creation and ETag-based updates, but the current
storage interface does not expose them and compatible vendors need verification.
If eliminating per-segment SQL becomes necessary, qualify conditional writes,
ownership fencing, and manifest recovery explicitly before selecting that design.
[Conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html).
### 2. Live viewing, durability, and the pending tail
Live output need not wait for an S3 PUT to appear in the browser. Preserve its
stable source identity so reconnect/reload can reconcile it with durable history.
An open viewer can see the pending tail; a different server can read the published
S3 prefix. Do not promise zero-lag cross-server tail reads until a replay/relay
path proves them. The flush threshold bounds normal durable-read lag, not a
promise that storage is always available.
The hard problem is **who retains output until publication**, not which database
column it uses. A timer uploading from disposable web memory alone cannot
preserve the pending tail after a crash. Distinguish these boundaries:
- Store durable receipt: only after the segment/index and metadata publication.
- Runner control-protocol ACK: currently may allow streaming output to leave
replay retention before log projection. Inspect and qualify this exact path.
- Live browser delivery: presentation, not permission to discard the source.
Selected after inspecting the real ACK path: reuse the native control port's
already-durable canonical event rows. The S3 head advances their export cursor
atomically with the log bytes. Takeover reads later native rows in bounded
16-row pages, verifies source identity and hash, then uses the same formatter,
redaction and compaction as live recording. The ordinary contiguous stream adds
no SQL. Startup, missing callback ranges, and interrupted settled archives use
bounded recovery reads; settled repair waits for the old native lease to expire.
This avoids a second payload copy, new per-record DB writes, and a new Runner
retention protocol. Other stdout and raw trace bytes are not inferred from this
source. A retained native control row remains readable after Runner cleanup.
The existing Runner outbox retains unacknowledged records. No second spool,
broker, always-on logging service, or full transcript checkpoint is added.
An alternative is holding the existing transport ACK until a batch is published,
but use it only if a real protocol experiment proves bounded pipelining and
prompt Stop/control handling. Serial awaited callbacks plus a five-second timer
can become a five-second wait per record; that is unacceptable. Do not rebuild
full transcript checkpoints in Postgres to sidestep this issue.
For server-only diagnostics with no surviving source, successful durable receipts
still require S3 publication. Local enqueue alone is never advertised as durable.
Missing uncommitted optional diagnostics are explicitly incomplete. Preserve the
original strict guarantee for every acknowledged durable record; any proposal
to accept a bounded crash-loss window for ordinary output is a separate product
decision, not an automatic relaxation in this revision. Host-local agent-process
survival remains outside the web-server replacement guarantee.
On S3 failure, retain bounded pending work at the source, back off, and surface
capture/persistence state without falsely completing the log. At capacity,
backpressure or stop new admission according to the existing execution owner;
do not allocate unbounded memory/disk or switch payloads into Postgres. Optional
trace overflow can become incomplete/truncated. Preserve essential control,
Stop, task results, and usage settlement. Runner cleanup cannot destroy a pending
required-output obligation; either finish export or leave a durable cleanup hold.
### 3. Reading and storage amplification
Read the published pointer, then bounded manifest/index pages and the segments
covering the requested byte range. Cache immutable contents by digest. Preserve
byte offsets, NDJSON/UTF-8 alignment, EOF, cancellation, and bounded memory.
Compressed segments need logical offset metadata and bounded decompression;
HTTP ranges over compressed bytes are not logical log offsets. Preserve exact
full-stream digest semantics when finalizing.
The first release can serve a finalized log by streaming its segments. Do not
require a full re-upload or database compaction pass at every completion. Add
optional segment coalescing only if measured read/request savings justify its
extra I/O, publication/cleanup complexity, and temporary storage. Retention and
read-path limits must account for small-object growth and indexes. Avoid one
object per token/frame and repeated whole-log mirroring.
S3 publication is not atomic with Postgres; object upload precedes pointer
publication. An orphan object is recoverable cleanup, not proof of an accepted
append. A missing/corrupt published object is a storage integrity error, not an
empty log. Verify the actual provider's PUT/GET/range/checksum behavior rather
than assuming AWS guarantees for every compatible backend.
[S3 consistency model](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#ConsistencyModel).
### 4. Provider trace boundaries
Use the same batched object-payload mechanism with trace-specific ownership.
Keep capture opt-in, existing secret/identity suppression, admin access,
company/task-read checks, no-store responses, reveal/download audit, and trusted
workspace-diff reprojection. Raw bytes never enter ordinary events, live logs,
activity payloads, or general attachments. Provider parsing remains in drivers.
Preserve native and rehydration channels, frame IDs, source/debug sequences,
interpretation links, and raw digests. Capture transport must retain pending
records outside disposable web storage until a storage receipt. Reconstruct
frame lookup on takeover. A complete trace requires all required channel ranges
and terminal records durably published and the integrity assessment passing.
Capture failure yields incomplete/truncated without restarting the provider or
blocking task/accounting settlement. Qualify all currently supported producer
families; do not silently advertise a legacy path-only producer as durable.
Encrypt raw trace objects with the existing stable-key facilities and bind them
to company/trace/channel/segment identity. Keep only metadata and the appropriate
key reference in Postgres. Preserve the existing 24-hour expiry from original
capture and 64 MiB capture limit, with separate bounds for encoded/index overhead.
Restart/import never extends expiry. Historical imports preserve exact bytes,
including native-then-rehydration download ordering and digest.
Expiry/deletion first creates a durable tombstone and fences publication, then
cleans segments, index pages, abandoned uploads, and caches. Retain unresolved
cleanup obligations through restart and concurrent uploads; no resurrection.
Preserve `{ok:true}` only after required payload cleanup. Verify object-version
behavior for the actual bucket, and document historical backup retention
separately. Restore must honor expiry/deletion receipts before exposing traces.
Audit contains content-free metadata only.
### 5. Idle behavior and recovery
A nonempty buffer schedules one actual flush deadline; successful settlement
cancels it. Use the existing coordinator/scheduler for server-owned work and the
Runner lifecycle for producer-owned work. Retries exist only for unresolved
uploads/publications/deletes. Coalesce work and bound global as well as per-run
upload concurrency; prevent many runs flushing at once from starving HTTP or
connection pools. Do not create per-agent storage clients or idle keepalives.
Start/resume/takeover reconciles active stream metadata, source retention, and
unfinished cleanup. Existing DB work signals are process-local; do not rely on
them to wake another server. Until the parent's cross-instance wake mechanism
exists, retain explicit owner/recovery/sleep obligations. Completed history alone
creates no recurring scan. Trace expiry uses its real deadline and the existing
supported wake/blocker contract.
## Delivery slices
| Slice | Useful result and owned surfaces | Evidence/dependency |
| --- | --- | --- |
| 1. Operational and durability prototype | Workload measurement, S3 segments, publication/cost model, actual Runner pending-tail/ACK experiment | Select batching and one publication scheme; demonstrate acceptable cost and control responsiveness before schema commitment |
| 2. S3-backed run logs | Store/readers, bounded buffering/indexes, small metadata schema, replay/ownership, cleanup and local compatibility | Slice 1; two-process replacement, ambiguous PUT/commit, stale-writer and backlog tests |
| 3. S3-backed provider traces | Producer sinks/rehydration, trace store/routes, expiry/deletion | Slice 2 primitives; actual capture integrity, access and cleanup races |
| 4. History migration and qualification | Resumable importer, compatibility/authority switch, documentation and staging evidence | All retained inventory accounted for; empty-disk reads and supported rollback after new writes |
These are reviewable boundaries, not a fixed PR count. Keep each usable with the
existing instance storage profile. Raw trace capture remains opt-in. Keep storage
logic in its owners, not large heartbeat/executor additions.
Generate necessary additive metadata migrations with `pnpm db:generate` and
follow snapshot-retention rules. There is no DB payload migration or compactor.
Update `doc/run-log-events.md`, `doc/DATABASE.md`, `doc/DEVELOPING.md`, and
`doc/testing-native-remote-continuity.md` with actual guarantees and operating
limits. Update affected run/inspection feature/access map entries and run
`node scripts/check-maps.mjs`. Change shared types/UI only when their contract
changes; keep incidental storage mechanics out of normal product controls.
## Migration and rollback
1. Ship a compatibility release that reads both stores, understands new formats,
and enforces the persisted minimum reader/writer capability on external runs.
Keep the volume and legacy writing enabled initially. Record this release as
the rollback floor before switching any authority; an environment flag alone
cannot stop an old binary from mishandling new rows. The deployment/rollback
preflight must reject binaries older than that floor: a pre-feature binary
cannot be expected to understand a newly introduced DB guard.
2. Inventory every retained run log and unexpired trace by company/run. Classify
local complete files, remote mirrors, active files, missing payloads, conflicting
copies, and expired/deleted traces. A mirror may be a partial prefix: verify
against authoritative bytes/metadata, never assume it is complete.
3. Backfill settled resources with bounded indexed keyset batches and resumable
receipts. Preserve original bytes, offsets, digests, attempt information,
source attribution, and trace expiry. Expired/deleted traces stay unavailable.
4. Under the parent's permitted one-time maintenance window, stop admission and
settle/fence remaining legacy writers. Copy final active tails and both trace
channels. Recheck hashes and byte totals after fencing; no copy-while-writing
race can establish completion.
5. Publish each resource's backend/reference transactionally. Record a migration
manifest of migrated, already-remote, expired/deleted, and unresolved resources.
Missing/conflicting retained bytes block the preservation gate. Report genuine
pre-existing loss; neither fabricate history nor quietly waive Dotta's choice.
6. Start B with the log/trace directories absent and old storage inaccessible.
Read historical data and append new data through normal APIs. Confirm every
in-scope inventory item has a valid disposition. Keep the old volume retained
under the parent's recovery policy; this slice does not detach/delete it.
7. Roll back to the compatibility release against the same external data and
prove it reads writes made after cutover. Do not switch authoritative reads
back to stale local files. Defer destructive schema cleanup until rollback
readers have retired.
## Acceptance matrix
| Case | Observable finish condition | Evidence |
| --- | --- | --- |
| Operational envelope | No new payload/per-token SQL; batch metadata and object traffic fit measured budgets | Real workload baseline, WAL/transactions, request/storage/transfer model and app/control latency |
| Active/completed run | B retrieves every durable receipt exactly once after A and its disk disappear | Separate processes/directories; normal authorized reader; byte/digest comparison |
| Pending tail | Web replacement retains/replays required output without duplicating provider work | Actual Runner spool/ACK experiment; kill between generation, upload and publication |
| Existing history | All retained inventory preserved under original retention | Full checksums, resumable manifest; missing/conflicting data blocks cutover |
| Ambiguous PUT/commit | Retry reconciles publication and never duplicates bytes | Fault after object PUT or DB commit but before response |
| Ownership | Stale A cannot publish/finalize after B takes over | Concurrent server and manifest-generation tests |
| Reads | Stable offsets/digests across segments, compression, caches and replacement | UTF-8/boundary/EOF/cancellation; bounded memory and GET counts |
| Storage failure | No unbounded queues or SQL-payload fallback; Stop/API remain usable | Slow/failed S3, queue capacity, recovery and source cleanup hold |
| Traces | Accepted frames survive; completeness and privacy remain truthful | All producer families, channel hashes, disabled capture and denied actors |
| Downstream consumers | Issue previews, feedback, inspection and reprojection keep working | Normal routes and current authorization checks |
| Expiry/delete | Deny after tombstone; clean payloads; no late-upload resurrection | Capture/upload/delete races, restart, actual bucket version behavior |
| Rollback/local mode | Compatible release reads post-cutover writes; local defaults unchanged | Adjacent-version and local-store tests |
| Idle | No recurring SQL/network work after settlement; real work wakes owners | Configured idle, completed history, retry/drain/resume observations |
## First useful experiment and qualification order
Use real Postgres, the existing S3 provider implementation against a disposable
S3-compatible target, and two actual service processes with separate temporary
directories. Produce representative output through the production recorder,
including repeated chunks, multibyte text, and tool bursts. Measure the candidate
batch settings and baseline API traffic. Test both published data and the pending
tail: kill A at each boundary and read/replay through B. Never pre-populate the
storage result being tested. Record revision, configuration, receipts, timing,
SQL/WAL and object-operation evidence.
Use the existing remote-continuity harness with a deterministic provider and
actual Runner to qualify retention independently of control ACKs and responsive
Stop. The decisive question is whether durable batching can coexist with ongoing
provider execution and web replacement. Do not commit to a protocol mechanism
based on a mocked callback or a successful single PUT.
Before staging, record the chosen publication scheme, batching/concurrency
limits, backlog capacity, supported recovery behavior, and measured cost/latency
operating envelope. If those fail, revise the design before a broad schema or
migration implementation. No production sizing/cost has yet been measured.
Then verify the actual staging bucket and remote capture path with a bounded
live provider run, normal UI/inspection/download, and post-replacement reads.
Make the old web filesystem inaccessible. Preserve prior failures alongside
successful retries. Full Railway HTTP handoff and volume removal remain parent
plan gates.
Run narrow storage/output/trace/continuity/auth/migration checks while iterating.
At implementation PR handoff run `pnpm -r typecheck`, `pnpm test:run`, and
`pnpm build`, plus affected Runner checks and user-journey verification. Do not
claim live-provider or fleet evidence from deterministic fixtures.
## Implementation state and remaining gates
- Configuration follow-up: shared storage resolution now serves product files,
run logs and traces. S3 instances need no extra switches or connection fields.
Cloud's existing per-stack bucket and credentials remain the isolation
boundary; existing prefixes and company/run subfolders are preserved. The
configuration regression failed before the change, then 70 affected tests
passed, including real S3-compatible bucket/prefix isolation and cleanup tests.
Four existing Cloud environment-contract tests also passed without Cloud code
or deployment changes.
- User follow-up: remove the added run/adapters map sections. Retained-history
conversion now belongs to an automatic, bounded delivery worker; the command
is optional diagnosis/repair. Existing local references remain durable work.
Startup reconstructs it, active or blocked history retries with backoff, and
completed history has no recurring SQL/network scan. Migration stays with its
owning service; index.ts only registers its lifecycle, and the shared delivery
registry adds the queue name. The source volume must survive until all retained
history is verified. Mixed versions must stop creating new local-file runs.
- Trace encryption reuses either existing vault key source, including the
configured key file, and never creates a replacement key for external traces.
Both strict missing-key and file-key round-trip regressions failed before the fix.
The configuration/storage regression run passed 91 tests. Automatic migration
and retention passed 10 tests, including a real Postgres/S3 upgrade followed by
source-directory deletion. Harness failures remain in the local evidence: a
cold import exceeded a test deadline (moved module setup into the fixture), a
sandbox denied localhost access (rerun with network permission), and the first
automatic fixture bucket name exceeded S3's length limit (shortened the name).
- Shutdown review: automatic trace import now propagates cancellation through
source reads, opening, conditional-write qualification, upload and verification.
Five real-SDK stalled-request regressions failed before the fix, then passed.
A shared capability probe survives one cancelled claimant and aborts when its
last claimant leaves. Source and remote trace hashing stream bounded chunks.
A multi-chunk native sidecar without a final newline retains exact download
ordering and digest. Existing file-key and empty-web-home app checks passed at
a142424fc9; subsequent changes affect copy cancellation and verification only.
- Base integration: 879ba0d1fd reached Greptile 5/5 and all CI checks passed,
then master advanced and produced two append-only documentation conflicts.
Preserve both documentation sections and integrate the new Cloud snapshot
startup contract. The startup regression reproduced a redundant config read;
worker registration now accepts only the already-loaded storage-provider value.
All 103 startup controls plus storage/lifecycle controls passed (112 tests),
including saved Cloud profile reuse after the legacy config file is absent.
A further combined run passed all 124 storage and real-Postgres Cloud runtime
configuration controls on the merged source.
- Further review: trace retention now registers independently of heartbeat
scheduling, and optional issue-comment attribution reads the published log
prefix after a failed best-effort native repair. Both defects reproduced red;
all 116 selected startup, retention, attribution and cancellation controls pass.
The 19f702d780 browser session-recovery check timed out; its job log is retained.
No UI behavior or browser assertion was changed to suppress that failure.
- Branch: `codex/external-run-logs-traces`; this chat owns integration and PR review.
- S3 holds payloads from the first committed batch. Conditional object heads
hold cursors and writer generations. Existing Postgres rows hold lifecycle and
references only. No schema change, per-record SQL, or DB payload compactor.
- Implemented: stored-backend reader dispatch, bounded batching and I/O,
immutable binary range indexes, exact digest/range reads, ownership fencing,
encrypted optional trace channels, durable deletion/expiry obligations, and a
bounded resumable history importer. Local defaults remain unchanged.
- Exercised locally with real Postgres and Moto 5.2.5 through the real AWS SDK:
encrypted trace replacement/deletion and history import with original source
directories removed. Missing, corrupt, and active history blocks migration.
- Abrupt process qualification: a writer commits 1,000 records, receives storage
receipts, dies by `SIGKILL`, and loses its working directory. A separate
process reads every record exactly and continues the same journal.
- App journey: create a deterministic process agent, click Run now, open its
run history, kill the web process, restart against the same external services,
and reload the viewer. All 31 NDJSON records remained byte-identical through
the normal authorized API (6,206 bytes; SHA-256
`3e8415c79379c583b1c4c37902187593fd69ebe85db78d6486d6077f205bacba`).
- Local workload qualification reports burst and eight-stream steady request,
transfer, stored-object, and timing counts. It is a synthetic envelope,
not a fleet cost forecast. The first index prototype left obsolete COW
objects; the selected immutable binary index removes that normal growth.
- Full workspace typecheck and build passed after native replay and the storage
review fixes. The pushed server revision was rebuilt for the final app check.
The native Postgres package initially lacked its postinstall-created library
symlinks; its standard hydration helper repaired that local test prerequisite.
- Native pending-tail qualification passed with the real Rust recovery fixture,
`PaperclipControlPlanePort`, production recorder, real Postgres and Moto. A
received ACK released 32 records from the Runner while S3's source cursor was
still zero. After `SIGKILL` and deletion of the old controller directory, B
exported all 64 records once while the same Runner PID continued. Snapshot and
Stop commands completed in 453 ms and 122 ms. The first attempt failed because
its new controller directory had mode 0755; the fixture now creates mode 0700.
Both failed and successful evidence remain in this chat's temporary artifacts.
This is a deterministic protocol experiment, not a live model or Railway test.
- Final app verification used a new web home on `c2cedcd116`. Retained history
remained byte-identical (31 records, 6,206 bytes), and a new UI-triggered process
run succeeded with every generated record present in its S3-backed transcript.
The browser initially reached the listener before startup completed; after
readiness, the normal page and API checks passed.
- Durable log receipts and live log messages follow S3 publication. The native
export reuses existing control rows, adds no payload writes, and repairs missing
callback ranges. Interrupted settled archives can be repaired on normal reads
after the old native lease expires. Optional raw capture and server-only
diagnostics without a surviving source can remain incomplete after a crash.
- Review fixes cover absent-head deletion fencing, shared capture ownership,
abort/drain on deletion, local expiry deadlines and multi-page cleanup, bounded
transient retries and admission backpressure, partial-startup cleanup, and
explicit trace-import cleanup markers. A source integrity failure leaves the
native archive's final size unknown instead of certifying a partial result.
- Full local Vitest could not run on host embedded Postgres because concurrent
clusters exhausted macOS SysV semaphores. Affected suites are running against
isolated Docker databases. One local full typecheck attempt also lacked sandbox
permission for tsx IPC; the authorized rerun uses the normal workspace command.
The isolated Docker run then passed 1,014 affected storage/native tests and
all 27 privacy-route tests. Subsequent targeted review regressions passed for
native source classification, native import coverage, shared-reader repair
cancellation, serving the published prefix on repair conflicts, and a failed
local-file cleanup retaining its deadline. Live and recovered native output
redact existing agent identity material; runtime log secrets are registered
once through the existing encrypted redaction registry, without copying agent
private keys into run contexts.
- Native history migration accepts finalized files whose recorded size and
digest match, preserving the existing formatter's truncated large display
chunks. Files missing either completeness field still require ordered source
coverage. Corrupt metadata blocks publication; the conditional switch checks
that the original completeness fields have not changed.
- PR: https://github.com/paperclipai/paperclip/pull/18327. Initial CI passed 51
checks with one E2E teardown lock timeout. That E2E shard passed on the next
revision; its Runner shard then failed the Antigravity authentication fixture.
These failures are retained, and fresh-head CI remains required.
- Production remains disabled. Railway Buckets are the parent fleet's current
S3 provider; their conditional-write, version/deletion, workload cost, remote
capture, and sleep/wake behavior need actual-target qualification. No fleet
environment or volume was changed.
- Next gate: finish fresh-source checks and PR review; retain actual-provider,
live remote capture, and parent cutover gates before any volume-free rollout.
- No Paperclip issue/run context was supplied. This repository plan is the state
record; no issue work product or production migration is claimed.
doc/recovery-file-cutover.md
+112
View File
@@ -0,0 +1,112 @@
# Check existing recovery files before web-server replacement
This check covers only observed legacy files on one explicitly selected instance.
There is no verified production backlog. An empty installation needs no migration.
This check does not inspect Postgres obligations, Runner delivery, other volumes,
other instances, or the rest of web-server replacement readiness.
Use the retained volume and the instance root from the old server configuration.
The instance root is the directory that contains `accounting-receipts/` and
`quota-credential-recovery/`. It is not the parent directory of all instances.
Use canonical absolute paths. The check rejects symlink components, including
system aliases such as `/var` on macOS; use the actual `/private/var` path there.
The chosen instance directory must exist. Missing recovery directories are normal.
Run this command from a checkout that includes the check. Node is the only runtime
dependency. It needs no database connection, master key, or package installation.
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance
```
The default command reports JSON and exits successfully even when work remains.
For an automation gate, add `--check`:
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance --check
```
| Exit code | Meaning |
| --- | --- |
| `0` | Report produced; with `--check`, all three scans are clear |
| `1` | With `--check`, unresolved files or an unknown/incomplete scan block cutover |
| `2` | Invalid arguments or the inventory could not run |
The sandbox directory defaults to `data/sandbox-orphan-cleanup/` below the chosen
instance. If the old server uses `SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR`, supply that same
environment value when running the check. An explicit `--orphan-directory` takes
precedence. A relative or empty override is rejected; supply its canonical absolute
path for this check. An override is an additional explicitly selected root and can
be outside the instance directory. The check does not search for other overrides.
```sh
node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance \
--orphan-directory /absolute/retained/orphans --check
```
The report gives one summary for each of the three recovery kinds. It never
prints filenames, input paths, file contents, identifiers, token values, decrypted
material, or raw filesystem errors. `unverifiedFiles` includes every regular file,
including JSON, temporary, probe, truncated, corrupt, and unreadable files. The
check does not read file bytes and cannot claim that a file is valid or settled.
That decision belongs to the existing recovery owner and its format definitions.
Only the credential directory's immediate UUID company directories are traversed;
empty company directories are normal after recovery. Accounting and orphan files
use flat directories. Unexpected directories, deeper nesting, symlinks, and
special entries count as `unknownEntries` and block. The check does not follow
symlinks or inspect unexpected subtrees. Directory/stat/close failures increment
`readFailures` or leave the scan incomplete. An invalid or missing instance root
also blocks; it is not an empty installation.
The default shared budget is 10,000 observed entries across all three scans,
including company directories. `--max-entries` accepts 1 through 100,000. At the
limit the scanner reads at most one extra entry per open directory to detect
truncation. An unfinished scan reports `complete: false` and status `unknown`.
Counts are observed lower bounds, not total records or total pending operations.
A complete scan with any files reports `blocked`; any uncertainty reports
`unknown`. Only a complete empty scan reports `clear`.
For the final cutover gate, stop the old producers first and retain the old volume.
Run the check after the relevant recovery owners finish their work and producers
are stopped again. A live scan is a point-in-time inventory, not an atomic drain
certificate. Concurrent renames or writes invalidate the snapshot; the tool does
not lock writers or protect against a hostile process that changes paths during
inspection. Do not use it to certify a live writable or untrusted volume.
If files exist, keep the old volume and let the existing owners settle actual
entries with their existing permissions and database checks:
- Accounting: `run-cost-accounting.ts` uses the `usage-receipts.ts` replay and
per-run recovery paths. The receipt owner validates the accounting envelope and
handles database conflicts and settlement.
- Credentials: `ai-connections.ts` and `ai-connection-runtime.ts` use
`quota-credential-recovery.ts` during account credential access and refresh.
The owner validates encrypted material with the original instance master key
and preserves grant, company and credential version fences.
- Orphan cleanup: `environment-runtime.ts` exposes `flushDeferredOrphanCleanups`
to the existing cleanup lifecycle. `sandbox-orphan-cleanup-spool.ts` owns the
record format and removes a file only after the recovery path saves ownership.
These are service entry points, not standalone operator commands. This change
adds no import, deletion, replay, decryption, or bulk migration command. Do not
trigger new paid runs or token exchanges merely to empty the report. If an owner
cannot settle a record, preserve it and record an explicit reconciliation outcome
through the existing incident/operations process. Corrupt files need owner review;
they do not disappear from the gate because a recovery reader skipped them.
Never delete unresolved files to obtain a clear result. An approved exception
requires separate evidence; this tool continues to block while the files remain.
Retain the volume and required secrets through rollback compatibility review.
Verify the check without any real instance data:
```sh
node --test scripts/recovery-cutover.test.mjs
```
The tests use disposable synthetic directories and spawn the actual CLI. They
cover pending legacy layouts, malformed bytes, temporary files, empty directories,
symlinks, overrides, scan limits, safe diagnostics, failure injection, exit codes,
and preservation of source bytes. They establish file inventory behavior only.
They do not establish that any production volume is clear or that domain recovery
has completed.
doc/run-evidence-storage.md
+276
View File
@@ -0,0 +1,276 @@
# Run log and provider trace object storage
Instances configured for S3-compatible file storage automatically use the
`s3_segments` backend for new run logs and opted-in provider traces. Postgres
retains the existing run/trace references and lifecycle rows.
It adds no payload table, per-record SQL, or database-to-object compactor.
Structured `heartbeat_run_events` retain their existing database path. Native
events already committed there before a Runner ACK also provide the recovery
source for an interrupted canonical native log export. No additional event copy
or payload write is introduced.
When native object logging uses runtime observability credentials, their log
redaction values are registered once in the existing encrypted run-secret
registry. This is bounded redaction metadata, not a transcript copy or a
per-event write. Agent identity private material stays in its existing key store.
## Configuration
No additional storage configuration is required. The existing instance
`storage.provider` setting, or `PAPERCLIP_STORAGE_PROVIDER`, selects storage.
The same config-file S3 fields and `PAPERCLIP_STORAGE_S3_*` overrides select the
bucket, endpoint, region, prefix, and path style used by product files. Credentials
use the same AWS SDK chain. Local-disk instances keep `local_file` logs and traces.
The instance's existing S3 prefix is preserved. The folders underneath it are:
```text
<existing-prefix>/run-logs/segments-v1/<company>/<agent>/<run>/
<existing-prefix>/provider-traces/segments-v1/<company>/<run>/<trace>/<channel>/
<existing-prefix>/<company>/... existing product files
```
Cloud's existing provisioning supplies a separate bucket and bucket-scoped
credentials for each stack, plus the stack's encryption key. The app reuses that
boundary rather than selecting a global customer bucket. Within a stack, journal
keys remain company/run scoped. Cleanup lists only the resource's own prefix;
it cannot sweep other companies, other stacks' prefixes, or product files.
Folders are organization, not a substitute for bucket permissions or API access
checks. Shared-bucket self-hosted deployments must retain their existing distinct
instance prefixes and appropriate storage credentials.
Legacy run rows retain their recorded backend until verified migration. Their
pre-existing `RUN_LOG_S3_*` mirror settings and key paths remain supported for
legacy reads and writes. They do not override instance storage for new journals.
Use stable bucket/prefix settings across replacements and releases.
As with existing product files, changing the instance bucket or prefix is a
separate storage move and requires copying the complete namespace and preserving
its keys before changing configuration. Switching to local storage does not move
existing S3 evidence. Readers reject an unavailable external store; they never
substitute an empty local log. Restore the original stack's storage profile to
read its history. This upgrade migrates legacy local evidence into the existing
stack store; it does not introduce bucket relocation or reverse migration.
Raw capture still requires its existing explicit capture request. Its encryption
uses the existing vault key, from `PAPERCLIP_SECRETS_MASTER_KEY` or the existing
configured key file. Cloud already supplies the environment key. No separate
trace key is introduced, and a missing vault key fails rather than creating a
replacement. Disposable deployments must preserve the existing vault key along
with their other secrets.
Raw payload and index objects
use AES-GCM with a key derived for the particular company/run/trace/channel.
The bounded head holds only ownership, range, digest, and publication metadata.
Existing capture opt-in, redaction, instance-admin access, task/company access,
reveal/download auditing, and the 24-hour capture expiry remain in effect.
The provider must enforce `If-None-Match: *` and ETag `If-Match`, return ETags,
and support prefix-scoped paginated listing. Claiming the first journal performs
a conditional-write capability probe. A backend which ignores preconditions is
rejected. Before deploying this release to an S3-configured production instance,
qualify the provider's conditional writes, range/read consistency, encryption,
object versions, and cleanup. AWS API compatibility alone is insufficient evidence.
## Publication and operating limits
Each batch uploads immutable payload segments and merges equal-height index
roots into immutable binary nodes, then conditionally publishes one head object.
The head carries at most 53 ordered roots. Every normally published payload and
index node remains reachable; appends do not leave superseded index copies.
Interrupted or fenced uploads can leave unpublished objects, which remain inside
the resource prefix for scoped deletion. The head is the commit
boundary. A stale writer can upload an orphan but cannot replace the current
owner's published history. An uncertain response is reconciled against the exact
head before returning a durability receipt. Committed reads use immutable roots,
so publication cannot change the bytes behind an already selected snapshot.
An individual store `append` resolves only after publication. Heartbeat's
`onLog` diagnostic callback waits for bounded admission without serially
waiting for each network batch. At capacity it waits for space; admission beyond
the bounded waiting allowance is rejected without poisoning accepted batches.
Transient network, 409, 429 and server errors receive bounded retries. A 412
or a changed head fences the stale writer; a 409 alone does not prove ownership
loss or conditional-write support. Live log messages and progress are published
after storage confirmation. Finalization drains admitted writes before closing
and hashing the stream. Logging failure reports incomplete persistence and
leaves the final size/digest unknown rather than claiming a complete snapshot.
The original progress projection remains coarsely throttled; concurrent batch
completions share a single progress write.
Defaults and fixed safeguards:
| Limit | Value |
| --- | --- |
| Dirty batch deadline | 1 second, configurable from 1 ms to 30 seconds |
| Segment target | 1 MiB |
| Pending bytes per writer | 8 MiB |
| Pending run-log bytes per process | 64 MiB |
| Additional waiting admission | 8 MiB / 32 calls per process |
| Concurrent run-log flushes | 4 |
| Concurrent journal object operations | 8, with 32 queued operations |
| Object operation/batch cancellation deadline | 15 seconds |
| Current index fanout / maximum roots | 2 children / 53 roots |
| Journal head read bound | 64 KiB |
| Trace encoded payload bound per sidecar | 192 MiB, in addition to the existing raw capture limit |
A configured but empty log store sends no periodic request. Dirty buffers,
capture file-change events, and actual retention/retry obligations drive work.
Use request/transfer counts, byte volume, steady versus bursty producers,
concurrent runs, and control/API latency to size a deployment. The local S3
qualification is a synthetic workload, not a production cost forecast.
Finalization streams the committed bytes once to compute the existing complete
file SHA-256. It does not re-upload the entire log. Range indexes avoid loading
all segment descriptors to read a tail. Historical consumers, UTF-8/NDJSON record
alignment, issue previews, feedback, and customer-success inspection dispatch
through the persisted backend. Logical reads retain the existing authorization.
On-demand native repair closes its verified committed range with an exact byte
count and leaves an unavailable full-file SHA-256 null. It does not make a log
page request re-download the entire archive to compute that optional digest.
## Durability boundary
Committed log receipts and captured object prefixes survive web disk loss.
Output still buffered only by the current web process has no durable receipt.
The live log path waits for publication before showing that output. Raw sidecar
capture remains optional; a crash can leave an incomplete uncommitted tail, and
only durably closed channels plus valid terminal markers qualify as complete.
Native PRP events have a surviving source: the control port already commits
validated, redacted events before acknowledging them. The S3 head checkpoints
the highest exported database event cursor with the batch's bytes. A replacement
replays later native source rows in bounded 16-row pages, checks their identity
and canonical hash, and exports the same `paperclip.prp.event` representation
through normal log redaction/compaction. Late callbacks and duplicate delivery
cannot append the same source record again. This is a canonical native export;
generic stdout, stderr, and raw provider traces are not reconstructed from event
rows. The control ACK and sparse checkpoint formats remain unchanged.
Recovery runs before native execution reconnects. If the run settled before its
archive finished, the normal log API, issue preview, or feedback reader can repair
it after the native controller lease expires. Concurrent requests share recovery;
final metadata is saved only when no later native source remains. Read-only
customer-success inspection reads the currently published prefix. A failed
repair leaves final size/digest unknown and preserves the existing source.
Retained native events are not deleted by Runner cleanup. Recovery adds bounded
reads and one final metadata update, with no new per-record SQL on the normal
contiguous output path. Raw optional capture can still be incomplete after a crash.
Credential-update, retained-cleanup, and trace-reprojection records have separate
source namespaces and do not enter this log export. Corrupt canonical records
fail integrity validation. Shared on-demand repair has a 30-second lifetime;
cancellation removes a waiting reader and aborts storage work when no readers
remain. It preserves already published cursors so a later request can resume.
Live export and recovery both redact the existing agent identity and known run
secrets before publication. Recovery reads existing identity material without
provisioning a key. If best-effort repair loses ownership or fails, ordinary log
reads serve the published prefix and leave completion metadata unknown; request
cancellation still propagates.
The [remote continuity](testing-native-remote-continuity.md) protocol and the
parent deployment's other filesystem and HTTP handoff requirements still apply.
## Trace deletion, expiry, and sleep
Deletion/expiry denies reads through a database tombstone before payload cleanup.
Object journal tombstones prevent further publication. If an active writer may
have an in-flight upload, cleanup waits at least 30 seconds and remains a durable
retry obligation. A successful manual DELETE requires payload cleanup; an
unsettled cleanup returns a retryable failure, and the shared retention worker
retries it. Tombstones remain after object removal. Historical backups and
object-version retention need a separate operator policy; an ordinary DELETE
must not be mistaken for erasure of historical versions.
The shared coordinator restores expiry/cleanup work on startup and registers
actual deadlines for local and external captures. New local inserts also wake it;
expired backlogs drain in bounded pages. No completed-history polling is added. Trace expiry
is not yet part of Cloud's durable wake contract, so outstanding trace deadlines
conservatively block instance sleep. Removing that blocker requires qualification
of the corresponding external wake, not a longer polling interval.
Trace expiry and failed-deletion recovery remain registered when
`HEARTBEAT_SCHEDULER_ENABLED=false`. Disabling agent scheduling does not disable
retained-data cleanup. The worker still respects warm standby, admission drain,
actual deadlines, and shutdown.
## History migration and rollback
S3-configured servers automatically migrate retained local history in the background.
No command or extra configuration is required. The existing delivery coordinator
starts the upgrade outside request handling, independently of the heartbeat
scheduler toggle. Each batch processes at most one log and one trace, then yields
for one second. Existing references are the durable work inventory. Restarting
reconstructs unfinished work from those references; already migrated rows are
excluded. No new payload or job table is added.
Active writers, missing files, corrupt data, and storage failures leave history
unresolved. Other pages continue. The server logs each blocked resource and a
pass summary. Unresolved passes retry after five minutes, with backoff up to
30 minutes. Transport failures retry through the shared coordinator. After all
retained history is migrated, the worker stops querying and contacting storage.
Trace source hashing and remote verification stream bounded chunks instead of
loading a complete capture into server memory. The worker cancellation signal
reaches file reads, trace opening, conditional probing, uploads and verification.
Local-disk instances do not register this worker. Outstanding migration conservatively
blocks instance sleep, and shutdown aborts the batch and waits for owned work.
Deploy compatible readers and stop old binaries from starting new local runs.
The upgrade waits for existing writers to settle and their leases to expire.
Keep the source volume attached until migration completes and clean-filesystem
reads pass. Automatic copying cannot recover a file after its only copy is lost.
`server/scripts/migrate-run-evidence.ts` remains an optional diagnostic and repair
tool. Its default is a read-only inventory; `--apply` runs the same verified
importer used by the automatic worker. `--company`, `--after-run`, `--after-trace`
and `--limit` allow scoped investigation against an explicitly selected
`DATABASE_URL`. Normal upgrades do not require this tool. It does not stop agents,
select a production target, delete source files, or certify volume removal.
For logs, migration verifies exact local bytes or a mirror with proven length
and digest, compares any previously uploaded prefix, verifies remote SHA-256,
then conditionally changes the run's backend/reference. Unknown-completeness
mirrors, missing/corrupt payloads, and active writers block publication. Existing
trace bytes, download ordering, integrity state and original expiry are retained.
Neither path fabricates missing historical data or extends trace retention.
For finalized native logs, matching the recorded size and SHA-256 proves the
complete legacy file, including display chunks truncated by the existing 64 KiB
formatter. Migration preserves those exact bytes and records the settled source
cursor. If either completeness field is absent, a bounded NDJSON scan must prove
ordered coverage of the existing canonical source records. Missing ranges or
chunks whose identities cannot be verified block publication as
`native_log_source_coverage_unproven`. A database's latest event cursor alone
does not establish completeness. Publication checks that the original reference,
completeness metadata, and source cursor have not changed. Keep the original
volume and resolve blocked inventory before cutover.
An interrupted trace import keeps an `s3-import-v1:` reference: reads still use
the original local sidecars, while deletion and expiry retain responsibility for
the staged object prefix. The automatic worker resumes the verified switch after restart or retry.
Ordinary local traces have no S3 cleanup dependency.
Keep the volume until every retained item has been verified from a clean web
filesystem. Rollback must use a compatible reader against the same remote state.
An older binary or stale local volume is not a valid rollback target after new
external writes. Enforce that floor in the owning deployment procedure.
## Qualification
Run the storage, recorder, migration, privacy and retention tests. The opt-in
S3/Postgres integration fixtures use `PAPERCLIP_LOG_TEST_S3_ENDPOINT` and a dedicated
`PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL` ending in `_test`. They create
unique test buckets and schemas, and remove only those resources. The privacy
route suite also accepts `PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL` for a
dedicated, fully migrated database ending in `_test`; it clears its fixtures.
`server/scripts/qualify-object-run-logs.ts` exercises the actual AWS SDK against
a loopback test backend, confirms exact replacement reads and stale-writer
rejection, and reports requests/bytes/timing. It uses synthetic credentials and
refuses a non-loopback target. This proves its tested transport and workload;
production-provider qualification and actual Railway deployment remain separate.
`server/scripts/qualify-native-log-recovery.ts` uses the real Rust
`recovery-fixture` example, real Postgres, the control port, production recorder,
and S3 backend. It kills the controller after Runner ACKs but before the S3
cursor catches up, deletes that controller's directory, and restores sparse
control state into a fresh controller. It checks exact source order, no duplicate
records, the same surviving Runner PID, and control/Stop responsiveness. Build
the example with `cargo build --locked --manifest-path packages/paperclip-runner/runner/Cargo.toml -p paperclip-runner-core --example recovery-fixture`.
It accepts the same two explicit loopback test endpoints and creates a private
schema and bucket. This deterministic protocol experiment does not qualify a
live provider, remote execution environment, or production S3 vendor.
doc/run-log-events.md
+10
View File
@@ -694,3 +694,13 @@ telemetry-read authorization. Ordinary PRP reads/live delivery retain run-secret
and current-user redaction. The task page uses one history read on mount/reconnect
and the existing live transcript stream, then makes a bounded plugin readiness
check. No panel discovery timer scans runs or provider state while connected.
## Payload storage
Structured events in this guide remain in `heartbeat_run_events`. When the
instance's existing file storage uses S3, new stdout/stderr NDJSON logs use the
same stack bucket and configured prefix under `run-logs/`, with existing run
references in Postgres. Local-disk instances remain local. Raw provider
traces have their own restricted, encrypted object path. See
[run evidence storage](run-evidence-storage.md) for batching, durability,
configuration, migration, retention, and qualification boundaries.
doc/testing-native-remote-continuity.md
+127 -4
View File
@@ -172,10 +172,15 @@ disk outbox. No second remote journal or new polling loop is introduced.
The live event window keeps the latest identity, turn, result and usage anchors,
as well as unsettled semantic inputs, through long output streams. The 4,096-event
and 500-command limits remain. This checkpoint is intentionally not a complete
transcript: an abrupt crash can lose display output already acknowledged but not
yet projected to the existing run log. The same provider turn continues; recovery
never reruns work to reconstruct display text. Tool outcomes and terminal state
retain their durable barriers.
transcript. With local-only logs, an abrupt crash can lose a display projection
already acknowledged by the controller. The opt-in `s3_segments` log backend
checkpoints its canonical native export cursor in S3 and replays missing records
from the existing validated `heartbeat_run_events` source on takeover. This adds
no payload writes or per-record checkpoint transactions. Generic process output
and optional raw provider sidecars have separate durability boundaries. See
[run evidence storage](run-evidence-storage.md). The same provider turn continues;
recovery never reruns provider work to reconstruct display text. Tool outcomes
and terminal state retain their durable barriers.
Rollback must use a controller that understands sparse checkpoints and the signed
recovery cursor. A v2 decoder alone is insufficient. Runners admitted before this
@@ -278,3 +283,121 @@ This is protocol and persistence evidence. The separate
`everyday-workflows.runner-codex.daytona.recover-controller` Product E2E cell
qualifies the actual Daytona/provider path, including completion and workspace
finalization. Neither test substitutes for the other's failure boundary.
## Required native usage acceptance
Built-in native remote Runner execution accepts structured accounting before
its PRP transport ACK. `usage.reported`, turn-start, and turn-terminal facts
produce immutable `run_usage_receipts` rows. The receipt stores only structured
usage and turn identity. It does not depend on output logs or raw trace capture.
The source UUID derives from the run and Runner identity. Provider-turn IDs and
turn-start source facts preserve aggregation when several provider turns share
one PRP envelope turn. Source event IDs and
sequences remain stable across receiver replacement. Repeated delivery checks
the saved fact and does not create another receipt for the same report.
A failed receipt transaction rejects acceptance. The remote Runner keeps its
outbox report and reconnects through its existing protocol lifecycle. A lost
commit response can replay the same immutable receipt. A replacement receiver
restores accepted per-turn aggregation from Postgres, then accepts pending
reports from the same producer. The existing source fence prevents a retired
producer from replacing the current source's usage. Missing or partial usage
keeps accounting pending; it does not produce a zero-cost charge.
The native session authority epoch supplies the current run's accounting owner
to reused warm transports. The usual controller checkpoint and generation checks
still control transport acknowledgement and attachment. Ledger settlement,
pricing, reservations, and budget enforcement retain their existing owners.
No accounting receipt directory is created for this supported remote path.
Local native execution, legacy adapters, and external Dot/Muse execution retain
their existing recorder behavior. This change does not establish disposable-web
accounting durability for those paths. Server-originated model calls keep their
existing admission and settlement records. No S3 accounting spool is introduced.
The independent run-log/trace storage change can consume its own persisted
sources; output acceptance is not accounting acceptance.
Run the focused PostgreSQL/Runner experiment with:
```sh
pnpm exec vitest run server/src/services/native-runtime/native-usage-delivery.integration.test.ts --maxWorkers=1
```
The provider fixture emits a real PRP usage report through the durable Rust
Runner. A PostgreSQL trigger fails receipt insertion. The test removes the first
receiver directory, replaces its accounting owner, reconnects the same Runner,
and independently reads the receipt and ledger. The provider is controlled;
no paid provider request is made. Additional cases cover lost commit responses,
replay conflicts, per-turn aggregation, partial usage, source replacement, and
budget holds. Optional log and trace callbacks are absent from the experiment.
The cold fixture build runs before opening PostgreSQL, with the existing native
integration build budget. PR CI routes this suite through the existing cached
native Runner lane. CI builds the release fixture; local execution uses debug.
The behavior test has its own bounded receiver/process deadline after compilation.
### Codex usage authority and release order
The qualified Codex producer keeps its current-run baseline and latest session
totals in its existing durable provider state. It observes historical resume
snapshots, preserves that baseline across process recovery, and resets the
baseline to the latest totals when attaching the next run. Multiple request
reports therefore emit the whole run delta, rather than the provider's last
request. Regressing or incomplete live counters cannot certify completion.
Authoritative Codex reports carry `runDeltaBasis: "run"`. This requires a Runner
artifact built from the provider baseline implementation in this change. A
version string alone does not establish support; use the qualified artifact
digest. Existing unmarked Codex reports remain incomplete and unpriced in the
new accounting path. Their budget debt remains pending for reconciliation.
No deployed Runner artifact has been qualified by this source-only change.
PRP negotiates envelope versions and the admitted Runner digest. It does not
negotiate this usage payload field. The existing event envelope validators leave
usage payloads open, but an old standalone usage-schema consumer rejects this
new field. Update receivers and strict usage-schema consumers first, then
release the matching Runner artifact before claiming this durability guarantee.
Finish or explicitly reconcile older admitted Codex runs during cutover. Do not
claim that an old producer's final-request delta is a whole-run receipt.
Heartbeat completion reads the current immutable accepted receipt, including its
saved exact rate-card amount and provenance. It does not reprice with current
rates or replace that accepted estimate with an absent provider dollar amount.
The terminal UPDATE preserves the current producer-owned accounting projection
atomically. If another report arrived after completion read its checkpoint, the
newer receipt, readiness, price, and budget exposure win; unrelated session and
completion metadata can still update. A newer partial receipt stays pending.
Non-accounting attachment control events keep their existing handoff acceptance;
an old run's accounting report still cannot acquire a successor's authority.
The provider drains its bounded settled-turn usage tail before publishing the
terminal event that closes the run. An authoritative active goal keeps its
existing continuation path unless an uncorrected completion rejection closes
the run. The
same owner predicate selects both final usage drain and terminal outcome. A
completed turn remains authoritative after a
fully drained provider exit; ordinary session-exit handling still records an
unavailable process independently. This keeps a late cumulative increase on the old run before the
next run resets its baseline. Readiness and attachment retain accounting facts
in the existing provider checkpoint instead of discarding them, including facts
consumed before a later quiescence error. An increase
that arrives after authoritative completion requires explicit reconciliation:
PostgreSQL rejects changed finalized native usage or authoritative monetary evidence, so the existing Runner
outbox retains it under the old source identity without another charge. Equal
snapshots are harmless. No recurring scan or automatic adjustment is added.
If delayed usage arrives after attachment and cannot belong to the current
provider turn, the provider retains its cumulative evidence and the matching
saved prior identity, leaves the successor baseline unchanged, and emits a
failed run terminal. The existing reconciliation lifecycle blocks further work,
including after restart. Unknown older ownership stays explicit rather than
being inferred from provider-supplied run identity. Manual reconciliation or a
fresh session is required for this rare unsupported timing case.
Prior-turn increases are checked against the saved attachment baseline, even
when a successor has already reported a higher session total. Required usage
bypasses the optional diagnostic limit. An affected successor publishes an
explicit incomplete usage report before failure; this preserves known budget
exposure without settling ambiguous usage. That required report and its receipt
retain the accounting measurements and matching saved prior identity, so an
optional diagnostic acknowledgement cannot discard reconciliation evidence.
docs/deploy/storage.md
+3
View File
@@ -4,6 +4,9 @@ summary: Local disk vs S3-compatible storage
---
Paperclip stores uploaded files (issue attachments, images) using a configurable storage provider.
New run logs and explicitly requested provider traces reuse the same settings.
S3 instances store these under `run-logs/` and `provider-traces/` within the
existing prefix. Local-disk instances keep them local.
## Local Disk (Default)
maps/agent-access-map/agent-chat.md
+3 -1
View File
@@ -46,7 +46,9 @@ Review: `baseline`.
API: `present` — representative operation IDs:
- `GET /api/issues/{id}/comments`.
- `GET /api/issues/{id}/recent-comments` (20 contiguous comments for a standing chat).
- `GET /api/issues/{id}/comments` (cursor pages for older history).
- `GET /api/issues/{id}/runs?runIds=...` (at most 100 requested IDs, constrained to the accessible issue and company).
- `POST /api/issues/{id}/comments`.
- `GET /api/issues/{id}/decision-receipts` (own standing-chat board read).
- `GET /api/issues/{id}/chat-handoffs` (own standing-chat board read, paginated).
maps/feature-map/agent-chat.md
+4
View File
@@ -27,6 +27,8 @@ Open `/chats`, choose an agent, or use `/chats/:agentRef` and its conversation s
With Coordinator enabled, the composer exposes the shared model/thinking-level picker for supported agents. The agent stays fixed in this control; switching agents remains in the chat header. Choices save as conversation overrides on send, preserving the agent's configuration. A first message creates the conversation before saving settings and submitting through the ordered comments endpoint.
Standing chats open with the latest 20 contiguous messages. Scroll upward to fetch earlier messages, answer receipts, and created tasks. The loading indicator stays in the thread; history failures offer an inline Retry while preserving the draft and reading position. Opening a chat does not fetch its lifetime comment, activity, or run history. Already observed messages survive live refresh and Back within the query-cache session.
Coordinator answer receipts from this conversation jump to the answer bubble without leaving chat. Answer updates from an execution task link to that source task and its decision context. Compact document links read “Open [document title] in the sidebar”.
### `project-handoff`
@@ -43,6 +45,8 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Enab
### `agent-conversation`
Local browser recipe: [long-chat startup and scrolling](../../tests/perf/agent-chat/agent-chat-startup.spec.ts), with fixture and command details in the [performance README](../../tests/perf/agent-chat/README.md). It covers recent-only reveal, delayed and failed pages, Back, older comment links, and live refresh. Synthetic local data is not deployed acceptance evidence.
Automated: [chat landing](../../ui/src/pages/AgentChats.test.tsx) checks recent-chat routing; [conversation service](../../server/src/__tests__/agent-conversations.test.ts) covers persisted conversation behavior.
Automated: [composer picker](../../ui/src/components/task-chat/ComposerRunSettingsPicker.test.tsx) covers fixed-agent model/effort selection on desktop and mobile; [composer](../../ui/src/components/task-chat/TaskChatComposer.test.tsx) covers Coordinator gating and settings submission without reassignment. [Task detail](../../ui/src/pages/IssueDetail.test.tsx) verifies canonical first/existing-chat IDs, settings-before-comment ordering, and no comment submission when settings fail to save. These tests mock the API and do not prove provider execution with the chosen model.
maps/feature-map/projects.md
+8
View File
@@ -44,6 +44,14 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use
Automated: [projects UI](../../ui/src/pages/Projects.test.tsx) and [project detail](../../ui/src/pages/ProjectDetail.test.tsx) are component tests.
Automated: the explicit [`project-task-context` deployment journey](../../tests/runner-e2e/product-journeys/project-context-flow.ts)
creates a project and a self-assigned task through the production dialogs. It
reopens both, verifies the project task list and task properties after reload,
and independently checks the saved company, project, prompt, assignment and
standard work mode with zero agent runs. [Oracle tests](../../tests/runner-e2e/product-journeys/project-context-evidence.test.ts)
reject lost links and mismatched records. This describes coverage; each target
still needs its own live pass before daily promotion.
Manual: Create a project, add a task from its view, and reload. Verify the task belongs to that project, list counts are plausible, and links preserve company scope. Archive only the disposable project and inspect list/task behavior.
### `repository-settings`
maps/feature-map/runs-adapters.md
+1 -1
View File
@@ -348,7 +348,7 @@ revised-source live onboarding and follow-up proof.
`/company/settings/connections` configures decision/fast-response models,
not the general model-account library.
- The run log is instance database data; it is not the opt-out first-party Telemetry system or operator-configured OpenTelemetry tracing.
- Structured run events stay in the instance database. Optional stdout/stderr and raw trace payload storage can use S3; see [run evidence storage](../../doc/run-evidence-storage.md). Both are separate from first-party Telemetry and operator OpenTelemetry export.
- Authentication/model discovery does not establish tool, image, steering, or resumption capability parity.
- Use [recovery](./recovery.md) for interrupted runs and [execution environments](./execution-environments.md) for local/SSH/sandbox differences.
packages/paperclip-runner/protocol/manifest.json
+1 -1
View File
@@ -140,7 +140,7 @@
{
"path": "schemas/usage.schema.json",
"id": "https://paperclip.dev/schemas/prp/v1/usage.schema.json",
"sha256": "34b1478d2054f898d64b54b62a9bb738670087893bb1e5967da541539d2daf36"
"sha256": "be9609768b224459546b800236045b375448f2cb2cb47ccb337e149e0b016361"
},
{
"path": "schemas/workspace-diff.schema.json",
packages/paperclip-runner/protocol/schemas/usage.schema.json
+67
View File
@@ -11,6 +11,73 @@
"providerRequestId": { "type": ["string", "null"], "maxLength": 240 },
"cumulative": { "$ref": "#/$defs/measurement" },
"runDeltaAvailable": { "type": "boolean" },
"runDeltaBasis": { "const": "run" },
"reconciliation": {
"type": "object",
"required": [
"reason",
"receivedTurnId",
"cumulative",
"priorOrigin"
],
"properties": {
"reason": {
"const": "accounting_reconciliation_required"
},
"receivedTurnId": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"cumulative": {
"$ref": "#/$defs/measurement"
},
"priorOrigin": {
"type": [
"object",
"null"
],
"required": [
"runnerInstanceId",
"runId",
"normalizedSessionId",
"turnId",
"itemId"
],
"properties": {
"runnerInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 240
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"runDelta": { "$ref": "#/$defs/measurement" }
},
"additionalProperties": false,
packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs
+31 -3
View File
@@ -21,6 +21,8 @@ struct Script {
retention_limit: usize,
release_file: PathBuf,
receipt_file: PathBuf,
#[serde(default)]
accounting: bool,
}
#[derive(Default)]
@@ -86,11 +88,37 @@ fn workload(script: Script, shared: Shared) {
digest = Sha256::digest(digest);
}
work = lock.lock().unwrap();
let (event_type, payload) = if script.accounting {
if step == 1 {
(
"usage.reported",
json!({"provider":"openai", "model":"test-model",
"runDeltaAvailable":true, "runDeltaBasis":"run", "runDelta":{"inputTokens":120,"outputTokens":10,"providerCostUsd":0.25}}),
)
} else if step == 2 {
("turn.completed", json!({"status":"completed"}))
} else {
(
"usage.reported",
json!({"provider":"openai","model":"test-model","runDeltaAvailable":true,"runDeltaBasis":"run",
"runDelta":{"inputTokens":150,"outputTokens":15,"providerCostUsd":0.5}}),
)
}
} else {
(
"item.delta",
json!({"text": format!("step={step} sha256={digest:x}\n")}),
)
};
work.events.push_back(PolledEvent {
executor_event_id: format!("fixture-step-{step}"),
event_type: "item.delta".to_owned(),
priority: EventPriority::P2,
payload: json!({"text": format!("step={step} sha256={digest:x}\n")}),
event_type: event_type.to_owned(),
priority: if script.accounting {
EventPriority::P0
} else {
EventPriority::P2
},
payload,
});
work.completed = step;
work.peak_retained = work.peak_retained.max(work.events.len());
packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+81 -4
View File
@@ -333,20 +333,60 @@ fn finish_turn_with_send(
"text": "Codex completed the fake turn."
}}
}))?;
let cumulative_accounting = std::env::args().any(|arg| arg == "--cumulative-accounting-usage");
let warm_tail_accounting = std::env::args().any(|arg| arg == "--warm-tail-accounting");
let total_input = if warm_tail_accounting {
if state.next_turn == 1 {
100
} else {
150
}
} else if cumulative_accounting {
100 + state.next_turn * 100
} else {
12
};
let total_output = if warm_tail_accounting {
if state.next_turn == 1 {
10
} else {
15
}
} else if cumulative_accounting {
20 + state.next_turn * 10
} else {
3
};
if cumulative_accounting {
send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":total_input-40,"outputTokens":total_output-4},"last":{"inputTokens":10,"outputTokens":1}}}}),
)?;
}
send(json!({
"method": "thread/tokenUsage/updated",
"params": {
"threadId": state.thread_id,
"tokenUsage": {
"total": {"inputTokens": 12, "outputTokens": 3},
"last": {"inputTokens": 12, "outputTokens": 3, "requests": 1}
"total": {"inputTokens": total_input, "outputTokens": total_output},
"last": {"inputTokens": if cumulative_accounting {40} else {12}, "outputTokens": if cumulative_accounting {4} else {3}, "requests": 1}
}
}
}))?;
if state.next_turn > 1 && std::env::args().any(|arg| arg == "--hold-second-accounting-terminal")
{
return Ok(());
}
send(json!({
"method": "turn/completed",
"params": {"turn": {"id": turn_id, "status": status}}
}))?;
if std::env::args().any(|arg| arg == "--accounting-tail-after-terminal") {
send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}),
)?;
}
Ok(())
}
@@ -1285,7 +1325,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
for _ in 0..2 {
send(json!({"method": "thread/tokenUsage/updated", "params": {
"threadId": state.thread_id, "turnId": format!("provider-turn-{}", state.next_turn),
"tokenUsage": {"total": {"inputTokens": 100, "outputTokens": 20}, "last": {"inputTokens": 50, "outputTokens": 10}}
"tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {100 + state.next_turn * 100} else {100}, "outputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {20 + state.next_turn * 10} else {20}}, "last": {"inputTokens": 50, "outputTokens": 10}}
}}))?;
}
}
@@ -1889,6 +1929,43 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
}
}))?;
}
if state.next_turn == 1 {
if let Some(gate) =
argument(&args, "--late-accounting-tail-gate").map(PathBuf::from)
{
let thread_id = state.thread_id.clone();
let turn_id = provider_turn_id.clone();
let unsafe_tail =
args.iter().any(|arg| arg == "--unsafe-after-late-usage");
let duplicate_burst = args
.iter()
.any(|arg| arg == "--late-accounting-duplicate-burst");
thread::spawn(move || {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
while !gate.is_file() && std::time::Instant::now() < deadline {
thread::sleep(Duration::from_millis(1));
}
if gate.is_file() {
if duplicate_burst {
for _ in 0..33 {
let _ = send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id,"tokenUsage":{"total":{"inputTokens":100,"outputTokens":10}}}}),
);
}
}
let _ = send(
json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id,
"tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}),
);
if unsafe_tail {
let _ = send(
json!({"method":"turn/started","params":{"threadId":thread_id,"turn":{"id":"unsafe-next-turn"}}}),
);
}
}
});
}
}
if emit_post_completion_warning {
send(json!({
"method": "warning",
@@ -1900,7 +1977,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
json!({
"method": "thread/tokenUsage/updated",
"params": {"threadId": state.thread_id, "turnId": provider_turn_id,
"tokenUsage": {"total": {"inputTokens": 120, "outputTokens": 12},
"tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {150} else {120}, "outputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {15} else {12}},
"last": {"inputTokens": 20, "outputTokens": 2}}}
}),
json!({
packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+184 -100
View File
@@ -1342,6 +1342,7 @@ impl CodexProvider {
&mut self,
authorized_tools: impl IntoIterator<Item = AuthorizedTool>,
completion_contract: Option<(&str, &[String])>,
accounting: &mut Vec<(String, Value)>,
) -> Result<bool, LocalRunnerError> {
let authorized_tools = authorized_tools.into_iter().collect::<Vec<_>>();
let completion_contract =
@@ -1354,7 +1355,15 @@ impl CodexProvider {
{
return Ok(false);
}
let blockers = self.warm_run_attachment_blockers(true)?;
let (tail, result) = self.drain_completed_turn_tail_for_warm_attachment();
accounting.extend(tail);
result?;
if !accounting.is_empty() {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment requires pending accounting reconciliation",
));
}
let blockers = self.warm_run_attachment_blockers()?;
if !blockers.is_empty() {
return Err(LocalRunnerError::invalid(
format!(
@@ -1375,49 +1384,53 @@ impl CodexProvider {
Ok(true)
}
fn drain_completed_turn_tail_for_warm_attachment(&mut self) -> Result<(), LocalRunnerError> {
let Some(completed_turn_id) = self
.completed_turn_authority
.as_ref()
.map(|authority| authority.provider_turn_id.clone())
else {
return Ok(());
};
if self.active_provider_turn_id.is_some() {
return Ok(());
}
// Readiness probes run over the PRP command channel while provider
// stdout is drained by runnerd's adjacent control-loop iteration. A
// final usage/warning/item frame can therefore land after the last
// successful probe but before run.attach executes. Close that race in
// the same critical section as authority rotation. Only bounded tail
// notifications for the already-settled turn may be discarded: a new
// turn, provider request, process exit, or mismatched turn remains a
// fail-closed attachment error.
let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
let mut quiet_since: Option<std::time::Instant> = None;
let mut drained = 0usize;
loop {
if std::time::Instant::now() >= deadline {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail did not become quiescent",
));
pub(crate) fn drain_completed_turn_tail_for_warm_attachment(
&mut self,
) -> (Vec<(String, Value)>, Result<(), LocalRunnerError>) {
let mut accounting = Vec::new();
let result = (|| {
let Some(completed_turn_id) = self
.completed_turn_authority
.as_ref()
.map(|authority| authority.provider_turn_id.clone())
else {
return Ok(());
};
if self.active_provider_turn_id.is_some() {
return Ok(());
}
match self.poll()? {
Some(CodexProviderEvent::Notification { method, params }) => {
quiet_since = None;
drained = drained.saturating_add(1);
if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail exceeded its bounded frame limit",
));
}
let names_other_turn = notification_turn_id(&params)
.is_some_and(|turn_id| turn_id != completed_turn_id);
let safe_tail_method = matches!(
method.as_str(),
"warning"
// Readiness probes run over the PRP command channel while provider
// stdout is drained by runnerd's adjacent control-loop iteration. A
// final usage/warning/item frame can therefore land after the last
// successful probe but before run.attach executes. Close that race in
// the same critical section as authority rotation. Only bounded tail
// notifications for the already-settled turn may be discarded: a new
// turn, provider request, process exit, or mismatched turn remains a
// fail-closed attachment error.
let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE;
let mut quiet_since: Option<std::time::Instant> = None;
let mut drained = 0usize;
loop {
if std::time::Instant::now() >= deadline {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail did not become quiescent",
));
}
match self.poll()? {
Some(CodexProviderEvent::Notification { method, params }) => {
quiet_since = None;
drained = drained.saturating_add(1);
if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment tail exceeded its bounded frame limit",
));
}
let names_other_turn = notification_turn_id(&params)
.is_some_and(|turn_id| turn_id != completed_turn_id);
let safe_tail_method = matches!(
method.as_str(),
"warning"
| "configWarning"
| "remoteControl/status/changed"
| "mcpServer/startupStatus/updated"
@@ -1436,73 +1449,82 @@ impl CodexProvider {
| "thread/status/changed"
| "turn/diff/updated"
| "turn/plan/updated"
);
if names_other_turn || !safe_tail_method {
return Err(LocalRunnerError::invalid(format!(
);
if names_other_turn || !safe_tail_method {
return Err(LocalRunnerError::invalid(format!(
"Codex warm run attachment observed unsafe post-terminal provider method {}",
bounded_method(&method)
)));
}
if let Some(frame_id) = self.take_provider_trace_frame_id() {
self.record_provider_trace_interpretation(
}
if matches!(
method.as_str(),
"thread/tokenUsage/updated" | "paperclip/resumeUsageSnapshot"
) {
accounting.push((method, params));
continue;
}
if let Some(frame_id) = self.take_provider_trace_frame_id() {
self.record_provider_trace_interpretation(
frame_id,
"codex.warm_attachment.completed_turn_tail",
"ignored",
Vec::new(),
"Provider emitted a bounded tail notification after the prior turn terminal and before run attachment",
);
}
}
}
Some(CodexProviderEvent::ResourceLimit { .. }) => {
return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
}
Some(CodexProviderEvent::ProtocolFailure { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex protocol integrity failed during warm attachment",
));
}
Some(CodexProviderEvent::DescendantNotification { .. }) => {
// Child output cannot certify a quiescent root attachment.
return Err(LocalRunnerError::invalid(
"Codex descendant remains active during warm run attachment",
));
}
Some(CodexProviderEvent::ToolCall { .. })
| Some(CodexProviderEvent::RuntimeRequest { .. })
| Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment observed a post-terminal provider request",
));
}
Some(CodexProviderEvent::Exited { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex exited while quiescing for warm run attachment",
));
}
None => {
let now = std::time::Instant::now();
let quiet_start = quiet_since.get_or_insert(now);
if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
return Ok(());
Some(CodexProviderEvent::ResourceLimit { .. }) => {
return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation"));
}
Some(CodexProviderEvent::ProtocolFailure { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex protocol integrity failed during warm attachment",
));
}
Some(CodexProviderEvent::DescendantNotification { .. }) => {
// Child output cannot certify a quiescent root attachment.
return Err(LocalRunnerError::invalid(
"Codex descendant remains active during warm run attachment",
));
}
Some(CodexProviderEvent::ToolCall { .. })
| Some(CodexProviderEvent::RuntimeRequest { .. })
| Some(CodexProviderEvent::McpPermissionRequest { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex warm run attachment observed a post-terminal provider request",
));
}
Some(CodexProviderEvent::Exited {
completed_turn_authoritative,
completion_reconciles_exit,
..
}) if completion_reconciles_exit
|| (completed_turn_authoritative && self.process.stdout_drained()) =>
{
return Ok(())
}
Some(CodexProviderEvent::Exited { .. }) => {
return Err(LocalRunnerError::invalid(
"Codex exited while quiescing for warm run attachment",
));
}
None => {
let now = std::time::Instant::now();
let quiet_start = quiet_since.get_or_insert(now);
if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW {
return Ok(());
}
std::thread::sleep(Duration::from_millis(1));
}
std::thread::sleep(Duration::from_millis(1));
}
}
}
})();
(accounting, result)
}
pub(crate) fn warm_run_attachment_blockers(
&mut self,
quiesce_completed_tail: bool,
) -> Result<Vec<&'static str>, LocalRunnerError> {
// Only an explicit attachment-readiness probe may consume the bounded,
// already-settled provider suffix. Ordinary checkpoint snapshots run
// immediately after a terminal frame while the provider can still be
// unwinding; turning those observations into quiescence barriers can
// quarantine an otherwise reusable runner before its next turn.
if quiesce_completed_tail {
self.drain_completed_turn_tail_for_warm_attachment()?;
}
let mut blockers = Vec::new();
if self.process.try_wait()?.is_some() {
blockers.push("process_exited");
@@ -1517,7 +1539,18 @@ impl CodexProvider {
blockers.push("ambiguous_turn_start");
}
if !self.pending_messages.is_empty() {
blockers.push("pending_messages");
blockers.push(
if self.pending_messages.iter().any(|message| {
matches!(
message.value.get("method").and_then(Value::as_str),
Some("paperclip/resumeUsageSnapshot" | "thread/tokenUsage/updated")
)
}) {
"pending_accounting_reconciliation"
} else {
"pending_messages"
},
);
}
if !self.deferred_ambiguous_messages.is_empty() {
blockers.push("deferred_messages");
@@ -2919,13 +2952,11 @@ impl CodexProvider {
.and_then(Value::as_str);
if notification_turn_id.is_some()
&& notification_turn_id != self.active_provider_turn_id.as_deref()
&& notification_turn_id
&& (notification_turn_id
.is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id))
|| (self.active_provider_turn_id.is_none()
&& method == "thread/tokenUsage/updated"))
{
self.notification_identity_diagnostics += 1;
if self.notification_identity_diagnostics > 32 {
return Ok(None);
}
// Resume replays the cumulative usage of the last settled turn.
// Keep its identity and baseline, but never bill its `last`
// measurement to the newly attached run.
@@ -2941,6 +2972,10 @@ impl CodexProvider {
}),
}));
}
self.notification_identity_diagnostics += 1;
if self.notification_identity_diagnostics > 32 {
return Ok(None);
}
return Ok(Some(CodexProviderEvent::Notification {
method: "warning".to_owned(),
params: json!({
@@ -4392,9 +4427,58 @@ done
});
// poll() normalizes settled-turn usage to paperclip/resumeUsageSnapshot.
// That accounting fact is not new work and must not force replacement.
let result = provider.drain_completed_turn_tail_for_warm_attachment();
let (accounting, result) = provider.drain_completed_turn_tail_for_warm_attachment();
provider.shutdown().unwrap();
result.expect("historical usage must not break provider continuity");
result.expect("historical usage must remain available to its owner");
assert_eq!(accounting.len(), 1);
assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
}
#[cfg(unix)]
#[test]
fn failed_warm_drain_retains_consumed_usage_before_unsafe_tail() {
for attach in [false, true] {
let mut provider = completion_tail_provider();
provider
.restore_completed_turn_authority(true, Some(1), Some("reader-tail-1"))
.unwrap();
provider.active_provider_turn_id = None;
for value in [
json!({"method":"thread/tokenUsage/updated","params":{"threadId":"reader-tail-thread","turnId":"reader-tail-1","tokenUsage":{"total":{"inputTokens":120,"outputTokens":12}}}}),
json!({"method":"turn/started","params":{"threadId":"reader-tail-thread","turn":{"id":"unsafe-next-turn"}}}),
] {
provider
.pending_messages
.push_back(BufferedProviderMessage {
value,
trace_frame_id: None,
});
}
let (accounting, result) = if attach {
let mut accounting = Vec::new();
let tools = provider.authorized_tools.clone();
let contract = provider.completion_contract.clone();
let result = provider
.attach_run_in_place(
tools,
contract.as_ref().map(|contract| {
(
contract.revision.as_str(),
contract.criterion_ids.as_slice(),
)
}),
&mut accounting,
)
.map(|_| ());
(accounting, result)
} else {
provider.drain_completed_turn_tail_for_warm_attachment()
};
provider.shutdown().unwrap();
assert!(result.is_err());
assert_eq!(accounting.len(), 1);
assert_eq!(accounting[0].1["total"]["inputTokens"], 120);
}
}
#[cfg(unix)]
packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs
+234
View File
@@ -0,0 +1,234 @@
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
pub(crate) struct CodexRunUsage {
baseline: Option<BTreeMap<String, Value>>,
latest: BTreeMap<String, Value>,
}
impl CodexRunUsage {
pub(crate) fn new(requires_baseline: bool) -> Self {
Self {
baseline: (!requires_baseline).then(BTreeMap::new),
latest: BTreeMap::new(),
}
}
pub(crate) fn is_valid(&self) -> bool {
self.baseline
.iter()
.flat_map(|values| values.values())
.chain(self.latest.values())
.all(|value| {
value
.as_f64()
.is_some_and(|value| value.is_finite() && value >= 0.0)
})
}
pub(crate) fn increases_known_usage(&self, total: &Value) -> bool {
!self.latest.is_empty()
&& total.as_object().is_some_and(|total| {
total.iter().any(|(key, value)| {
value.as_f64().is_some_and(|value| {
value > self.latest.get(key).and_then(Value::as_f64).unwrap_or(0.0)
})
})
})
}
pub(crate) fn increases_run_baseline(&self, total: &Value) -> bool {
self.baseline.as_ref().is_some_and(|baseline| {
total.as_object().is_some_and(|total| {
total.iter().any(|(key, value)| {
value.as_f64().is_some_and(|value| {
value > baseline.get(key).and_then(Value::as_f64).unwrap_or(0.0)
})
})
})
})
}
pub(crate) fn has_baseline(&self) -> bool {
self.baseline.is_some()
}
pub(crate) fn begin_run(&mut self) {
if self.latest.is_empty() {
return;
}
self.baseline = (valid_token(self.latest.get("inputTokens"))
&& valid_token(self.latest.get("outputTokens")))
.then(|| self.latest.clone());
}
pub(crate) fn observe(&mut self, total: &Value, historical: bool) -> (Value, bool) {
let measurement: BTreeMap<String, Value> = total
.as_object()
.into_iter()
.flat_map(|value| value.iter())
.filter(|(_, value)| {
value
.as_f64()
.is_some_and(|value| value.is_finite() && value >= 0.0)
})
.map(|(key, value)| (key.clone(), value.clone()))
.collect();
let complete = self.baseline.is_some()
&& (historical
|| measurement.iter().all(|(key, value)| {
!key.ends_with("Tokens")
|| value.as_u64().is_some_and(|value| {
value >= self.latest.get(key).and_then(Value::as_u64).unwrap_or(0)
&& value
>= self
.baseline
.as_ref()
.and_then(|baseline| baseline.get(key))
.and_then(Value::as_u64)
.unwrap_or(0)
})
}))
&& valid_token(measurement.get("inputTokens"))
&& valid_token(measurement.get("outputTokens"))
&& measurement
.iter()
.all(|(key, value)| !key.ends_with("Tokens") || valid_token(Some(value)))
&& self
.latest
.keys()
.all(|key| !key.ends_with("Tokens") || valid_token(measurement.get(key)));
if historical
&& self.baseline.is_none()
&& valid_token(measurement.get("inputTokens"))
&& valid_token(measurement.get("outputTokens"))
{
self.baseline = Some(measurement.clone());
}
for (key, value) in measurement {
if value.as_f64().unwrap()
>= self.latest.get(&key).and_then(Value::as_f64).unwrap_or(0.0)
{
self.latest.insert(key, value);
}
}
let delta: serde_json::Map<String, Value> = self
.latest
.iter()
.map(|(key, value)| {
let baseline = self
.baseline
.as_ref()
.and_then(|baseline| baseline.get(key));
let delta = if let Some(value) = value.as_u64() {
json!(value.saturating_sub(baseline.and_then(Value::as_u64).unwrap_or(0)))
} else {
json!((value.as_f64().unwrap()
- baseline.and_then(Value::as_f64).unwrap_or(0.0))
.max(0.0))
};
(key.clone(), delta)
})
.collect();
(Value::Object(delta), complete)
}
}
fn valid_token(value: Option<&Value>) -> bool {
value
.and_then(Value::as_u64)
.is_some_and(|value| value <= 9_007_199_254_740_991)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn preserves_resumed_baselines_across_requests_recovery_and_attachment() {
let mut usage = CodexRunUsage::new(true);
usage.observe(
&json!({"inputTokens":100,"outputTokens":10,"cachedInputTokens":20}),
true,
);
let (first, complete) = usage.observe(
&json!({"inputTokens":140,"outputTokens":16,"cachedInputTokens":25}),
false,
);
assert!(complete);
assert_eq!(first["inputTokens"], 40);
let mut restored: CodexRunUsage =
serde_json::from_value(serde_json::to_value(&usage).unwrap()).unwrap();
let (second, complete) = restored.observe(
&json!({"inputTokens":190,"outputTokens":25,"cachedInputTokens":30}),
false,
);
assert!(complete);
assert_eq!(
second,
json!({"inputTokens":90,"outputTokens":15,"cachedInputTokens":10})
);
restored.begin_run();
let (next, complete) = restored.observe(
&json!({"inputTokens":220,"outputTokens":29,"cachedInputTokens":32}),
false,
);
assert!(complete);
assert_eq!(
next,
json!({"inputTokens":30,"outputTokens":4,"cachedInputTokens":2})
);
}
#[test]
fn attachment_before_paid_work_preserves_a_proven_empty_baseline() {
let mut usage = CodexRunUsage::new(false);
usage.begin_run();
let (delta, complete) = usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
assert!(complete);
assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
let mut unknown = CodexRunUsage::new(true);
unknown.begin_run();
assert!(
!unknown
.observe(&json!({"inputTokens":100,"outputTokens":10}), false)
.1
);
}
#[test]
fn regressing_live_totals_remain_incomplete_without_losing_known_exposure() {
let mut usage = CodexRunUsage::new(false);
usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false);
let (delta, complete) = usage.observe(&json!({"inputTokens":90,"outputTokens":9}), false);
assert!(!complete);
assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10}));
let mut resumed = CodexRunUsage::new(true);
resumed.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
assert!(
!resumed
.observe(&json!({"inputTokens":90,"outputTokens":9}), false)
.1
);
}
#[test]
fn unknown_baselines_and_missing_counters_cannot_certify_usage() {
let mut usage = CodexRunUsage::new(true);
assert!(
!usage
.observe(&json!({"inputTokens":140,"outputTokens":16}), false)
.1
);
usage.observe(&json!({"inputTokens":100,"outputTokens":10}), true);
assert!(!usage.observe(&json!({"inputTokens":160}), false).1);
assert!(
usage
.observe(&json!({"inputTokens":170,"outputTokens":20}), false)
.1
);
}
}
packages/paperclip-runner/runner/crates/runner-core/src/lib.rs
+1
View File
@@ -13,6 +13,7 @@ pub mod aws_agentcore_provider;
pub mod claude_managed_provider;
pub mod codex_provider;
mod codex_startup_trust;
mod codex_usage;
mod configured_environment;
mod credential_observer;
pub mod dot_provider_backend;
packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs
+229 -11
View File
@@ -17,6 +17,7 @@ use crate::codex_provider::{
ProviderStartupObservation, ProviderStartupStage, RejectedAcceptedTurn,
MAX_SETTLED_PROVIDER_TURN_IDS,
};
use crate::codex_usage::CodexRunUsage;
use crate::durable::{
create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value,
validate_semantic_tool_input, verify_private_directory, Command, CommandExecution,
@@ -93,6 +94,13 @@ struct ProviderEventIdentity {
item_id: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct CodexAccountingOrigin {
identity: ProviderEventIdentity,
provider_turn_id: String,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
#[serde(rename_all = "camelCase", deny_unknown_fields)]
struct ProviderStartupCommand {
@@ -608,6 +616,105 @@ fn normalize_provider_notification(
method: &str,
params: &Value,
) -> Result<Vec<NormalizedProviderEvent>, DurableRunnerError> {
if state.config.provider == "codex"
&& ["thread/tokenUsage/updated", "paperclip/resumeUsageSnapshot"].contains(&method)
{
let historical = method == "paperclip/resumeUsageSnapshot";
let total = if historical {
params.get("total")
} else {
params.pointer("/tokenUsage/total")
}
.unwrap_or(&Value::Null);
let increases_known_usage = state
.codex_usage
.as_ref()
.is_some_and(|usage| usage.increases_known_usage(total));
let received_turn = params.get("turnId").and_then(Value::as_str);
let increases_prior_baseline = state.previous_accounting_origin.is_some()
&& received_turn != state.completed_provider_turn_id.as_deref()
&& state
.codex_usage
.as_ref()
.is_some_and(|usage| usage.increases_run_baseline(total));
if historical
&& (increases_known_usage || increases_prior_baseline)
&& params.get("turnId").and_then(Value::as_str)
!= state.active_provider_turn_id.as_deref()
{
state.lifecycle = "reconciliation_required".to_owned();
let owns_completed_turn = state.active_provider_turn_id.is_none()
&& state
.completed_provider_turn_id
.as_deref()
.is_some_and(|id| params.get("turnId").and_then(Value::as_str) == Some(id));
if !owns_completed_turn {
if state.active_provider_turn_id.is_some() {
state.settle_active_provider_turn_identity()?;
}
state.active_provider_turn_id = None;
let mut invalidation = normalize_codex_notification("thread/tokenUsage/updated", &json!({
"threadId":params.get("threadId"), "model":state.config.model, "tokenUsage":{"total":total}
})).remove(0);
invalidation.payload["runDeltaBasis"] = json!("run");
invalidation.payload["reconciliation"] = json!({"reason":"accounting_reconciliation_required",
"receivedTurnId":params.get("turnId"),"cumulative":invalidation.payload["cumulative"],
"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| received_turn == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity)});
return Ok(vec![
NormalizedProviderEvent {
event_type: "harness.diagnostic".to_owned(),
priority: EventPriority::P0,
payload: json!({"code":"codex_usage_accounting_integrity", "classification":"accounting_reconciliation_required",
"receivedTurnId":params.get("turnId"),"cumulative":total,"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| params.get("turnId").and_then(Value::as_str) == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity),
"message":"Late provider usage cannot be attributed to the attached run; explicit reconciliation is required"}),
},
invalidation,
NormalizedProviderEvent {
event_type: "turn.failed".to_owned(),
priority: EventPriority::P0,
payload: json!({"provider":"codex","status":"failed","code":"codex_usage_accounting_integrity","recoverable":false,
"message":"Late provider usage requires explicit accounting reconciliation"}),
},
NormalizedProviderEvent {
event_type: "run.terminal".to_owned(),
priority: EventPriority::P0,
payload: json!({"schema":"paperclip.prp.terminal.v1","provider":"codex","turnTerminalState":"failed","runTerminalState":"failed","reportedWorkDisposition":"needs_review"}),
},
]);
}
}
let had_baseline = state
.codex_usage
.as_ref()
.is_some_and(CodexRunUsage::has_baseline);
let (delta, complete) = state
.codex_usage
.get_or_insert_with(|| CodexRunUsage::new(true))
.observe(total, historical);
let mut normalized_params = params.clone();
let emit_usage = !historical || (had_baseline && increases_known_usage);
if historical && emit_usage {
normalized_params = json!({"threadId": params.get("threadId"), "turnId": params.get("turnId"),
"tokenUsage":{"total":total,"last":delta}});
} else if !historical {
normalized_params["tokenUsage"]["last"] = delta;
}
let mut events = normalize_codex_notification(
if emit_usage {
"thread/tokenUsage/updated"
} else {
method
},
&normalized_params,
);
for event in &mut events {
if event.event_type == "usage.reported" {
event.payload["runDeltaAvailable"] = json!(complete);
event.payload["runDeltaBasis"] = json!("run");
}
}
return Ok(events);
}
if method != "paperclip/runResult" {
return Ok(normalize_codex_notification(method, params)
.into_iter()
@@ -640,14 +747,21 @@ fn normalize_provider_notification(
}
}
fn terminal_completion_rejected(state: &CodexProviderState) -> bool {
state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none()
}
fn goal_continues_after_turn(state: &CodexProviderState, goal_active: bool) -> bool {
goal_active && !terminal_completion_rejected(state)
}
fn terminal_events(
state: &CodexProviderState,
event_type: &str,
goal_status: Option<&str>,
) -> Vec<NormalizedProviderEvent> {
let rejected_completion =
state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none();
if goal_status == Some("active") && !rejected_completion {
let rejected_completion = terminal_completion_rejected(state);
if goal_continues_after_turn(state, goal_status == Some("active")) {
return Vec::new();
}
let Some(contract) = state.completion_contract.as_ref() else {
@@ -1007,6 +1121,10 @@ struct CodexProviderState {
#[serde(default)]
opencode_launch_profile_digest: Option<String>,
#[serde(default)]
codex_usage: Option<CodexRunUsage>,
#[serde(default)]
previous_accounting_origin: Option<CodexAccountingOrigin>,
#[serde(default)]
completion_contract: Option<CompletionContractBinding>,
#[serde(default)]
tool_bridge: ProviderToolBridge,
@@ -1107,11 +1225,15 @@ impl CodexProviderState {
tool_bridge: ProviderToolBridge,
) -> Self {
let thread_id = config.provider_session_id.clone();
let codex_usage =
(config.provider == "codex").then(|| CodexRunUsage::new(thread_id.is_some()));
Self {
schema: PROVIDER_STATE_SCHEMA.to_owned(),
lifecycle: "prepared".to_owned(),
config,
opencode_launch_profile_digest: None,
codex_usage,
previous_accounting_origin: None,
completion_contract,
tool_bridge,
thread_id,
@@ -1154,6 +1276,13 @@ impl CodexProviderState {
self.tool_bridge.validate_recovered().map_err(|error| {
DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}"))
})?;
if self
.codex_usage
.as_ref()
.is_some_and(|usage| !usage.is_valid())
{
return Err(DurableRunnerError::invalid("invalid Codex usage baseline"));
}
let mut pending_event_ids = HashSet::new();
if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS
|| self
@@ -2501,7 +2630,43 @@ impl CodexCommandExecutor {
stable
}
fn persist_completed_accounting_tail(
&mut self,
tail: Vec<(String, Value)>,
failed: bool,
) -> Result<(), DurableRunnerError> {
if !tail.is_empty() || failed {
let state = self.state.as_mut().ok_or_else(|| {
DurableRunnerError::invalid("Codex provider has not been prepared")
})?;
for (method, params) in tail {
let events = normalize_provider_notification(state, &method, &params)?;
state.extend_events(events)?;
}
if failed {
state.lifecycle = "reconciliation_required".to_owned();
}
self.save_state()?;
}
Ok(())
}
fn retain_completed_accounting_tail(&mut self) -> Result<(), DurableRunnerError> {
let (tail, result) = self
.provider
.as_mut()
.map(CodexProvider::drain_completed_turn_tail_for_warm_attachment)
.unwrap_or_else(|| (Vec::new(), Ok(())));
self.persist_completed_accounting_tail(tail, result.is_err())?;
result.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain completed Codex accounting: {error}"
))
})
}
fn attach_run(&mut self, payload: &Value) -> Result<(), DurableRunnerError> {
self.retain_completed_accounting_tail()?;
let mut next_state = self
.state
.clone()
@@ -2600,6 +2765,17 @@ impl CodexCommandExecutor {
"run.attach tool contract could not be rebound: {error}"
))
})?;
if let Some(usage) = next_state.codex_usage.as_mut() {
usage.begin_run();
}
next_state.previous_accounting_origin = self
.event_identity
.clone()
.zip(next_state.completed_provider_turn_id.clone())
.map(|(identity, provider_turn_id)| CodexAccountingOrigin {
identity,
provider_turn_id,
});
next_state.completion_contract = completion_contract;
next_state.completed_turn_authoritative = false;
next_state.completed_turn_process_generation = None;
@@ -2613,10 +2789,12 @@ impl CodexCommandExecutor {
next_state.active_provider_result_disposition = None;
next_state.active_terminal_tool_rejected = false;
next_state.last_agent_message = None;
let retained_provider = if let Some(provider) = self.provider.as_mut() {
!runtime_launch_changed
&& !upgraded_skill_config
&& provider
let mut attachment_accounting = Vec::new();
let retention = if let Some(provider) = self.provider.as_mut() {
if runtime_launch_changed || upgraded_skill_config {
Ok(false)
} else {
provider
.attach_run_in_place(
next_state.tool_bridge.authorized_tools().cloned(),
next_state.completion_contract.as_ref().map(|contract| {
@@ -2625,12 +2803,14 @@ impl CodexCommandExecutor {
contract.criterion_ids.as_slice(),
)
}),
&mut attachment_accounting,
)
.map_err(|error| {
DurableRunnerError::invalid(format!(
"failed to retain Codex for warm run attachment: {error}"
))
})?
})
}
} else if next_state.lifecycle == "prepared"
&& next_state.provider_process_generation > 0
&& next_state.pending_events.iter().all(is_startup_audit_event)
@@ -2640,12 +2820,17 @@ impl CodexCommandExecutor {
// validated run-scoped settings here; open_session then restores
// that same thread under the new authority. Never restart during
// drain/suspend or require the stopped process to still exist.
false
Ok(false)
} else {
return Err(DurableRunnerError::invalid(
"run.attach requires the restored Codex provider process",
));
};
self.persist_completed_accounting_tail(attachment_accounting, retention.is_err())?;
let retained_provider = match retention {
Ok(retained) => retained,
Err(error) => return Err(error),
};
if !retained_provider {
if let Some(provider) = self.provider.as_mut() {
provider.shutdown().map_err(|error| {
@@ -4044,10 +4229,13 @@ impl CodexCommandExecutor {
.get("quiesceForWarmAttach")
.and_then(Value::as_bool)
.unwrap_or(false);
if quiesce_for_warm_attach {
self.retain_completed_accounting_tail()?;
}
let mut warm_attach_blockers = self
.provider
.as_mut()
.map(|provider| provider.warm_run_attachment_blockers(quiesce_for_warm_attach))
.map(|provider| provider.warm_run_attachment_blockers())
.transpose()
.map_err(|error| {
DurableRunnerError::invalid(format!(
@@ -4071,6 +4259,9 @@ impl CodexCommandExecutor {
if !state.queued_events.is_empty() {
warm_attach_blockers.push("durable_queued_events");
}
if state.lifecycle == "reconciliation_required" {
warm_attach_blockers.push("reconciliation_required");
}
let warm_attach_ready = warm_attach_blockers.is_empty();
Ok(CommandExecution::result(json!({
"status": state.lifecycle,
@@ -4281,6 +4472,22 @@ impl CodexCommandExecutor {
} else {
None
};
let goal_remains_active = match &goal_reconciliation {
Some(Ok(snapshot)) => normalize_codex_goal(snapshot, false)
.is_some_and(|goal| goal.status == "active"),
_ => self
.state
.as_ref()
.and_then(|state| state.goal.as_ref())
.is_some_and(|goal| goal.status == "active"),
};
let run_continues = self
.state
.as_ref()
.is_some_and(|state| goal_continues_after_turn(state, goal_remains_active));
if terminal_event_type.as_deref() == Some("turn.completed") && !run_continues {
self.retain_completed_accounting_tail()?;
}
let working_now = self
.provider
.as_ref()
@@ -4370,7 +4577,9 @@ impl CodexCommandExecutor {
state.receipt_limit_interrupt_attempts = 0;
state.receipt_limit_interrupt_deadline_unix_ms = None;
state.ambiguous_turn_start_pending = false;
state.lifecycle = "session_open".to_owned();
if state.lifecycle != "reconciliation_required" {
state.lifecycle = "session_open".to_owned();
}
if let Some(goal) = state.goal.as_mut() {
goal.working_now = false;
state.goal_revision = state.goal_revision.saturating_add(1);
@@ -4473,6 +4682,13 @@ impl CodexCommandExecutor {
})
.unwrap_or_default();
self.save_state()?;
if self
.state
.as_ref()
.is_some_and(|state| state.lifecycle == "reconciliation_required")
{
break;
}
if let (Some(frame_id), Some(provider)) =
(trace_frame_id, self.provider.as_mut())
{
@@ -5894,6 +6110,8 @@ mod tests {
conversation_mode: None,
},
opencode_launch_profile_digest: None,
codex_usage: Some(CodexRunUsage::new(false)),
previous_accounting_origin: None,
completion_contract: None,
tool_bridge: ProviderToolBridge::default(),
thread_id: Some("thread-1".to_owned()),
packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+404 -2
View File
@@ -3964,6 +3964,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"--durable-turn-ids",
"--emit-post-completion-warning",
"--emit-post-completion-passive-statuses",
"--warm-tail-accounting",
],
);
let runner_config = durable_config(&directory);
@@ -3991,6 +3992,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
))
.expect("start the first turn");
let mut old_usage = Vec::new();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
assert!(
@@ -3998,6 +4000,12 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
"the first turn must settle before attachment"
);
let events = poll_and_ack(&mut executor).expect("poll the first turn terminal");
old_usage.extend(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.cloned(),
);
if events
.iter()
.any(|event| event.event_type == "turn.completed")
@@ -4010,6 +4018,11 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
std::thread::sleep(std::time::Duration::from_millis(1));
}
assert_eq!(
old_usage.last().unwrap().payload["runDelta"]["inputTokens"],
120
);
assert_eq!(old_usage.last().unwrap().payload["runDeltaAvailable"], true);
let readiness = executor
.execute(&command(
"readiness",
@@ -4034,6 +4047,23 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta
.iter()
.any(|(event_type, _, _)| event_type == "run.attached"));
executor
.execute(&command(
"next-turn",
6,
"turn.start",
json!({"text":"Complete the next paid turn."}),
))
.unwrap();
let next_events = wait_for_executor_event_batch(&mut executor, "turn.completed");
let next_usage = next_events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap();
assert_eq!(next_usage.payload["runDelta"]["inputTokens"], 30);
assert_eq!(next_usage.payload["runDelta"]["outputTokens"], 3);
assert_eq!(next_usage.payload["runDeltaAvailable"], true);
executor.shutdown().expect("stop the warm provider");
fs::remove_dir_all(directory).expect("remove Codex integration-test directory");
}
@@ -6256,6 +6286,9 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
let mut config = provider_config(&directory, &["--complete-after-tool-call"]);
if scenario == "controller-goal" {
config.args.push("--agent-created-goal-on-open".to_owned());
config
.args
.push("--accounting-tail-after-terminal".to_owned());
}
config.args.extend([
"--terminal-result-file".to_owned(),
@@ -6327,6 +6360,13 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() {
.unwrap();
assert!(failure < turn && turn < terminal);
if scenario == "controller-goal" {
let final_usage = emitted
.iter()
.rposition(|event| event.event_type == "usage.reported")
.unwrap();
assert!(final_usage < failure && final_usage < turn && final_usage < terminal);
assert_eq!(emitted[final_usage].payload["runDelta"]["inputTokens"], 120);
assert_eq!(emitted[final_usage].payload["runDeltaAvailable"], true);
let settled_goal = emitted
.iter()
.rposition(|event| {
@@ -6687,7 +6727,11 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
let directory = temporary_directory("resume-usage-snapshot");
let config = provider_config(
&directory,
&["--durable-turn-ids", "--resume-usage-snapshot"],
&[
"--durable-turn-ids",
"--resume-usage-snapshot",
"--cumulative-accounting-usage",
],
);
let runner_config = durable_config(&directory);
let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
@@ -6756,7 +6800,7 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() {
.collect();
assert_eq!(snapshots.len(), 2);
assert_eq!(snapshots[0].payload["receivedTurnId"], "provider-turn-1");
assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 100);
assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 200);
assert!(!events.iter().any(|event| matches!(
event.event_type.as_str(),
"provider.notice.recorded" | "usage.reported"
@@ -6988,3 +7032,361 @@ fn skill_flag_can_be_added_to_an_old_checkpoint_at_settled_run_attach() {
);
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn durable_codex_usage_keeps_run_baselines_through_requests_recovery_and_attachment() {
let directory = temporary_directory("accounting-run-baseline");
fs::write(
directory.join("fake-state.json"),
serde_json::to_vec(&json!({"threadId":"thread-1","activeTurnId":null,"nextTurn":1}))
.unwrap(),
)
.unwrap();
let mut config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--resume-usage-snapshot",
"--cumulative-accounting-usage",
],
);
config.provider_session_id = Some("thread-1".to_owned());
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
for _ in 0..20 {
if poll_and_ack(&mut executor)
.unwrap()
.iter()
.any(|event| event.payload["code"] == "codex_resume_usage_snapshot")
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(5));
}
fn run(
executor: &mut CodexCommandExecutor,
sequence: u64,
expected_input: u64,
expected_output: u64,
) {
executor
.execute(&command(
&format!("turn-{sequence}"),
sequence,
"turn.start",
json!({"text":"Complete the accounting turn."}),
))
.unwrap();
let events = wait_for_executor_event_batch(executor, "turn.completed");
let usage = events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.expect("provider emitted usage");
assert_eq!(usage.payload["runDeltaBasis"], "run");
assert_eq!(usage.payload["runDeltaAvailable"], true);
assert_eq!(usage.payload["runDelta"]["inputTokens"], expected_input);
assert_eq!(usage.payload["runDelta"]["outputTokens"], expected_output);
for _ in 0..20 {
if poll_and_ack(executor).unwrap().is_empty() {
break;
}
}
}
run(&mut executor, 3, 100, 10);
run(&mut executor, 4, 200, 20);
executor.shutdown().unwrap();
drop(executor);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
run(&mut executor, 5, 300, 30);
executor
.execute(&command(
"attach",
6,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap();
run(&mut executor, 7, 100, 10);
executor.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
#[test]
fn late_codex_usage_requires_durable_reconciliation_before_or_after_handoff() {
for mode in ["before", "after", "after-paid", "unsafe"] {
let handoff_first = mode.starts_with("after");
let directory = temporary_directory("late-accounting-reconciliation");
let gate = directory.join("release-late-usage");
let gate_arg = gate.to_string_lossy().into_owned();
let config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--warm-tail-accounting",
"--late-accounting-tail-gate",
&gate_arg,
if mode == "after-paid" {
"--hold-second-accounting-terminal"
} else {
"--ordinary-terminal"
},
if mode == "after-paid" {
"--late-accounting-duplicate-burst"
} else {
"--ordinary-tail-count"
},
if mode == "unsafe" {
"--unsafe-after-late-usage"
} else {
"--ordinary-late-usage"
},
],
);
let runner_config = durable_config(&directory);
let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor
.execute(&command(
"turn",
3,
"turn.start",
json!({"text":"Finish before the delayed accounting report."}),
))
.unwrap();
let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
assert_eq!(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap()
.payload["runDelta"]["inputTokens"],
100
);
if handoff_first {
executor
.execute(&command(
"attach",
4,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap();
let mut next = runner_config.clone();
next.run_id = "run-2".to_owned();
executor.rotate_authority(&next);
}
if mode == "after-paid" {
executor
.execute(&command(
"successor-turn",
5,
"turn.start",
json!({"text":"Start successor paid work."}),
))
.unwrap();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
assert!(std::time::Instant::now() < deadline);
let events = poll_and_ack(&mut executor).unwrap();
assert!(!events
.iter()
.any(|event| event.event_type == "turn.completed"));
if let Some(usage) = events
.iter()
.find(|event| event.event_type == "usage.reported")
{
assert_eq!(usage.payload["runDelta"]["inputTokens"], 50);
break;
}
}
}
fs::write(&gate, "release").unwrap();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
let retained = loop {
assert!(std::time::Instant::now() < deadline);
let events = if mode == "unsafe" {
match executor.execute(&command(
"quiesce",
4,
"session.snapshot",
json!({"quiesceForWarmAttach":true}),
)) {
Ok(_) => {
std::thread::sleep(std::time::Duration::from_millis(1));
continue;
}
Err(error) => {
assert!(error
.to_string()
.contains("failed to retain completed Codex accounting"));
executor.retained_events().unwrap()
}
}
} else {
executor.poll_events().unwrap()
};
if events.iter().any(|event| {
event.event_type == "usage.reported"
|| event.payload["code"] == "codex_usage_accounting_integrity"
}) {
break events;
}
executor.acknowledge_events(events.len()).unwrap();
};
if handoff_first {
assert!(retained
.iter()
.filter(|event| event.event_type == "usage.reported")
.all(|event| event.payload["runDeltaAvailable"] == false));
assert!(retained
.iter()
.any(|event| event.event_type == "usage.reported"
&& event.payload["runDeltaAvailable"] == false));
assert!(retained
.iter()
.any(|event| event.event_type == "run.terminal"
&& event.payload["runTerminalState"] == "failed"));
let required = retained
.iter()
.find(|event| event.event_type == "usage.reported")
.unwrap();
assert_eq!(
required.payload["reconciliation"]["cumulative"]["inputTokens"],
120
);
assert_eq!(
required.payload["reconciliation"]["priorOrigin"]["runId"],
"run-1"
);
let evidence = retained
.iter()
.find(|event| event.payload["code"] == "codex_usage_accounting_integrity")
.unwrap();
assert_eq!(evidence.payload["priorOrigin"]["runId"], "run-1");
assert_eq!(evidence.payload["receivedTurnId"], "provider-turn-1");
assert_eq!(evidence.payload["cumulative"]["inputTokens"], 120);
} else {
assert_eq!(
retained
.iter()
.find(|event| event.event_type == "usage.reported")
.unwrap()
.payload["runDelta"]["inputTokens"],
120
);
}
let saved: Value =
serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
.unwrap();
assert_eq!(saved["lifecycle"], "reconciliation_required");
assert!(!saved["pendingEvents"].as_array().unwrap().is_empty());
let error = executor
.execute(&command(
"blocked-attach",
5,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap_err();
assert!(error
.to_string()
.contains("requires explicit reconciliation"));
executor.shutdown().unwrap();
drop(executor);
let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config);
let error = restored
.execute(&command(
"blocked-after-restart",
6,
"run.attach",
json!({"authorizedTools":task_context_tool_set()}),
))
.unwrap_err();
assert!(error
.to_string()
.contains("requires explicit reconciliation"));
restored.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
}
#[test]
fn durable_codex_usage_keeps_completed_turn_through_reconciled_provider_exit() {
for exit_flag in [
"--exit-after-turn-completion",
"--fail-after-turn-completion",
] {
let directory = temporary_directory("accounting-reconciled-exit");
let config = provider_config(
&directory,
&[
"--durable-turn-ids",
"--warm-tail-accounting",
"--emit-post-completion-passive-statuses",
exit_flag,
],
);
let mut executor =
CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory));
executor
.execute(&command(
"prepare",
1,
"run.prepare",
json!({"provider":config,"authorizedTools":task_context_tool_set()}),
))
.unwrap();
executor
.execute(&command("open", 2, "session.open", json!({})))
.unwrap();
executor
.execute(&command(
"turn",
3,
"turn.start",
json!({"text":"Complete accounting before exiting."}),
))
.unwrap();
let events = wait_for_executor_event_batch(&mut executor, "turn.completed");
assert_eq!(
events
.iter()
.filter(|event| event.event_type == "usage.reported")
.last()
.unwrap()
.payload["runDelta"]["inputTokens"],
120
);
assert!(!events.iter().any(|event| event.event_type == "turn.failed"));
let saved: Value =
serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap())
.unwrap();
assert_ne!(saved["lifecycle"], "reconciliation_required");
assert_eq!(saved["completedTurnAuthoritative"], true);
executor.shutdown().unwrap();
fs::remove_dir_all(directory).unwrap();
}
}
packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+37
View File
@@ -1123,6 +1123,43 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre
}
}, 15_000);
it("installs required accounting acceptance before the Runner acknowledgement callback returns", async () => {
const directory = await mkdtemp(join(tmpdir(), "runnerd-required-accounting-"));
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
let callbacks!: ConstructorParameters<typeof OriginalCore>[0];
const spy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) {
callbacks = options;
return new OriginalCore(options);
});
let release!: () => void;
const accepted = new Promise<void>(resolve => { release = resolve; });
const onAccountingEvent = vi.fn(async () => accepted);
const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: directory, sourceCodexHome: "", environment: {},
runnerBinary: process.execPath, runnerArtifact: { version: "test", digest: `sha256:${"a".repeat(64)}` },
codexCommand: process.execPath, codexArgs: [], onAccountingEvent,
controlPlaneRegistration: async () => { throw new Error("stop before Runner dispatch"); },
});
try {
await expect(bundle.transport.request("thread/start", { cwd: directory })).rejects.toThrow();
const report = { eventType: "usage.reported", sourceEventId: "required-usage" } as PrpEvent;
let returned = false;
const result = callbacks.onCommittedEvent!(report).then(() => { returned = true; });
await vi.waitFor(() => expect(onAccountingEvent).toHaveBeenCalledExactlyOnceWith(report));
expect(returned).toBe(false);
release();
await result;
expect(returned).toBe(true);
const failure = new Error("receipt unavailable");
onAccountingEvent.mockRejectedValueOnce(failure);
await expect(callbacks.onCommittedEvent!(report)).rejects.toBe(failure);
} finally {
release();
await bundle.transport.close().catch(() => undefined);
spy.mockRestore();
await rm(directory, { recursive: true, force: true });
}
});
it("uses the same API result envelope for live dispatch and recovered receipts", async () => {
const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-recovered-api-"));
const OriginalCore = durableControlPlane.DurablePrpControlPlane;
packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+3
View File
@@ -1240,6 +1240,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD
provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx";
observeCredentials?: boolean;
onCredentialsUpdated?: (event: PrpEvent) => Promise<void>;
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"];
opencodePermissionMode?: NativeOpenCodePermissionMode;
acpxAgent?: QualifiedAcpxAgent;
@@ -5239,6 +5240,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
@@ -5691,6 +5693,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
onSemanticToolInput: (call) => this.#handleSemanticToolInput(call),
recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call),
onCommittedEvent: async (event) => {
await this.options.onAccountingEvent?.(event);
if (event.eventType === "auth.credentials.updated") {
await this.options.onCredentialsUpdated?.(event);
}
packages/paperclip-runner/src/protocol/generated/schema-bundle.ts
+69
View File
@@ -3385,6 +3385,75 @@ export const usageSchema = {
"runDeltaAvailable": {
"type": "boolean"
},
"runDeltaBasis": {
"const": "run"
},
"reconciliation": {
"type": "object",
"required": [
"reason",
"receivedTurnId",
"cumulative",
"priorOrigin"
],
"properties": {
"reason": {
"const": "accounting_reconciliation_required"
},
"receivedTurnId": {
"type": [
"string",
"null"
],
"maxLength": 240
},
"cumulative": {
"$ref": "#/$defs/measurement"
},
"priorOrigin": {
"type": [
"object",
"null"
],
"required": [
"runnerInstanceId",
"runId",
"normalizedSessionId",
"turnId",
"itemId"
],
"properties": {
"runnerInstanceId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"runId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"normalizedSessionId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"turnId": {
"type": "string",
"minLength": 1,
"maxLength": 240
},
"itemId": {
"type": "string",
"minLength": 1,
"maxLength": 240
}
},
"additionalProperties": false
}
},
"additionalProperties": false
},
"runDelta": {
"$ref": "#/$defs/measurement"
}
packages/paperclip-runner/src/protocol/replay-contract.test.ts
+24
View File
@@ -1,3 +1,5 @@
import Ajv2020 from "ajv/dist/2020.js";
import { usageSchema } from "./generated/schema-bundle.js";
import { readFile } from "node:fs/promises";
import { describe, expect, it } from "vitest";
@@ -6,6 +8,7 @@ import {
negotiateProtocolVersion,
parsePrpFixtureText,
PRP_PROTOCOL_VERSION,
validatePrpEvent,
} from "./replay-contract.js";
import { reducePrpFixture } from "../reducer/session-reducer.js";
@@ -371,3 +374,24 @@ describe("PRP v1 JSON Schema contract", () => {
).toBeNull();
});
});
it("accepts marked whole-run usage through the runtime standalone event validator", async () => {
const fixture = await readFixture();
const event = (fixture.events as Array<Record<string, unknown>>)[0];
const measurement = { inputTokens: 120, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 0,
activeSeconds: 0, requests: 1, providerCostUsd: 0 };
const result = validatePrpEvent({ ...event, eventType: "usage.reported", priority: 0,
payload: { provider: "codex", model: "gpt-6.1-sol", cumulative: measurement, runDelta: measurement,
runDeltaAvailable: true, runDeltaBasis: "run" } });
expect(result).toMatchObject({ ok: true });
});
it("declares required incomplete reconciliation evidence in the strict usage contract", () => {
const validate = new Ajv2020({ strict: true, allErrors: true }).compile(usageSchema);
const measurement = { inputTokens: 120, outputTokens: 12, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 0, providerCostUsd: 0 };
const payload = { provider: "codex", model: null, cumulative: measurement, runDelta: { ...measurement, inputTokens: 0, outputTokens: 0 }, runDeltaAvailable: false, runDeltaBasis: "run",
reconciliation: { reason: "accounting_reconciliation_required", receivedTurnId: "provider-turn-1", cumulative: measurement,
priorOrigin: { runnerInstanceId: "runner-1", runId: "old-run", normalizedSessionId: "session", turnId: "turn", itemId: "item" } } };
expect(validate(payload), JSON.stringify(validate.errors)).toBe(true);
expect(validate({ ...payload, reconciliation: { ...payload.reconciliation, arbitraryProviderData: "excluded" } })).toBe(false);
});
packages/shared/src/types/issue.ts
+1
View File
@@ -1779,6 +1779,7 @@ export interface IssueAttachment {
/** Bounded cold-open window; the complete chronological feed stays paginated. */
export interface IssueRecentComments {
latestCommentId: string | null;
oldestCommentId?: string | null;
comments: IssueComment[];
hasMore: boolean;
}
scripts/__tests__/release-verify-workflow.test.mjs
+34
View File
@@ -579,6 +579,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
readWorkflow("chat-feature-testops.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -618,6 +619,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
"chat-feature-testops.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -650,6 +652,38 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
(block) => [...block.matchAll(providerSecretReference)].length > 0,
);
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
if (name === "chat-feature-testops.yml") {
assert.equal(providerJobs.length, 1);
const assertBoundary = (block) => {
assert.match(block, /^ chat:\n/);
assert.match(block, /\n needs: \[authorize, select, build\]\n/);
assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
assert.match(block, /\n environment: runner-e2e-private-paid\n/);
assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/);
assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/);
assert.doesNotMatch(block, /ALLURE_TOKEN/);
assert.equal([...block.matchAll(providerSecretReference)].length, 1);
assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference));
};
const block = providerJobs[0];
assertBoundary(block);
assert.equal([...workflow.matchAll(providerSecretReference)].length, 1);
assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/);
assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/);
assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/);
for (const mutant of [
block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
block.replace(" - *guard\n", ""),
block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"),
block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"),
block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"),
]) assert.throws(() => assertBoundary(mutant));
continue;
}
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
scripts/__tests__/run-vitest-stable-shard.test.mjs
+4 -1
View File
@@ -320,6 +320,7 @@ test("the real serialized shard partition is complete, non-overlapping and durat
const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts";
const nativeRunnerSuitePath =
"server/src/services/native-runtime/native-codex-runner.integration.test.ts";
const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts";
const dotRunnerSuitePath = "server/src/__tests__/dot-runner.test.ts";
const restartRecoverySuitePath =
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts";
@@ -363,7 +364,8 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes
assert.ok(!files.includes(nativeRunnerSuitePath));
assert.ok(!files.includes(dotRunnerSuitePath));
assert.ok(!files.includes(restartRecoverySuitePath));
assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
assert.ok(!files.includes(usageDeliverySuitePath));
assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort());
assert.equal(new Set(files).size, files.length);
const defaultRun = dryRunJson([], prEnv);
assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
@@ -412,6 +414,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => {
"server/src/services/native-runtime/native-codex-runner.integration.test.ts",
dotRunnerSuitePath,
restartRecoverySuitePath,
usageDeliverySuitePath,
]);
});
scripts/lib/recovery-cutover-inventory.mjs
+99
View File
@@ -0,0 +1,99 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import { classifyRecoveryEntry, cutoverStatus, recoveryKinds } from "./recovery-cutover-policy.mjs";
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
// Inspect every ancestor: lstat on only the final component would follow an
// intermediate symlink. No directory/file names or OS errors enter the report.
async function directoryState(directory, io) {
const parsed = path.parse(directory);
let current = parsed.root;
for (const part of directory.slice(parsed.root.length).split(path.sep).filter(Boolean)) {
current = path.join(current, part);
try {
const stat = await io.lstat(current);
if (!stat.isDirectory() || stat.isSymbolicLink()) return "unsafe";
} catch (error) {
return error?.code === "ENOENT" ? "absent" : "unreadable";
}
}
return "directory";
}
/** Read-only, bounded entry inventory. Run with producers stopped on the retained
* volume. This is not an atomic snapshot and does not establish DB settlement.
* io exists for deterministic fault tests; production uses the actual filesystem. */
export async function inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries = 10_000 }, io = fs) {
if (!path.isAbsolute(instanceRoot) || (orphanDirectory !== undefined && !path.isAbsolute(orphanDirectory))) {
throw new Error("Recovery inventory requires absolute directories");
}
if (!Number.isSafeInteger(maxEntries) || maxEntries < 1 || maxEntries > 100_000) {
throw new Error("Entry limit must be between 1 and 100000");
}
const root = path.resolve(instanceRoot);
const rootState = await directoryState(root, io);
const directories = [path.join(root, "accounting-receipts"), path.join(root, "quota-credential-recovery"),
orphanDirectory === undefined ? path.join(root, "data", "sandbox-orphan-cleanup") : path.resolve(orphanDirectory)];
let remaining = maxEntries;
const summaries = [];
for (const [index, directory] of directories.entries()) {
const summary = { kind: recoveryKinds[index], presence: "unknown", observedEntries: 0,
unverifiedFiles: 0, unknownEntries: 0, readFailures: 0, complete: true, status: "unknown" };
if (rootState !== "directory") {
summary.complete = false;
} else {
const state = await directoryState(directory, io);
if (state === "absent") summary.presence = "absent";
else if (state !== "directory") {
summary.unknownEntries++;
summary.complete = false;
if (state === "unreadable") summary.readFailures++;
} else {
summary.presence = "present";
async function scan(target, depth) {
let handle;
try {
handle = await io.opendir(target);
while (true) {
// Read one sentinel entry to distinguish exactly-at-limit from
// truncated scans. The sentinel is not included in observed counts.
const entry = await handle.read();
if (entry === null) break;
if (remaining === 0) { summary.complete = false; break; }
remaining--;
summary.observedEntries++;
const entryKind = entry.isFile() ? "file"
: entry.isDirectory() && uuid.test(entry.name) ? "company-directory" : "other";
const classification = classifyRecoveryEntry(summary.kind, depth, entryKind);
if (classification === "descend") {
const child = path.join(target, entry.name);
// Recheck the entry before traversing; never intentionally follow
// links. Operators must quiesce writers for the snapshot to hold.
if (await directoryState(child, io) !== "directory") {
summary.unknownEntries++;
summary.complete = false;
} else await scan(child, depth + 1);
} else summary[classification]++;
if (!summary.complete && remaining === 0) break;
}
} catch {
summary.readFailures++;
summary.complete = false;
} finally {
if (handle) {
try { await handle.close(); }
catch { summary.readFailures++; summary.complete = false; }
}
}
}
await scan(directory, 0);
}
}
summary.status = cutoverStatus(summary);
summaries.push(summary);
}
return { schema: "paperclip/recovery-cutover-inventory/v1", status:
summaries.some(s => s.status === "unknown") ? "unknown" : summaries.some(s => s.status === "blocked") ? "blocked" : "clear",
instanceStatus: rootState, contentInspection: "none", counts: "observed-lower-bounds", maxEntries, directories: summaries };
}
scripts/lib/recovery-cutover-policy.mjs
+14
View File
@@ -0,0 +1,14 @@
/** File contents belong to each recovery owner. Inventory never parses them:
* even an apparently valid file cannot establish that its obligation settled. */
export const recoveryKinds = ["accounting-receipts", "quota-credential-recovery", "sandbox-orphan-cleanup"];
export function classifyRecoveryEntry(kind, depth, entry) {
if (entry === "file") return "unverifiedFiles";
if (entry === "company-directory" && kind === "quota-credential-recovery" && depth === 0) return "descend";
return "unknownEntries";
}
export function cutoverStatus(summary) {
if (!summary.complete || summary.unknownEntries > 0 || summary.readFailures > 0) return "unknown";
return summary.unverifiedFiles > 0 ? "blocked" : "clear";
}
scripts/recovery-cutover.mjs
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env node
import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
const usage = "Usage: node scripts/recovery-cutover.mjs --instance-root /absolute/instance [--orphan-directory /absolute/override] [--max-entries 10000] [--check]";
const args = process.argv.slice(2);
try {
let instanceRoot;
let orphanDirectory = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
let maxEntries = 10_000;
let check = false;
const seen = new Set();
for (let i = 0; i < args.length; i++) {
const argument = args[i];
if (seen.has(argument)) throw new Error("duplicate");
seen.add(argument);
if (argument === "--check") check = true;
else if (["--instance-root", "--orphan-directory", "--max-entries"].includes(argument)) {
const value = args[++i];
if (value === undefined || value.startsWith("--")) throw new Error("missing");
if (argument === "--instance-root") instanceRoot = value;
else if (argument === "--orphan-directory") orphanDirectory = value;
else maxEntries = Number(value);
} else throw new Error("unknown");
}
if (!instanceRoot) throw new Error("missing");
const report = await inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries });
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
if (check && report.status !== "clear") process.exitCode = 1;
} catch {
// Never echo args, filesystem errors or paths: they may contain secrets.
process.stderr.write(`Recovery inventory could not run. ${usage}\n`);
process.exitCode = 2;
}
scripts/recovery-cutover.test.mjs
+163
View File
@@ -0,0 +1,163 @@
import { afterEach, test } from "node:test";
import assert from "node:assert/strict";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs";
import { classifyRecoveryEntry, cutoverStatus } from "./lib/recovery-cutover-policy.mjs";
const cli = fileURLToPath(new URL("./recovery-cutover.mjs", import.meta.url));
const roots = [];
const company = "00000000-0000-4000-8000-000000000001";
const grant = "00000000-0000-4000-8000-000000000002";
async function fixture() {
// Canonicalize macOS /var -> /private/var, since the check rejects links.
const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "cutover-")));
roots.push(root);
return root;
}
async function write(root, relative, content = "secret-token-must-never-appear") {
const file = path.join(root, relative);
await fs.mkdir(path.dirname(file), { recursive: true });
await fs.writeFile(file, content);
return file;
}
function run(root, args = [], env = {}) {
return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], {
encoding: "utf8", env: { ...process.env, SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: "", ...env },
});
}
// An unset override differs from an invalid empty override.
function cleanRun(root, args = []) {
const env = { ...process.env };
delete env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR;
return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], { encoding: "utf8", env });
}
afterEach(async () => { await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); });
test("pure policy never clears files or unknown entries", () => {
assert.equal(classifyRecoveryEntry("accounting-receipts", 0, "file"), "unverifiedFiles");
assert.equal(classifyRecoveryEntry("quota-credential-recovery", 0, "company-directory"), "descend");
assert.equal(classifyRecoveryEntry("quota-credential-recovery", 1, "company-directory"), "unknownEntries");
assert.equal(cutoverStatus({ complete: false, unknownEntries: 0, readFailures: 0, unverifiedFiles: 0 }), "unknown");
});
test("CLI: absent and empty legacy layouts clear without migration", async () => {
const root = await fixture();
let result = cleanRun(root, ["--check"]);
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
for (const directory of ["accounting-receipts", `quota-credential-recovery/${company}`, "data/sandbox-orphan-cleanup"]) {
await fs.mkdir(path.join(root, directory), { recursive: true });
}
result = cleanRun(root, ["--check"]);
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
});
test("CLI: real pending legacy files block, preserve bytes and reveal no secrets or names", async () => {
const root = await fixture();
const receipt = JSON.stringify({ schema: "paperclip/accounting-receipt/v1", id: grant, companyId: company,
runId: company, sourceId: company, sequence: 1, receivedAt: "2026-10-30T00:00:00.000Z", adapterType: "codex_local",
receipt: { complete: true, usage: { inputTokens: 1, outputTokens: 1 } } });
const files = await Promise.all([
write(root, `accounting-receipts/${grant}.json`, receipt),
write(root, `quota-credential-recovery/${company}/${grant}.json`, '{"ciphertext":"secret-token-must-never-appear"}'),
write(root, "data/sandbox-orphan-cleanup/secret-filename.json", JSON.stringify({ companyId: company,
environmentId: company, executionWorkspaceId: null, issueId: null, heartbeatRunId: null, provider: "daytona",
providerLeaseId: "private-resource", metadata: {} })),
]);
const before = await Promise.all(files.map(file => fs.readFile(file)));
const result = cleanRun(root, ["--check"]);
assert.equal(result.status, 1);
const report = JSON.parse(result.stdout);
assert.equal(report.status, "blocked");
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 1]);
assert.doesNotMatch(result.stdout + result.stderr, /secret-token|secret-filename|private-resource|ciphertext|00000000-/);
assert.deepEqual(await Promise.all(files.map(file => fs.readFile(file))), before);
assert.equal(cleanRun(root).status, 0, "report-only mode does not fail on obligations");
});
test("truncated, temporary, and unreadable files are unverified, never clear", async () => {
const root = await fixture();
await write(root, "accounting-receipts/truncated.json", '{"schema":');
const file = await write(root, `quota-credential-recovery/${company}/${grant}.json.probe`);
await fs.chmod(file, 0);
const report = await inventoryRecoveryFiles({ instanceRoot: root });
assert.equal(report.status, "blocked");
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 0]);
});
test("unknown nesting and symlinks block without traversing outside the chosen roots", async () => {
const root = await fixture();
const outside = await fixture();
await write(outside, "do-not-read.json");
await fs.mkdir(path.join(root, "accounting-receipts"));
await fs.symlink(outside, path.join(root, "accounting-receipts", "link"));
await fs.mkdir(path.join(root, "quota-credential-recovery", company, "unexpected"), { recursive: true });
const report = await inventoryRecoveryFiles({ instanceRoot: root });
assert.equal(report.status, "unknown");
assert.deepEqual(report.directories.map(s => s.unknownEntries), [1, 1, 0]);
assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [0, 0, 0]);
await fs.symlink(outside, path.join(root, "data"));
assert.equal((await inventoryRecoveryFiles({ instanceRoot: root })).directories[2].status, "unknown");
await fs.symlink(outside, path.join(root, "instance-link"));
assert.equal((await inventoryRecoveryFiles({ instanceRoot: path.join(root, "instance-link") })).status, "unknown");
});
test("entry budget gives incomplete lower-bound counts and never clears omitted work", async () => {
const root = await fixture();
for (let i = 0; i < 4; i++) await write(root, `accounting-receipts/${i}.json`);
const report = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 2 });
assert.equal(report.status, "unknown");
assert.equal(report.directories[0].observedEntries, 2);
assert.equal(report.directories[0].complete, false);
assert.equal(report.directories[0].unverifiedFiles, 2);
const exact = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 4 });
assert.equal(exact.directories[0].complete, true);
});
test("CLI respects configured orphan override and explicit override precedence", async () => {
const root = await fixture();
const override = await fixture();
await write(override, "pending.json");
let result = run(root, ["--check"], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
assert.equal(result.status, 1);
assert.equal(JSON.parse(result.stdout).directories[2].unverifiedFiles, 1);
const emptyOverride = await fixture();
result = run(root, ["--check", "--orphan-directory", emptyOverride], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override });
assert.equal(result.status, 0);
assert.equal(JSON.parse(result.stdout).status, "clear");
});
test("fault boundary reports stat, directory read and close failures without error data", async () => {
const root = await fixture();
await fs.mkdir(path.join(root, "accounting-receipts"));
for (const operation of ["lstat", "opendir", "read", "close"]) {
const sensitiveError = Object.assign(new Error("secret-token-must-never-appear"), { code: "EACCES" });
const io = { ...fs };
if (operation === "lstat" || operation === "opendir") io[operation] = async () => { throw sensitiveError; };
else io.opendir = async () => ({
read: async () => { if (operation === "read") throw sensitiveError; return null; },
close: async () => { if (operation === "close") throw sensitiveError; },
});
const report = await inventoryRecoveryFiles({ instanceRoot: root }, io);
assert.equal(report.status, "unknown");
if (operation === "lstat") assert.equal(report.instanceStatus, "unreadable");
else assert.ok(report.directories[0].readFailures > 0);
assert.doesNotMatch(JSON.stringify(report), /secret-token|EACCES/);
}
});
test("CLI fails closed for missing instance, unsafe roots and bad arguments", async () => {
const root = await fixture();
assert.equal(cleanRun(path.join(root, "absent"), ["--check"]).status, 1);
for (const args of [["--max-entries", "0"], ["--max-entries", "100001"], ["--max-entries", "NaN"],
["--orphan-directory", "relative-secret"], ["--bad-secret"], ["--check", "--check"]]) {
const result = cleanRun(root, args);
assert.equal(result.status, 2);
assert.doesNotMatch(result.stdout + result.stderr, /relative-secret|bad-secret/);
}
});
scripts/run-vitest-stable.mjs
+1
View File
@@ -92,6 +92,7 @@ const nativeRunnerSuites = [
"server/src/__tests__/dot-runner.test.ts",
// These process cases otherwise skip in server shards without Runner binaries.
"server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts",
"server/src/services/native-runtime/native-usage-delivery.integration.test.ts",
];
// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable
// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip
server/scripts/migrate-run-evidence.ts
+32
View File
@@ -0,0 +1,32 @@
import { createDb } from "@paperclipai/db";
import { migrateRunEvidence } from "../src/services/run-evidence-migration.js";
if (!process.env.DATABASE_URL)
throw new Error("Set DATABASE_URL to the intended instance; run with legacy writers drained");
const args = process.argv.slice(2);
const flags = new Set(["--apply", "--company", "--after-run", "--after-trace", "--limit"]);
for (let index = 0; index < args.length; index++) {
const flag = args[index]!;
if (!flags.has(flag)) throw new Error(`Unknown migration option: ${flag}`);
if (flag !== "--apply" && (!args[++index] || args[index]!.startsWith("--"))) {
throw new Error(`Missing value for ${flag}`);
}
}
const value = (flag: string) => {
const i = args.indexOf(flag);
return i < 0 ? undefined : args[i + 1];
};
const db = createDb(process.env.DATABASE_URL);
try {
const result = await migrateRunEvidence(db, {
apply: args.includes("--apply"),
companyId: value("--company"),
afterRunId: value("--after-run"),
afterTraceId: value("--after-trace"),
limit: Number(value("--limit") || 50),
});
console.log(JSON.stringify(result, null, 2));
if (result.items.some((item) => item.state === "blocked")) process.exitCode = 1;
} finally {
await db.$client.end();
}
server/scripts/qualify-native-log-recovery.ts
+445
View File
@@ -0,0 +1,445 @@
import assert from "node:assert/strict";
import { randomUUID } from "node:crypto";
import { spawn, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { setTimeout as delay } from "node:timers/promises";
import { eq, sql } from "drizzle-orm";
import { createDb, heartbeatRuns } from "@paperclipai/db";
import {
S3Client,
CreateBucketCommand,
DeleteBucketCommand,
ListObjectsV2Command,
DeleteObjectsCommand,
} from "@aws-sdk/client-s3";
import {
DurablePrpControlPlane,
decodeDurablePrpCheckpoint,
} from "../src/vendor/paperclip-runner/index.js";
import { createS3StorageProvider } from "../src/storage/s3-provider.js";
import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
import { createObjectByteJournal } from "../src/services/object-byte-journal.js";
import { createHeartbeatRunOutput } from "../src/services/heartbeat/run-output.js";
import { createAgentIdentityRedactor } from "../src/services/agent-identity-redaction.js";
import { appendHeartbeatRunEvent } from "../src/services/heartbeat-run-events.js";
import { PaperclipControlPlanePort } from "../src/services/native-runtime/paperclip-control-plane-port.js";
import { formatNativeRunLogEvent } from "../src/services/native-runtime/native-run-log-export.js";
import { createNativeRunLogTestTables } from "../src/__tests__/helpers/native-run-log-database.js";
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
const databaseUrl = process.env.PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL;
if (
!endpoint ||
!databaseUrl ||
[endpoint, databaseUrl].some((url) => !["127.0.0.1", "localhost"].includes(new URL(url).hostname))
)
throw new Error("Qualification requires explicit disposable loopback S3 and Postgres endpoints");
const childMode = process.argv[2] === "--controller";
type Config = {
schema: string;
bucket: string;
checkpoint: string;
companyId: string;
agentId: string;
runId: string;
sessionId: string;
runnerInstanceId: string;
environmentLeaseId: string;
turnId: string;
itemId: string;
};
const config: Config = childMode
? JSON.parse(await fs.readFile(process.argv[3]!, "utf8"))
: {
schema: `native_log_proof_${randomUUID().replaceAll("-", "")}`,
bucket: `pc-native-proof-${randomUUID()}`,
checkpoint: "",
companyId: randomUUID(),
agentId: randomUUID(),
runId: randomUUID(),
sessionId: randomUUID(),
runnerInstanceId: randomUUID(),
environmentLeaseId: randomUUID(),
turnId: randomUUID(),
itemId: randomUUID(),
};
const db = createDb(databaseUrl, { maxConnections: 1, idleTimeoutSeconds: 0 });
await db.execute(sql`set client_min_messages=warning`);
const provider = createS3StorageProvider({
endpoint,
bucket: config.bucket,
region: "us-east-1",
forcePathStyle: true,
});
const logStore = () =>
createObjectRunLogStore({
provider,
keyPrefix: "logs",
flushMs: 30_000,
assertWriterAuthority: async (input) => {
await input.assertWriterAuthority?.();
},
});
const identity = {
runnerInstanceId: config.runnerInstanceId,
environmentLeaseId: config.environmentLeaseId,
runId: config.runId,
normalizedSessionId: config.sessionId,
turnId: config.turnId,
itemId: config.itemId,
};
const expectedRunnerVersion = "0.3.0",
expectedRunnerDigest = `sha256:${"c".repeat(64)}`;
async function waitFor<T>(read: () => Promise<T>, done: (value: T) => boolean, timeout = 30_000) {
const until = Date.now() + timeout;
while (Date.now() < until) {
const value = await read();
if (done(value)) return value;
await delay(40);
}
throw new Error("Native log qualification deadline exceeded");
}
if (childMode) {
const home = process.argv[4]!,
portNumber = Number(process.argv[5] || 0);
await db.execute(sql.raw(`set search_path=${config.schema}`));
if (portNumber) {
const records = new Map<string, string>(
JSON.parse(await fs.readFile(config.checkpoint, "utf8")),
);
decodeDurablePrpCheckpoint(records).restoreSessionDirectory(home);
}
const store = logStore();
const [row] = await db
.select({
id: heartbeatRuns.id,
companyId: heartbeatRuns.companyId,
agentId: heartbeatRuns.agentId,
status: heartbeatRuns.status,
runtimeMode: heartbeatRuns.runtimeMode,
logStore: heartbeatRuns.logStore,
logRef: heartbeatRuns.logRef,
logBytes: heartbeatRuns.logBytes,
lastOutputSeq: heartbeatRuns.lastOutputSeq,
lastOutputAt: heartbeatRuns.lastOutputAt,
})
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, config.runId));
type Run = typeof heartbeatRuns.$inferSelect;
const output = createHeartbeatRunOutput(db, {
run: row as Run,
runId: config.runId,
issueId: null,
runLogStore: store,
identityRedactor: createAgentIdentityRedactor(),
secretKeys: new Set(),
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
appendRunEvent: async (_run, event) => {
await appendHeartbeatRunEvent(db, {
companyId: config.companyId,
agentId: config.agentId,
runId: config.runId,
...event,
});
},
assertWriterAuthority: async () => {
const [current] = await db
.select({ status: heartbeatRuns.status })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, config.runId));
if (current?.status !== "running") throw new Error("fixture_writer_revoked");
},
});
const handlers = await output.begin(row as Run);
const record = async (event: Parameters<typeof formatNativeRunLogEvent>[0], cursor: number) => {
await handlers.onPersistedNativeLog!(cursor, formatNativeRunLogEvent(event));
};
const persistence = new PaperclipControlPlanePort(
db,
{
...config,
issueId: randomUUID(),
sourceInstanceId: config.runnerInstanceId,
controlPlaneSourceInstanceId: `control-${config.runId}`,
completionContractId: randomUUID(),
completionContractSha256: "qualification",
},
{ onCommittedEvent: record, onDuplicateEvent: record },
);
const core = new DurablePrpControlPlane({
stateDirectory: path.join(home, "control-plane"),
identity,
expectedRunnerVersion,
expectedRunnerDigest,
onCommittedEvent: async (event) => {
await persistence.appendEvent(event);
},
});
core.setCheckpointPersistence(async (checkpoint) => {
const temporary = `${config.checkpoint}.pending`;
await fs.writeFile(temporary, JSON.stringify([...checkpoint.records]));
await fs.rename(temporary, config.checkpoint);
});
await core.start(portNumber || undefined);
const ticket = core.issueBootstrapTicket();
await core.flushCheckpoint();
process.on("message", async (message: { id: string; type: string }) => {
try {
if (message.type === "checkpoint") await core.flushCheckpoint();
else if (message.type === "finish-log") {
const result = await store.finalize(output.handle!);
await output.flushOutputProgress({ force: true });
if (result.bytes === null) throw new Error("qualification_log_incomplete");
} else {
core.queueCommand(message.type, {}, message.id, true);
await waitFor(
async () => core.getCommand(message.id)?.status,
(status) => status === "completed",
);
}
process.send!({ id: message.id, ok: true });
} catch (error) {
process.send!({ id: message.id, error: String(error) });
}
});
process.send!({ ready: true, connectUrl: core.connectUrl, ticket });
} else {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "native-log-qualification-"));
config.checkpoint = path.join(root, "checkpoint.json");
const configFile = path.join(root, "config.json"),
receiptFile = path.join(root, "receipt.json"),
releaseFile = path.join(root, "release"),
scriptFile = path.join(root, "script.json");
const children: ChildProcess[] = [];
const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
let bucketCreated = false;
async function controller(home: string, portNumber = 0) {
await fs.mkdir(home, { mode: 0o700 });
const child = spawn(
process.execPath,
[
"--import",
path.resolve("server/node_modules/tsx/dist/loader.mjs"),
fileURLToPath(import.meta.url),
"--controller",
configFile,
home,
String(portNumber),
],
{
cwd: process.cwd(),
env: { ...process.env, PAPERCLIP_HOME: home },
stdio: ["ignore", "ignore", "pipe", "ipc"],
},
);
children.push(child);
let diagnostics = "";
child.stderr!.on("data", (bytes) => {
diagnostics = (diagnostics + String(bytes)).slice(-8000);
});
const ready = await new Promise<{ connectUrl: string; ticket: string }>((resolve, reject) => {
const timeout = setTimeout(
() => reject(new Error(`Controller startup timeout: ${diagnostics}`)),
45_000,
);
child.on("message", (message: { ready?: boolean; connectUrl: string; ticket: string }) => {
if (message.ready) {
clearTimeout(timeout);
resolve(message);
}
});
child.once("exit", () => {
clearTimeout(timeout);
reject(new Error(`Controller startup failed: ${diagnostics}`));
});
});
async function command(type: string) {
const id = randomUUID();
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
child.off("message", onMessage);
reject(new Error(`Command timeout: ${type}: ${diagnostics}`));
}, 35_000);
function onMessage(message: { id: string; error?: string }) {
if (message.id !== id) return;
clearTimeout(timer);
child.off("message", onMessage);
message.error ? reject(new Error(message.error)) : resolve();
}
child.on("message", onMessage);
child.send({ id, type });
});
}
return { child, ready, command };
}
try {
await db.execute(sql.raw(`create schema ${config.schema}`));
await db.execute(sql.raw(`set search_path=${config.schema}`));
await createNativeRunLogTestTables(db);
await db.execute(
sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${config.runId},${config.companyId},${config.agentId},'running')`,
);
await client.send(new CreateBucketCommand({ Bucket: config.bucket }));
bucketCreated = true;
await fs.writeFile(configFile, JSON.stringify(config));
await fs.writeFile(
scriptFile,
JSON.stringify({ steps: 64, waitAfter: 32, retentionLimit: 8, receiptFile, releaseFile }),
);
const homeA = path.join(root, "web-a"),
a = await controller(homeA);
const runner = spawn(
path.resolve("packages/paperclip-runner/runner/target/debug/examples/recovery-fixture"),
[
"--fixture-script",
scriptFile,
"--connect-url",
a.ready.connectUrl,
"--state-dir",
path.join(root, "runner"),
"--runner-id",
config.runnerInstanceId,
"--environment-lease-id",
config.environmentLeaseId,
"--run-id",
config.runId,
"--session-id",
config.sessionId,
"--turn-id",
config.turnId,
"--item-id",
config.itemId,
"--runner-version",
expectedRunnerVersion,
"--runner-digest",
expectedRunnerDigest,
"--max-outbox-bytes",
"65536",
"--p0-reserve-bytes",
"16384",
"--reconnect-grace-ms",
"60000",
],
{
env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: a.ready.ticket },
stdio: ["ignore", "ignore", "pipe"],
},
);
children.push(runner);
let runnerDiagnostics = "";
runner.stderr!.on("data", (bytes) => {
runnerDiagnostics = (runnerDiagnostics + String(bytes)).slice(-4000);
});
const receipt = async () =>
JSON.parse(await fs.readFile(receiptFile, "utf8").catch(() => "{}")) as {
pid: number;
acknowledgedEvents: number;
completedSteps: number;
retainedEvents: number;
peakRetainedEvents: number;
};
await a.command("turn.start");
await fs.writeFile(releaseFile, "go");
const acknowledged = await waitFor(
receipt,
(value) => value.acknowledgedEvents === 32 && value.retainedEvents === 0,
);
const journal = createObjectByteJournal({
provider,
prefix: `logs/segments-v1/${config.companyId}/${config.agentId}/${config.runId}`,
});
const before = (await journal.head())!.head;
assert(
before.sourceCursor < 32,
"Experiment must kill a controller with acknowledged output still pending in S3",
);
const controlStarted = performance.now();
await a.command("session.snapshot");
const controlMs = performance.now() - controlStarted;
await a.command("checkpoint");
const exitA = once(a.child, "exit");
a.child.kill("SIGKILL");
await exitA;
await fs.rm(homeA, { recursive: true });
const b = await controller(path.join(root, "web-b"), Number(new URL(a.ready.connectUrl).port));
await b.command("turn.steer");
const completed = await waitFor(
receipt,
(value) => value.acknowledgedEvents === 64 && value.retainedEvents === 0,
);
assert.equal(completed.pid, acknowledged.pid);
assert.equal(runner.exitCode, null, runnerDiagnostics);
const stopStarted = performance.now();
await b.command("turn.stop");
const stopMs = performance.now() - stopStarted;
await b.command("finish-log");
const store = logStore(),
content = (
await store.read({
store: "s3_segments",
logRef: `${config.companyId}/${config.agentId}/${config.runId}`,
})
).content;
const events = content
.trim()
.split("\n")
.map((line) => JSON.parse(JSON.parse(line).chunk).event);
assert.deepEqual(
events.map((event) => Number(event.payload.text.match(/^step=(\d+)/)[1])),
Array.from({ length: 64 }, (_, index) => index + 1),
);
assert.equal(new Set(events.map((event) => event.sourceEventId)).size, 64);
const [source] = await db.execute(
sql`select count(*)::int as count from heartbeat_run_events where run_id=${config.runId}`,
);
assert.equal(source!.count, 64);
const shutdown = once(runner, "exit");
await b.command("runner.shutdown");
assert.equal((await shutdown)[0], 0);
process.stdout.write(
`${JSON.stringify({
result: "passed",
source: "real Rust recovery fixture and PaperclipControlPlanePort",
acknowledgedBeforeKill: 32,
objectSourceCursorBeforeKill: before.sourceCursor,
controllerLoss: "SIGKILL",
oldDirectoryRemoved: true,
sameRunnerProcess: true,
recoveredRecords: events.length,
duplicates: 0,
existingControlRows: source!.count,
controlCommandMs: Math.round(controlMs),
stopMs: Math.round(stopMs),
peakRunnerRetained: completed.peakRetainedEvents,
})}\n`,
);
} finally {
for (const child of children)
if (child.exitCode === null && child.signalCode === null) {
const exit = once(child, "exit");
child.kill("SIGKILL");
await exit;
}
if (bucketCreated) {
for (;;) {
const page = await client.send(new ListObjectsV2Command({ Bucket: config.bucket }));
if (!page.Contents?.length) break;
await client.send(
new DeleteObjectsCommand({
Bucket: config.bucket,
Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: config.bucket }));
}
client.destroy();
await db.execute(sql.raw(`drop schema if exists ${config.schema} cascade`));
await db.$client.end();
await fs.rm(root, { recursive: true, force: true });
}
}
server/scripts/qualify-object-run-logs.ts
+213
View File
@@ -0,0 +1,213 @@
import { randomUUID } from "node:crypto";
import { spawn } from "node:child_process";
import { once } from "node:events";
import { promises as fs } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import {
S3Client,
CreateBucketCommand,
DeleteBucketCommand,
ListObjectsV2Command,
DeleteObjectsCommand,
} from "@aws-sdk/client-s3";
import { createS3StorageProvider } from "../src/storage/s3-provider.js";
import { createObjectRunLogStore } from "../src/services/object-run-log-store.js";
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
if (!endpoint || !["127.0.0.1", "localhost"].includes(new URL(endpoint).hostname)) {
throw new Error(
"Qualification requires PAPERCLIP_LOG_TEST_S3_ENDPOINT on a disposable loopback backend",
);
}
const childMode = process.argv[2] === "--child-writer";
const bucket = childMode ? process.argv[3]! : `paperclip-logs-test-${randomUUID()}`;
const raw = createS3StorageProvider({
endpoint,
bucket,
region: "us-east-1",
forcePathStyle: true,
});
const counts = { get: 0, put: 0, uploadedBytes: 0 };
const latencies: number[] = [];
const provider = {
...raw,
async getObject(...args: Parameters<typeof raw.getObject>) {
counts.get++;
return raw.getObject(...args);
},
async putObjectConditional(...args: Parameters<NonNullable<typeof raw.putObjectConditional>>) {
counts.put++;
counts.uploadedBytes += args[0].contentLength;
const start = performance.now();
try {
return await raw.putObjectConditional!(...args);
} finally {
latencies.push(performance.now() - start);
}
},
};
const input = { companyId: "company", agentId: "agent", runId: "crash" };
const options = {
provider,
keyPrefix: "qualification",
flushMs: 1000,
assertWriterAuthority: async () => undefined,
};
const output = (seq: number) => ({
stream: "stdout" as const,
seq,
chunk: `${seq}: 再開😀 ${"x".repeat(900)}`,
ts: "2026-10-30T00:00:00Z",
});
if (childMode) {
const store = createObjectRunLogStore(options),
handle = await store.begin(input);
const before = counts.put;
await Promise.all(Array.from({ length: 1000 }, (_, i) => store.append(handle, output(i + 1))));
process.stdout.write(
JSON.stringify({ receipt: true, batchPuts: counts.put - before, counts }) + "\n",
);
setTimeout(() => process.exit(2), 30_000);
} else {
const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
await client.send(new CreateBucketCommand({ Bucket: bucket }));
const directory = await fs.mkdtemp(path.join(tmpdir(), "run-log-crash-"));
let child: ReturnType<typeof spawn> | undefined;
try {
const started = performance.now();
child = spawn(
process.execPath,
[
"--import",
path.resolve("server/node_modules/tsx/dist/loader.mjs"),
import.meta.filename,
"--child-writer",
bucket,
],
{ cwd: directory, env: process.env, stdio: ["ignore", "pipe", "pipe"] },
);
const exited = once(child, "exit");
let childError = "";
child.stderr!.on("data", (bytes) => {
childError = (childError + bytes).slice(-8192);
});
const receipt = await new Promise<{ batchPuts: number; counts: typeof counts }>(
(resolve, reject) => {
let text = "";
const timeout = setTimeout(
() => reject(new Error(`Writer receipt timed out: ${childError}`)),
30_000,
);
child!.stdout!.on("data", (bytes) => {
text += bytes;
if (!text.includes("\n")) return;
clearTimeout(timeout);
try {
resolve(JSON.parse(text.split("\n")[0]!));
} catch (error) {
reject(error);
}
});
child!.on("error", (error) => {
clearTimeout(timeout);
reject(error);
});
void exited.then(() => {
clearTimeout(timeout);
if (!text) reject(new Error(`Writer exited before its receipt: ${childError}`));
});
},
);
child.kill("SIGKILL");
const [, signal] = await exited;
if (signal !== "SIGKILL") throw new Error("Writer did not die by SIGKILL");
await fs.rm(directory, { recursive: true, force: true });
const successorStore = createObjectRunLogStore(options);
const successor = await successorStore.begin(input);
let content = "",
offset = 0;
for (;;) {
const page = await successorStore.read(successor, { offset, limitBytes: 131072 });
content += page.content;
if (page.nextOffset === undefined) break;
offset = page.nextOffset;
}
const lines = content
.trim()
.split("\n")
.map((line) => JSON.parse(line));
if (
lines.length !== 1000 ||
!lines.every((line, i) => line.seq === i + 1 && line.chunk === output(i + 1).chunk)
)
throw new Error("Replacement bytes did not match");
await successorStore.append(successor, output(1001));
const summary = await successorStore.finalize(successor);
const steadyBefore = { ...counts },
steadyStarted = performance.now();
const steady = createObjectRunLogStore(options);
const handles = await Promise.all(
Array.from({ length: 8 }, (_, i) => steady.begin({ ...input, runId: `steady-${i}` })),
);
const pending: Promise<number>[] = [];
for (let tick = 0; tick < 30; tick++) {
for (const handle of handles) pending.push(steady.append(handle, output(tick + 1)));
await delay(100);
}
await Promise.all(pending);
await Promise.all(handles.map((handle) => steady.finalize(handle)));
const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
if (objects.IsTruncated) throw new Error("Qualification inventory exceeded one bounded page");
latencies.sort((a, b) => a - b);
console.log(
JSON.stringify(
{
backend: "loopback S3-compatible test service",
flushMs: options.flushMs,
burst: {
records: lines.length,
batchPuts: receipt.batchPuts,
writerCounts: receipt.counts,
},
steady: {
streams: handles.length,
records: pending.length,
elapsedMs: Math.round(performance.now() - steadyStarted),
putRequests: counts.put - steadyBefore.put,
getRequests: counts.get - steadyBefore.get,
uploadedBytes: counts.uploadedBytes - steadyBefore.uploadedBytes,
},
readerCounts: counts,
committedCrashBytes: summary.bytes,
objects: objects.KeyCount,
storedBytes: objects.Contents?.reduce((sum, object) => sum + (object.Size ?? 0), 0),
putP95Ms: Math.round(latencies[Math.floor(latencies.length * 0.95)] ?? 0),
elapsedMs: Math.round(performance.now() - started),
abruptProcessLoss: signal,
oldDirectoryRemoved: true,
replacement: "passed",
},
null,
2,
),
);
} finally {
child?.kill("SIGKILL");
await fs.rm(directory, { recursive: true, force: true });
for (;;) {
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket, MaxKeys: 1000 }));
if (!page.Contents?.length) break;
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) },
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
client.destroy();
}
}
server/src/__tests__/activity-routes.test.ts
+14
View File
@@ -333,6 +333,20 @@ describe("activity routes", () => {
expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]);
});
it("validates bounded run IDs before using the authorized issue projection", async () => {
const runId = "10000000-0000-4000-8000-000000000001";
mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" });
mockActivityService.runsForIssue.mockResolvedValue([{ runId }]);
const app = await createApp();
const response = await requestApp(app, base => request(base).get(`/api/issues/PAP-1/runs?runIds=${runId}`));
expect(response.status).toBe(200);
expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1", undefined, [runId]);
for (const ids of ["", "not-a-uuid", Array.from({ length: 101 }, () => runId).join(",")]) {
const denied = await requestApp(app, base => request(base).get("/api/issues/PAP-1/runs").query({ runIds: ids }));
expect(denied.status).toBe(400);
}
});
it("requires company access before creating activity events", async () => {
const app = await createApp();
const res = await requestApp(app, (baseUrl) => request(baseUrl)
server/src/__tests__/agent-identity.test.ts
+7 -2
View File
@@ -10,6 +10,7 @@ import { eq } from "drizzle-orm";
import { agentIdentityKeys, agents, companies, createDb, closeRegisteredClients, ensurePostgresDatabase, runDatabaseBackup, runDatabaseRestore, type Db } from "@paperclipai/db";
import { resolveWorktreeSeedPlan } from "../../../cli/src/commands/worktree-lib.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { agentIdentityService, supportsManagedAgentIdentity } from "../services/agent-identity.js";
import { agentService } from "../services/agents.js";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
@@ -21,9 +22,10 @@ import { actorMiddleware } from "../middleware/auth.js";
import { errorHandler } from "../middleware/error-handler.js";
import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js";
const externalIdentityDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_IDENTITY_TEST_DATABASE_URL");
describe("agent cryptographic identity", () => {
let db: Db;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let database: Pick<Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, "connectionString" | "cleanup">;
let directory: string;
let companyId: string;
@@ -32,7 +34,9 @@ describe("agent cryptographic identity", () => {
vi.stubEnv("PAPERCLIP_HOME", directory);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "");
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key"));
database = await startEmbeddedPostgresTestDatabase("paperclip-identity-");
database = externalIdentityDatabase ? { connectionString: externalIdentityDatabase,
cleanup: () => closeRegisteredClients(externalIdentityDatabase) }
: await startEmbeddedPostgresTestDatabase("paperclip-identity-");
db = createDb(database.connectionString);
companyId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Identity test", issuePrefix: "KEY", defaultResponsibleUserId: "identity-test-owner" });
@@ -51,6 +55,7 @@ describe("agent cryptographic identity", () => {
it("migration and public reads leave existing agents unprovisioned", async () => {
const agent = await oldAgent();
expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toBeNull();
expect((await agentIdentityService(db).createExistingLogRedactor(companyId, agent.id)).values).toEqual([]);
expect(await db.select().from(agentIdentityKeys).where(eq(agentIdentityKeys.agentId, agent.id))).toHaveLength(0);
for (const type of ["http", "openclaw_gateway", "hermes_gateway"]) expect(supportsManagedAgentIdentity(type)).toBe(false);
for (const provider of ["claude_managed", "aws_agentcore"]) expect(supportsManagedAgentIdentity("paperclip_runner", { provider })).toBe(false);
server/src/__tests__/ai-connections.test.ts
+154 -34
View File
@@ -9,7 +9,8 @@ vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConne
import * as localCredentials from "../services/local-ai-credentials.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { createHash, randomUUID } from "node:crypto";
import { spawnSync } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import { createRequire } from "node:module";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { promises as fs } from "node:fs";
import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:fs/promises";
@@ -38,6 +39,8 @@ import { secretService } from "../services/secrets.js";
import { resolveExecutionRunAdapterConfig } from "../services/heartbeat.js";
import { aiConnectionBindingSchema, connectionPurposeTransportSchema, isAiConnectionCompatible } from "@paperclipai/shared";
import { openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution";
import { createServer } from "node:http";
import { once } from "node:events";
import express from "express";
import request from "supertest";
import { aiConnectionRoutes, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "../routes/ai-connections.js";
@@ -385,8 +388,8 @@ describe("managed AI connections", () => {
await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable");
});
it("persists first-time recovery directories before consuming a single-use token", async () => {
const owner = "quota-durable-path";
it("commits one exchanged token without writable credential recovery storage", async () => {
const owner = "quota-database-first";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { account_id: owner, access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
@@ -394,32 +397,32 @@ describe("managed AI connections", () => {
const file = quotaCredentialRecoveryPath(companyId, row.grant.id);
const recoveryRoot = path.dirname(path.dirname(file));
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
const synced: string[] = [];
let rejectRoot = true;
const realOpen = fs.open.bind(fs);
const writes: string[] = [];
const rejectRecoveryWrite = (target: unknown) => {
if (String(target).startsWith(recoveryRoot)) {
writes.push(String(target));
throw Object.assign(new Error("Fixture recovery storage is read-only"), { code: "EROFS" });
}
};
const realOpen = fs.open.bind(fs), realMkdir = fs.mkdir.bind(fs);
const open = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const handle = await realOpen(...args);
const sync = handle.sync.bind(handle);
vi.spyOn(handle, "sync").mockImplementation(async () => {
if (rejectRoot && args[0] === recoveryRoot) throw new Error("Injected recovery directory flush failure");
await sync(); synced.push(String(args[0]));
});
return handle;
rejectRecoveryWrite(args[0]);
return realOpen(...args);
});
const request = vi.spyOn(globalThis, "fetch").mockImplementation(async () => {
expect(synced).toContain(recoveryRoot);
expect(synced).toContain(path.dirname(recoveryRoot));
return Response.json({ access_token: "fresh-durable", refresh_token: "replacement-durable" });
const mkdir = vi.spyOn(fs, "mkdir").mockImplementation(async (...args: Parameters<typeof fs.mkdir>) => {
rejectRecoveryWrite(args[0]);
return realMkdir(...args);
});
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "fresh-direct", refresh_token: "replacement-direct" }));
try {
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow("Injected recovery directory flush failure");
expect(request).not.toHaveBeenCalled();
expect(await service.credential(row)).toBe(original);
rejectRoot = false;
await service.refreshQuotaCredential(row, original, new AbortController().signal);
const refreshed = await service.refreshQuotaCredential(row, original, new AbortController().signal);
expect(JSON.parse(refreshed)).toMatchObject({ tokens: { access_token: "fresh-direct", refresh_token: "replacement-direct" } });
expect(await service.credential(row)).toBe(refreshed);
expect(request).toHaveBeenCalledTimes(1);
expect(JSON.parse(await service.credential(row))).toMatchObject({ tokens: { access_token: "fresh-durable", refresh_token: "replacement-durable" } });
} finally { open.mockRestore(); request.mockRestore(); }
expect(writes).toEqual([]);
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" });
} finally { open.mockRestore(); mkdir.mockRestore(); request.mockRestore(); }
});
it.each(["timeout", "unexpected"] as const)("classifies a manual probe's %s lock failure before starting the provider", async (kind) => {
@@ -703,7 +706,7 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); }
});
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("recovers exchanged OAuth tokens after %s persistence failure without another exchange", async (failure) => {
it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("fails truthfully after %s persistence failure and permits sign-in recovery without a journal", async (failure) => {
const owner = `quota-rollback-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: `expired-${failure}`, refresh_token: "single-use", account_id: owner } });
@@ -723,17 +726,25 @@ describe("managed AI connections", () => {
await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
expect(request).toHaveBeenCalledTimes(1);
expect(await service.credential(row)).toBe(original);
const encrypted = await readFile(file, "utf8");
expect(encrypted).not.toContain(`fresh-${failure}`);
expect(encrypted).not.toContain(`replacement-${failure}`);
await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" });
await db.execute(sql.raw(`drop trigger quota_save_failure on ${table}; drop function quota_save_failure();`));
triggerDropped = true;
// A new service instance recovers from disk, without an in-memory cache.
const recovered = aiConnectionService(db);
await recovered.refreshQuotaCredential(row, original, new AbortController().signal);
expect(JSON.parse(await recovered.credential(row))).toMatchObject({ tokens: { access_token: `fresh-${failure}`, refresh_token: `replacement-${failure}` } });
// A new receiver has no replacement to recover. A failed use routes to the
// existing sign-in flow, rather than claiming the failed save succeeded.
const receiver = aiConnectionService(db);
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
} });
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
const signedIn = JSON.stringify({ tokens: { access_token: "signed-in", refresh_token: "new-login" } });
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription",
ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
const [current] = await receiver.quotaAccounts(companyId, owner);
expect(current.grant.status).toBe("active");
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
expect(request).toHaveBeenCalledTimes(1);
await expect(access(file)).rejects.toThrow();
expect((await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, ref.secretId))).map(version => version.version).sort()).toEqual([1, 2]);
} finally {
request.mockRestore();
@@ -819,6 +830,112 @@ describe("managed AI connections", () => {
} finally { request.mockRestore(); await oldRun.cleanup(); await nextRun?.cleanup(); await journal.clear(); }
});
it("requires sign-in after the receiver process dies with an exchanged but unsaved token", async () => {
const owner = "quota-receiver-process-loss";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired-process", refresh_token: "single-use-process" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
const held = deferredSignal(), release = deferredSignal();
const lockKey = 7192031;
const holder = db.transaction(async tx => {
await tx.execute(sql`select pg_advisory_xact_lock(${lockKey})`);
held.resolve(); await release.promise;
});
await held.promise;
await db.execute(sql.raw(`create function hold_receiver_save() returns trigger language plpgsql as $$ begin
if new.secret_id = '${ref.secretId}'::uuid then perform pg_advisory_xact_lock(${lockKey}); end if; return new; end $$;
create trigger hold_receiver_save before insert on company_secret_versions for each row execute function hold_receiver_save();`));
let exchanges = 0;
const provider = createServer((req, res) => {
exchanges++; req.resume();
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ access_token: "unsaved-process", refresh_token: "replacement-process" }));
});
provider.listen(0, "127.0.0.1"); await once(provider, "listening");
const address = provider.address() as { port: number };
const child = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), new URL("./helpers/quota-credential-process.ts", import.meta.url).pathname], {
env: { ...process.env, CREDENTIAL_TEST_DATABASE_URL: database.connectionString,
CREDENTIAL_TEST_PROVIDER_URL: `http://127.0.0.1:${address.port}`, CREDENTIAL_TEST_COMPANY_ID: companyId, CREDENTIAL_TEST_OWNER: owner },
stdio: "ignore",
});
const exited = once(child, "exit");
try {
let blocked = false;
const deadline = Date.now() + 25_000;
while (!blocked && Date.now() < deadline && child.exitCode === null) {
const rows = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and wait_event = 'advisory'`);
blocked = exchanges === 1 && rows.length > 0;
if (!blocked) await new Promise(resolve => setTimeout(resolve, 25));
}
expect(blocked).toBe(true);
child.kill("SIGKILL"); await exited;
} finally {
if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); await exited; }
release.resolve(); await holder;
await db.execute(sql`drop trigger hold_receiver_save on company_secret_versions; drop function hold_receiver_save()`);
await new Promise<void>(resolve => provider.close(() => resolve()));
}
expect(exchanges).toBe(1);
expect(await service.credential(row)).toBe(original);
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
const receiver = aiConnectionService(db);
const generation = createHash("sha256").update(original).digest("hex").slice(0, 16);
await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: {
provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id,
mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`,
} });
expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization");
const signedIn = JSON.stringify({ tokens: { access_token: "process-sign-in", refresh_token: "process-new-login" } });
await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn);
const [current] = await receiver.quotaAccounts(companyId, owner);
expect(await receiver.runtimeCredential(current)).toBe(signedIn);
}, 40_000);
it("does not exchange a token when a required database transaction fails", async () => {
const owner = "quota-db-prerequisite";
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const request = vi.spyOn(globalThis, "fetch");
const unavailableUrl = new URL(database.connectionString);
unavailableUrl.port = "1";
const unavailable = createDb(unavailableUrl.toString());
try {
await expect(aiConnectionService(unavailable).refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow();
expect(request).not.toHaveBeenCalled();
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
} finally { await unavailable.$client.end(); request.mockRestore(); }
});
it.each(["reconnect", "same_value_new_epoch", "revoked"] as const)("lets %s win after a returned token's save rolls back", async change => {
const owner = `quota-delayed-save-${change}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } });
await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original);
const [row] = await service.quotaAccounts(companyId, owner);
const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!;
const transaction = db.transaction.bind(db);
const interrupted = vi.spyOn(db, "transaction").mockImplementationOnce(async (work, config) => {
await transaction(async tx => { await work(tx); throw new Error("fixture rollback"); }, config).catch(() => {});
if (change === "revoked") await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, row.grant.id));
else await secretService(db).rotate(ref.secretId, { value: change === "reconnect" ? "new-login" : original });
throw new Error("fixture lost response");
});
const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "late", refresh_token: "late-refresh" }));
try {
const refresh = service.refreshQuotaCredential(row, original, new AbortController().signal);
if (change === "revoked") await expect(refresh).rejects.toThrow("credentials_unavailable");
else expect(await refresh).toBe(change === "reconnect" ? "new-login" : original);
expect(await service.credential(row)).toBe(change === "reconnect" ? "new-login" : original);
expect(request).toHaveBeenCalledTimes(1);
await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" });
} finally { interrupted.mockRestore(); request.mockRestore(); }
});
it.each(["rollback", "lost_commit_reply"] as const)("automatically saves one exchanged token across %s", async (failure) => {
const owner = `quota-once-${failure}`;
await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" });
@@ -863,7 +980,10 @@ describe("managed AI connections", () => {
triggerInstalled = true;
await expect(service.refreshQuotaCredential(row, current, new AbortController().signal)).rejects.toThrow();
expect(await service.credential(row)).toBe(current);
expect(JSON.parse((await journal.read())!.value).tokens.access_token).toBe("C");
expect(await journal.read()).toBeNull();
// Only observed records from the earlier release use the legacy reader.
await journal.write({ companyId, grantId: row.grant.id, connectionId: row.connection.id, secretId: ref.secretId,
baseHash: quotaCredentialHash(current), value: JSON.stringify({ tokens: { access_token: "C", refresh_token: "refresh-C" } }) });
await db.execute(sql`drop trigger delayed_quota_commit_failure on company_secret_versions; drop function delayed_quota_commit_failure()`);
triggerInstalled = false;
const recovered = await service.refreshQuotaCredential(row, original, new AbortController().signal);
server/src/__tests__/heartbeat-run-privacy-routes.test.ts
+9 -5
View File
@@ -32,8 +32,10 @@ import { activityRoutes } from "../routes/activity.js";
import { agentRoutes } from "../routes/agents.js";
import { authorizationService } from "../services/authorization.js";
import { canActorReadWorkspaceOperation, getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js";
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL");
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, () => {
@@ -42,8 +44,8 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
const previousPrivacyMode = process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
beforeAll(async () => {
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
db = createDb(tempDb.connectionString);
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-");
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString);
process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce";
}, 120_000);
@@ -66,6 +68,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
if (previousPrivacyMode === undefined) delete process.env.PAPERCLIP_ISSUE_PRIVACY_MODE;
else process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = previousPrivacyMode;
await tempDb?.cleanup();
if (externalDatabaseUrl) await db.$client.end({ timeout: 1 });
});
async function seedFixture() {
@@ -264,7 +267,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
runnerProfileJson: { instructions: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
}).where(eq(heartbeatRuns.id, f.privateRunId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
@@ -281,8 +284,9 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
} finally { capture.flush(); await observed.$client.end(); }
});
it("returns 404 for direct private-run content and metadata-only list stubs to a non-member", async () => {
it.each(["local_file", "s3_segments"])("returns 404 for private %s run content and metadata-only list stubs", async (logStore) => {
const fixture = await seedFixture();
await db.update(heartbeatRuns).set({ logStore }).where(eq(heartbeatRuns.id, fixture.privateRunId));
const app = createApp(fixture.companyId, fixture.otherAgentId);
for (const path of [
server/src/__tests__/helpers/native-run-log-database.ts
+22
View File
@@ -0,0 +1,22 @@
import { sql } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
export async function createNativeRunLogTestTables(db: Db) {
await db.execute(sql`create table heartbeat_runs(id uuid primary key, company_id uuid not null, agent_id uuid not null,
status text not null, runtime_mode text not null default 'native', native_session_id text, runner_instance_id text,
native_issue_id uuid, issue_id uuid, context_snapshot jsonb,
log_store text, log_ref text, log_bytes bigint,
log_sha256 text, last_output_seq bigint not null default 0, last_output_at timestamptz, last_output_stream text,
last_output_bytes bigint, next_event_seq bigint not null default 1, controller_lease_expires_at timestamptz,
created_at timestamptz not null default now(), updated_at timestamptz not null default now())`);
await db.execute(sql`create table heartbeat_run_events(id bigserial primary key, company_id uuid not null,
run_id uuid not null, agent_id uuid not null, seq bigint not null, event_type text not null, stream text,
level text, color text, message text, payload jsonb, source_instance_id text, source_event_id text,
source_seq bigint, source_payload_sha256 text, protocol_schema_version int, created_at timestamptz not null default now(),
unique(run_id,seq), unique(run_id,source_event_id), unique(run_id,source_instance_id,source_seq))`);
await db.execute(
sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`,
);
await db.execute(sql`create table agent_identity_keys(agent_id uuid primary key,company_id uuid,algorithm text,key_id text,
public_key_pem text,private_key_material jsonb,created_at timestamptz default now())`);
}
server/src/__tests__/helpers/quota-credential-process.ts
+19
View File
@@ -0,0 +1,19 @@
// A real receiver process for the credential-loss boundary test. No provider
// credentials leave this process through logs, IPC, or diagnostic exceptions.
import { createDb } from "@paperclipai/db";
import { aiConnectionService } from "../../services/ai-connections.js";
const db = createDb(process.env.CREDENTIAL_TEST_DATABASE_URL!);
const provider = process.env.CREDENTIAL_TEST_PROVIDER_URL!;
const request = globalThis.fetch;
globalThis.fetch = (url, init) => {
if (String(url) !== "https://auth.openai.com/oauth/token") throw new Error("Unexpected fixture network request");
return request(provider, init);
};
try {
const service = aiConnectionService(db);
const [row] = await service.quotaAccounts(process.env.CREDENTIAL_TEST_COMPANY_ID!, process.env.CREDENTIAL_TEST_OWNER!);
await service.refreshQuotaCredential(row, await service.credential(row), new AbortController().signal);
process.exitCode = 0;
} catch { process.exitCode = 1; }
finally { await db.$client.end(); }
server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts
+1
View File
@@ -1090,6 +1090,7 @@ describe("agent issue mutation checkout ownership", () => {
]);
expect(mockAccessService.decide).toHaveBeenCalledWith(expect.objectContaining({ action: "issue:read" }));
expect(mockIssueService.listComments).toHaveBeenCalledWith(issueId, {
companyId,
afterCommentId: null,
order: "desc",
limit: null,
server/src/__tests__/issue-scheduled-retry-routes.test.ts
+22 -5
View File
@@ -580,16 +580,33 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
.where(inArray(heartbeatRuns.id, work.map(row => row.id)));
clock();
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } });
const register = coordinator.register.bind(coordinator);
let initial: Awaited<ReturnType<typeof states>> | undefined;
let releaseFirstPass!: () => void;
const firstPassHeld = new Promise<void>(resolve => { releaseFirstPass = resolve; });
vi.spyOn(coordinator, "register").mockImplementation((queue, task) => register(queue, {
...task,
run: async signal => {
const result = await task.run(signal);
if (initial) return result;
const cancelled = (await states()).filter(row => row.status === "cancelled");
if (cancelled.length === 0) return result;
initial = cancelled;
await firstPassHeld;
return result;
},
}));
try {
await heartbeatService(db, { runtimeEnv: {} }).registerScheduledRetryCancellationWork(coordinator, () => true).ready;
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(0);
const earlierId = "00000000-0000-4000-8000-000000000000";
await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
work.push({ id: earlierId });
await writeIntent({ companyId: a.companyId, retryRunId: earlierId });
await vi.advanceTimersByTimeAsync(1000);
await expect.poll(async () => (await states()).filter(row => row.status === "cancelled").length).toBe(20);
const initial = (await states()).filter(row => row.status === "cancelled");
expect(new Set(initial.map(row => row.companyId)).size).toBe(2);
await expect.poll(() => initial).toBeDefined();
expect(initial).toHaveLength(20);
expect(new Set(initial!.map(row => row.companyId)).size).toBe(2);
releaseFirstPass();
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(29);
@@ -600,7 +617,7 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => {
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(180_000);
expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
} finally { await coordinator.stop(); }
} finally { releaseFirstPass(); await coordinator.stop(); }
});
});
server/src/__tests__/issues-service.test.ts
+30
View File
@@ -48,6 +48,7 @@ import {
readIssueCommentRunLogText,
} from "../services/issues.ts";
import { getRunLogStore } from "../services/run-log-store.js";
import * as nativeRunLogRecovery from "../services/native-runtime/native-run-log-recovery.js";
import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
import {
@@ -192,6 +193,35 @@ describeEmbeddedPostgres("issueService run attachment artifacts", () => {
});
describe("readIssueCommentRunLogText", () => {
it("does not read after a timed-out native repair", async () => {
const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockImplementationOnce((_db, _store, _ref, _redaction, signal) => new Promise((_resolve, reject) => {
signal!.addEventListener("abort", () => reject(signal!.reason), { once: true });
}));
const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValue({ content: "too late" });
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
const pending = readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType<typeof createDb>);
try {
await vi.advanceTimersByTimeAsync(3_000);
await expect(pending).resolves.toBe("");
expect(read).not.toHaveBeenCalled();
} finally {
await pending;
vi.useRealTimers();
repair.mockRestore();
read.mockRestore();
}
});
it("keeps published evidence when a concurrent native repair wins ownership", async () => {
const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockRejectedValueOnce(new Error("The output journal is owned by another writer"));
const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValueOnce({ content: "published evidence" });
try {
await expect(readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType<typeof createDb>)).resolves.toBe("published evidence");
expect(read).toHaveBeenCalledOnce();
} finally {
repair.mockRestore();
read.mockRestore();
}
});
it("cancels timed-out storage reads so later listings can recover", async () => {
let active = 0;
const cleanups: Array<() => void> = [];
server/src/__tests__/local-encrypted-cipher.test.ts
+14 -2
View File
@@ -1,9 +1,9 @@
import { randomBytes } from "node:crypto";
import { mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createLocalEncryptedCipher, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret, localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
describe("operation-scoped local encryption", () => {
let root: string;
@@ -60,4 +60,16 @@ describe("operation-scoped local encryption", () => {
code: "provider_unavailable", operation: "resolveVersion",
}));
});
it("derives the same instance secret from an existing key file or environment without creating missing keys", () => {
expect(() => deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false }))
.toThrow("Secrets master key file is missing");
expect(existsSync(keyFile)).toBe(false);
const key = randomBytes(32).toString("hex");
writeFileSync(keyFile, key, { mode: 0o600 });
const fromFile = deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false });
unlinkSync(keyFile);
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key);
expect(deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false })).toBe(fromFile);
expect(existsSync(keyFile)).toBe(false);
});
});
server/src/__tests__/native-harness-backup-expiration.test.ts
+7 -2
View File
@@ -645,8 +645,13 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
const available = await seedArchive(healthy.root, { name: `${dueAt - 7 * DAY_MS - 1}-${randomUUID()}` });
await startEvidence({ isSessionBusy: scope => scope === busy.scope }); await settled(dueAt);
await vi.advanceTimersByTimeAsync(dueAt - Date.now());
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(await exists(available)).toBe(false); });
expect(await exists(held)).toBe(true); expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(100);
expect(await exists(available)).toBe(false);
expect(idleWorkSnapshot().active).toBe(0);
expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000);
});
expect(await exists(held)).toBe(true);
await quiet(60_000);
});
server/src/__tests__/plugin-task-credential-writeback.test.ts
+21 -1
View File
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { mkdtemp, realpath, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { and, eq } from "drizzle-orm";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -90,6 +90,26 @@ suite("durable plugin task credential handoff", () => {
expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
expect(f.calls.at(-1)).toBe("complete");
});
it("retains the Runner credential after a database save failure until a fresh receiver commits", async () => {
const f = await fixture();
await db.execute(sql.raw(`create function reject_runner_credential_save() returns trigger language plpgsql as $$ begin
if new.secret_id = '${f.managed.credentialWriteback!.secretId}'::uuid then raise exception 'fixture receiver save unavailable'; end if; return new; end $$;
create constraint trigger reject_runner_credential_save after insert on company_secret_versions
deferrable initially deferred for each row execute function reject_runner_credential_save();`));
try {
await expect(f.settle()).rejects.toThrow();
expect(f.calls).toContain("credentials");
expect(f.calls).not.toContain("complete");
expect(await f.credential()).toBe(auth("fixture-original"));
expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "pending" });
} finally { await db.execute(sql`drop trigger reject_runner_credential_save on company_secret_versions; drop function reject_runner_credential_save()`); }
const receiver = createDb(database.connectionString);
expect(await settleDispatchedEnvironmentTask(receiver, f.workers, { companyId: f.companyId, id: f.leaseId })).toBe(true);
expect(await f.credential()).toBe(auth("fixture-refreshed", 2021));
expect(f.calls.filter(call => call === "credentials")).toHaveLength(2);
expect(f.calls.at(-1)).toBe("complete");
expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "complete" });
});
it("does not overwrite an explicit account replacement", async () => {
const f = await fixture();
await secretService(db).rotate(f.managed.credentialWriteback!.secretId, { value: auth("fixture-reconnected", 2022) }, { userId: f.owner });
server/src/__tests__/provider-trace-retention-work.test.ts
+66
View File
@@ -0,0 +1,66 @@
import type { Db } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { afterEach, expect, it, vi } from "vitest";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { registerProviderTraceRetentionWork } from "../services/provider-trace-retention-work.js";
const state = vi.hoisted(() => ({
cleanup: vi.fn(async () => []),
next: vi.fn<() => Promise<number | null>>(),
probe: undefined as undefined | (() => string),
unregister: vi.fn(),
}));
vi.mock("../services/provider-trace-store.js", () => ({
providerTraceStore: () => ({ cleanupExpired: state.cleanup, nextRetentionDeadline: state.next }),
}));
vi.mock("../services/idle-local-work.js", () => ({
registerIdleLocalWorkProbe: (probe: () => string) => {
state.probe = probe;
return state.unregister;
},
}));
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
afterEach(async () => {
await coordinator?.stop();
vi.useRealTimers();
vi.clearAllMocks();
});
it("reconstructs trace deadlines, stays quiet between obligations, and discharges the sleep blocker", async () => {
vi.useFakeTimers();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const onError = vi.fn();
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError });
state.next.mockResolvedValue(null);
const worker = registerProviderTraceRetentionWork(
owner as unknown as Db,
coordinator,
() => true,
);
expect(state.probe!()).toBe("unknown");
await worker.ready;
expect(state.probe!()).toBe("none");
expect(coordinator.nextWakeAt()).toBeNull();
const initial = state.cleanup.mock.calls.length;
await vi.advanceTimersByTimeAsync(300_000);
expect(state.cleanup).toHaveBeenCalledTimes(initial);
expect(state.next).toHaveBeenCalledOnce();
const deadline = Date.now() + 60_000;
state.next.mockResolvedValue(deadline);
worker.wake();
await vi.advanceTimersByTimeAsync(0);
expect(state.probe!()).toBe("present");
expect(coordinator.nextWakeAt()).toBe(deadline);
const scheduled = state.cleanup.mock.calls.length;
await vi.advanceTimersByTimeAsync(59_999);
expect(state.cleanup).toHaveBeenCalledTimes(scheduled);
state.next.mockResolvedValue(null);
await vi.advanceTimersByTimeAsync(1);
expect(state.cleanup).toHaveBeenCalledTimes(scheduled + 1);
expect(state.probe!()).toBe("none");
expect(coordinator.nextWakeAt()).toBeNull();
await worker.stop();
expect(state.unregister).toHaveBeenCalledOnce();
expect(onError).not.toHaveBeenCalled();
});
server/src/__tests__/recovery-cutover.test.ts
+15
View File
@@ -0,0 +1,15 @@
import { execFile } from "node:child_process";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { it } from "vitest";
const run = promisify(execFile);
// Keep the dependency-free, actual-CLI acceptance suite usable on a retained
// volume while also exercising the same cases in the normal CI test lane.
it("passes the read-only recovery cutover CLI acceptance suite", async () => {
await run(process.execPath, ["--test", fileURLToPath(new URL("../../../scripts/recovery-cutover.test.mjs", import.meta.url))], {
timeout: 45_000,
maxBuffer: 1024 * 1024,
});
}, 60_000);
server/src/__tests__/run-evidence-migration-work.test.ts
+168
View File
@@ -0,0 +1,168 @@
import type { Db } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
import { afterEach, expect, it, vi } from "vitest";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js";
const state = vi.hoisted(() => ({
provider: "s3",
migrate: vi.fn(),
probe: undefined as undefined | (() => string),
unregister: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
}));
vi.mock("../storage/config.js", () => ({
loadStorageConfig: () => ({ storageProvider: state.provider }),
}));
vi.mock("../services/run-evidence-migration.js", () => ({ migrateRunEvidence: state.migrate }));
vi.mock("../middleware/logger.js", () => ({ logger: { info: state.info, warn: state.warn } }));
vi.mock("../services/idle-local-work.js", () => ({
registerIdleLocalWorkProbe: (probe: () => string) => {
state.probe = probe;
return state.unregister;
},
}));
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
afterEach(async () => {
await coordinator?.stop();
coordinator = undefined;
state.provider = "s3";
state.probe = undefined;
vi.resetAllMocks();
vi.useRealTimers();
});
function setup(canRun = () => true) {
vi.useFakeTimers();
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
const onError = vi.fn();
coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError });
const worker = registerRunEvidenceMigrationWork(owner as unknown as Db, coordinator, canRun);
return { worker, onError };
}
it("does no migration work for local storage", async () => {
state.provider = "local_disk";
const { worker } = setup();
expect(worker).toBeNull();
await vi.advanceTimersByTimeAsync(3_600_000);
expect(state.migrate).not.toHaveBeenCalled();
expect(state.probe).toBeUndefined();
expect(coordinator!.nextWakeAt()).toBeNull();
});
it("automatically pages both resources, then leaves completed history quiet", async () => {
state.migrate
.mockResolvedValueOnce({
items: [{ kind: "log", id: "one", state: "migrated" }],
nextRunId: "one",
})
.mockResolvedValueOnce({
items: [{ kind: "log", id: "two", state: "migrated" }],
nextRunId: "two",
})
.mockResolvedValue({ items: [] });
const { worker, onError } = setup();
expect(state.probe!()).toBe("unknown");
await worker!.ready;
expect(state.migrate).toHaveBeenNthCalledWith(
1,
expect.anything(),
expect.objectContaining({
apply: true,
limit: 1,
kinds: ["log", "trace"],
}),
);
expect(state.probe!()).toBe("present");
await vi.advanceTimersByTimeAsync(2_000);
expect(state.migrate).toHaveBeenNthCalledWith(
2,
expect.anything(),
expect.objectContaining({
afterRunId: "one",
kinds: ["log"],
}),
);
expect(state.migrate).toHaveBeenNthCalledWith(
3,
expect.anything(),
expect.objectContaining({
afterRunId: "two",
kinds: ["log"],
}),
);
expect(state.probe!()).toBe("none");
expect(coordinator!.nextWakeAt()).toBeNull();
await vi.advanceTimersByTimeAsync(3_600_000);
expect(state.migrate).toHaveBeenCalledTimes(3);
expect(onError).not.toHaveBeenCalled();
await worker!.stop();
expect(state.unregister).toHaveBeenCalledOnce();
});
it("keeps blocked history visible and retries it without stranding later pages", async () => {
state.migrate
.mockResolvedValueOnce({
items: [{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" }],
nextRunId: "active",
})
.mockResolvedValueOnce({ items: [{ kind: "log", id: "later", state: "migrated" }] })
.mockResolvedValue({ items: [{ kind: "log", id: "active", state: "migrated" }] });
const { worker } = setup();
await worker!.ready;
await vi.advanceTimersByTimeAsync(1_000);
expect(state.warn).toHaveBeenCalledWith(
expect.objectContaining({ id: "active", reason: "legacy_writer_not_settled" }),
expect.any(String),
);
expect(state.probe!()).toBe("present");
await vi.advanceTimersByTimeAsync(299_999);
expect(state.migrate).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(1);
expect(state.migrate).toHaveBeenNthCalledWith(
3,
expect.anything(),
expect.objectContaining({ afterRunId: undefined, kinds: ["log", "trace"] }),
);
expect(state.probe!()).toBe("none");
expect(coordinator!.nextWakeAt()).toBeNull();
});
it("retains startup work during drain and retries a failed sweep before becoming quiet", async () => {
let admitted = false;
state.migrate
.mockRejectedValueOnce(new Error("storage unavailable"))
.mockResolvedValue({ items: [] });
const { worker, onError } = setup(() => admitted);
await worker!.ready;
await vi.advanceTimersByTimeAsync(60_000);
expect(state.migrate).not.toHaveBeenCalled();
expect(state.probe!()).toBe("unknown");
admitted = true;
await vi.advanceTimersByTimeAsync(30_000);
expect(onError).toHaveBeenCalledOnce();
await vi.advanceTimersByTimeAsync(30_000);
expect(state.migrate).toHaveBeenCalledTimes(2);
expect(state.probe!()).toBe("none");
expect(coordinator!.nextWakeAt()).toBeNull();
});
it("aborts owned work on shutdown and reconstructs outstanding history on restart", async () => {
let started: AbortSignal | undefined;
state.migrate.mockImplementationOnce(
(_db, options) =>
new Promise((_resolve, reject) => {
started = options.signal;
started!.addEventListener("abort", () => reject(started!.reason), { once: true });
}),
);
const { worker } = setup();
await vi.advanceTimersByTimeAsync(0);
expect(started?.aborted).toBe(false);
await worker!.stop();
await worker!.ready;
expect(started?.aborted).toBe(true);
expect(coordinator!.nextWakeAt()).toBeNull();
await coordinator!.stop();
state.migrate.mockResolvedValue({ items: [] });
const restarted = setup();
await restarted.worker!.ready;
expect(state.migrate).toHaveBeenCalledTimes(2);
expect(state.probe!()).toBe("none");
});
server/src/__tests__/server-startup-feedback-export.test.ts
+6
View File
@@ -697,6 +697,12 @@ describe("startServer feedback export wiring", () => {
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toEqual([]);
});
it.each([false, true])("registers trace retention independently of heartbeat scheduling (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.providerTraceRetention)).toHaveLength(1);
});
it.each([false, true])("registers skill cache retention only when enabled (%s)", async enabled => {
loadConfigMock.mockReturnValue(buildTestConfig({ skillVersionRetentionEnabled: enabled }));
await startServer(); await settleStartupHousekeeping();
server/src/__tests__/task-cold-open.test.ts
+46 -4
View File
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { activityLog, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
import { activityLog, agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { issueRoutes } from "../routes/issues.js";
import { errorHandler } from "../middleware/index.js";
@@ -11,16 +11,25 @@ import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
import { activityService } from "../services/activity.js";
import { documentService } from "../services/documents.js";
const support = await getEmbeddedPostgresTestSupport();
const externalDatabase = process.env.PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL;
if (externalDatabase) {
const target = new URL(externalDatabase);
if (!["localhost", "127.0.0.1", "[::1]"].includes(target.hostname) || !target.pathname.endsWith("_test"))
throw new Error("PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL must name a disposable loopback database ending in _test");
}
const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("task cold-open reads", () => {
let fixture: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let fixture: Pick<Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, "connectionString" | "cleanup">;
let db: ReturnType<typeof createDb>;
const companyId = randomUUID();
const issueId = randomUUID();
const agentId = randomUUID();
const runId = randomUUID();
beforeAll(async () => {
fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
if (externalDatabase) {
await applyPendingMigrations(externalDatabase);
fixture = { connectionString: externalDatabase, cleanup: () => closeRegisteredClients(externalDatabase) };
} else fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-");
db = createDb(fixture.connectionString);
await db.insert(companies).values({ id: companyId, name: "Cold open", issuePrefix: `T${companyId.slice(0, 6)}` });
await db.insert(agents).values({ id: agentId, companyId, name: "Writer", role: "engineer", adapterType: "codex_local" });
@@ -79,6 +88,39 @@ const support = await getEmbeddedPostgresTestSupport();
expect(response.body.latestCommentId).toBe(response.body.comments[1].id);
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${issueId}/recent-comments`)).status);
});
it("opens long chats with a contiguous recent window and pages every older message", async () => {
const chatId = randomUUID();
await db.insert(issues).values({ id: chatId, companyId, title: "Persistent chat", conversationAgentId: agentId, conversationUserId: "local-board", assigneeAgentId: agentId, conversationState: "waiting" });
const rows = Array.from({ length: 123 }, (_, i) => ({ id: randomUUID(), companyId, issueId: chatId,
authorType: "user" as const, authorUserId: "local-board", body: `Chat ${i}`, createdAt: new Date(1000 + Math.floor(i / 3)) }));
await db.insert(issueComments).values(rows);
const ordered = [...rows].sort((a, b) => +b.createdAt - +a.createdAt || b.id.localeCompare(a.id));
const response = await request(app()).get(`/api/issues/${chatId}/recent-comments`);
expect(response.status).toBe(200);
expect(response.body.comments.map((row: { id: string }) => row.id)).toEqual(ordered.slice(0, 20).map(row => row.id));
expect(response.body.oldestCommentId).toBe(ordered[19].id);
expect(response.body.hasMore).toBe(true);
const seen = response.body.comments.map((row: { id: string }) => row.id);
let cursor = response.body.oldestCommentId;
while (cursor) {
const page = await request(app()).get(`/api/issues/${chatId}/comments`).query({ order: "desc", limit: 50, after: cursor });
expect(page.status).toBe(200);
seen.push(...page.body.map((row: { id: string }) => row.id));
cursor = page.body.at(-1)?.id;
}
expect(seen).toEqual(ordered.map(row => row.id));
expect(new Set(seen).size).toBe(123);
expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${chatId}/recent-comments`)).status);
});
it("hydrates only requested runs linked to the company and issue", async () => {
const otherIssue = randomUUID(), otherRun = randomUUID();
await db.insert(issues).values({ id: otherIssue, companyId, title: "Other task" });
await db.insert(heartbeatRuns).values({ id: otherRun, companyId, agentId, issueId: otherIssue, status: "succeeded" });
const activity = activityService(db);
expect((await activity.runsForIssue(companyId, issueId, null, undefined, [runId, otherRun])).map(run => run.runId)).toEqual([runId]);
expect(await activity.runsForIssue(randomUUID(), issueId, null, undefined, [runId])).toEqual([]);
expect(await activity.runsForIssue(companyId, issueId, null, undefined, [])).toEqual([]);
});
it.each(["user", null] as const)("preserves initial and latest human instructions without a description (authorType=%s)", async (authorType) => {
const bareIssueId = randomUUID();
await db.insert(issues).values({ id: bareIssueId, companyId, title: "No description" });
server/src/config.ts
+4 -38
View File
@@ -16,13 +16,11 @@ import {
BIND_MODES,
DEPLOYMENT_EXPOSURES,
SECRET_PROVIDERS,
STORAGE_PROVIDERS,
type BindMode,
type AuthBaseUrlMode,
type DeploymentExposure,
type DeploymentMode,
type SecretProvider,
type StorageProvider,
inferBindModeFromHost,
resolveRuntimeBind,
validateConfiguredBindMode,
@@ -31,9 +29,9 @@ import {
resolveDefaultBackupDir,
resolveDefaultEmbeddedPostgresDir,
resolveDefaultSecretsKeyFilePath,
resolveDefaultStorageDir,
resolveHomeAwarePath,
} from "./home-paths.js";
import { loadStorageConfig, type StorageConfig } from "./storage/config.js";
if (!isCloudManagedInstance()) {
const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath();
@@ -63,7 +61,7 @@ const TAILSCALE_DETECT_TIMEOUT_MS = 3000;
type DatabaseMode = "embedded-postgres" | "postgres";
export interface Config {
export interface Config extends StorageConfig {
deploymentMode: DeploymentMode;
deploymentExposure: DeploymentExposure;
bind: BindMode;
@@ -116,13 +114,6 @@ export interface Config {
secretsProvider: SecretProvider;
secretsStrictMode: boolean;
secretsMasterKeyFilePath: string;
storageProvider: StorageProvider;
storageLocalDiskBaseDir: string;
storageS3Bucket: string;
storageS3Region: string;
storageS3Endpoint: string | undefined;
storageS3Prefix: string;
storageS3ForcePathStyle: boolean;
feedbackExportBackendUrl: string | undefined;
feedbackExportBackendToken: string | undefined;
heartbeatSchedulerEnabled: boolean;
@@ -162,7 +153,6 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()
: undefined;
const fileDatabaseBackup = fileConfig?.database.backup;
const fileSecrets = fileConfig?.secrets;
const fileStorage = fileConfig?.storage;
const providerFromEnvRaw = process.env.PAPERCLIP_SECRETS_PROVIDER;
const providerFromEnv =
@@ -172,25 +162,7 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()
const providerFromFile = fileSecrets?.provider;
const secretsProvider: SecretProvider = providerFromEnv ?? providerFromFile ?? "local_encrypted";
const storageProviderFromEnvRaw = process.env.PAPERCLIP_STORAGE_PROVIDER;
const storageProviderFromEnv =
storageProviderFromEnvRaw && STORAGE_PROVIDERS.includes(storageProviderFromEnvRaw as StorageProvider)
? (storageProviderFromEnvRaw as StorageProvider)
: null;
const storageProvider: StorageProvider = storageProviderFromEnv ?? fileStorage?.provider ?? "local_disk";
const storageLocalDiskBaseDir = resolveHomeAwarePath(
process.env.PAPERCLIP_STORAGE_LOCAL_DIR ??
fileStorage?.localDisk?.baseDir ??
resolveDefaultStorageDir(),
);
const storageS3Bucket = process.env.PAPERCLIP_STORAGE_S3_BUCKET ?? fileStorage?.s3?.bucket ?? "paperclip";
const storageS3Region = process.env.PAPERCLIP_STORAGE_S3_REGION ?? fileStorage?.s3?.region ?? "us-east-1";
const storageS3Endpoint = process.env.PAPERCLIP_STORAGE_S3_ENDPOINT ?? fileStorage?.s3?.endpoint ?? undefined;
const storageS3Prefix = process.env.PAPERCLIP_STORAGE_S3_PREFIX ?? fileStorage?.s3?.prefix ?? "";
const storageS3ForcePathStyle =
process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE !== undefined
? process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE === "true"
: (fileStorage?.s3?.forcePathStyle ?? false);
const storage = loadStorageConfig(fileConfig);
const feedbackExportBackendUrl =
process.env.PAPERCLIP_FEEDBACK_EXPORT_BACKEND_URL?.trim() ||
process.env.PAPERCLIP_TELEMETRY_BACKEND_URL?.trim() ||
@@ -447,13 +419,7 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile()
fileSecrets?.localEncrypted.keyFilePath ??
resolveDefaultSecretsKeyFilePath(),
),
storageProvider,
storageLocalDiskBaseDir,
storageS3Bucket,
storageS3Region,
storageS3Endpoint,
storageS3Prefix,
storageS3ForcePathStyle,
...storage,
feedbackExportBackendUrl,
feedbackExportBackendToken,
heartbeatSchedulerEnabled: process.env.HEARTBEAT_SCHEDULER_ENABLED !== "false",
server/src/index.ts
+11
View File
@@ -1,3 +1,5 @@
import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
import { registerDatabaseBackupWork } from "./services/database-backup-work.js";
import { registerToolConnectionHealthWork } from "./services/tool-connection-health-work.js";
@@ -1365,6 +1367,13 @@ async function startServerWithDatabaseTeardown(
}));
});
server.on("close", () => { void taskDrainExpiryWork.stop(); });
const runEvidenceMigrationWork = registerRunEvidenceMigrationWork(db, deliveryWork,
() => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), { storageProvider: config.storageProvider });
if (runEvidenceMigrationWork) trackHeartbeatSchedulerWork(runEvidenceMigrationWork.ready);
const providerTraceRetentionWork = registerProviderTraceRetentionWork(db, deliveryWork,
() => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby());
trackHeartbeatSchedulerWork(providerTraceRetentionWork.ready);
server.on("close", providerTraceRetentionWork.stop);
const executionSchedule = executionControlSchedule(db);
const workspaceRecoveryRuntime = environmentRuntimeService(db, { pluginWorkerManager });
const nativeRunnerEvidenceRetentionWorker = config.nativeHarnessBackupExpirationEnabled
@@ -2236,6 +2245,8 @@ async function startServerWithDatabaseTeardown(
unsubscribeNativeWorkspaceFinalization?.();
unsubscribeRetainedNativeSessionCleanup?.();
await deliveryWork.stop();
await runEvidenceMigrationWork?.stop();
await providerTraceRetentionWork.stop();
await skillVersionRetentionWorker?.dispose();
await workspaceRecoveryWork.stop();
unregisterTerminalWorkspaceProbe?.();
server/src/routes/activity.ts
+8
View File
@@ -373,6 +373,14 @@ export function activityRoutes(db: Db) {
if (!issue) return;
if (!(await assertIssueReadAllowed(req, res, issue))) return;
const viewerUserId = req.actor.type === "board" ? getActorInfo(req).actorId : null;
if (req.query.runIds !== undefined) {
const parsed = z.array(z.string().uuid()).min(1).max(100).safeParse(
typeof req.query.runIds === "string" ? req.query.runIds.split(",") : null,
);
if (!parsed.success) throw badRequest("runIds must contain 1 to 100 UUIDs");
res.json(await svc.runsForIssue(issue.companyId, issue.id, viewerUserId, undefined, parsed.data));
return;
}
const result = await (req.query.view === "recent"
? svc.runsForIssue(issue.companyId, issue.id, viewerUserId, 2)
: svc.runsForIssue(issue.companyId, issue.id, viewerUserId));
server/src/routes/issues.ts
+16 -4
View File
@@ -16449,12 +16449,23 @@ export function issueRoutes(
router.get("/issues/:id/recent-comments", async (req, res) => {
const issue = await getAccessibleResource(req, res, getIssueById(req, req.params.id as string), "Issue not found");
if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return;
if (issue.conversationAgentId) {
const recent = await svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 21 });
const comments = recent.slice(0, 20);
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
comments,
latestCommentId: comments[0]?.id ?? null,
oldestCommentId: comments.at(-1)?.id ?? null,
hasMore: recent.length > 20,
}));
return;
}
const [recent, human, firstInstruction] = await Promise.all([
svc.listComments(issue.id, { order: "desc", limit: 5 }),
svc.listComments(issue.id, { order: "desc", limit: 1, humanOnly: true }),
svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 5 }),
svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 1, humanOnly: true }),
issue.description?.trim()
? Promise.resolve([])
: svc.listComments(issue.id, { order: "asc", limit: 1, humanOnly: true }),
: svc.listComments(issue.id, { companyId: issue.companyId, order: "asc", limit: 1, humanOnly: true }),
]);
const comments = [...new Map([...firstInstruction, ...human, ...recent.slice(0, 4)].map((comment) => [comment.id, comment])).values()];
res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, {
@@ -16494,7 +16505,8 @@ export function issueRoutes(
limitRaw && Number.isFinite(limitRaw) && limitRaw > 0
? Math.min(Math.floor(limitRaw), MAX_ISSUE_COMMENT_LIMIT)
: null;
const comments = await svc.listComments(id, {
const comments = await svc.listComments(issue.id, {
companyId: issue.companyId,
afterCommentId,
order,
limit,
server/src/routes/openapi.ts
+3 -3
View File
@@ -4599,7 +4599,7 @@ registry.registerPath({
path: "/api/issues/{id}/recent-comments",
tags: ["issues"],
summary: "Read a bounded recent task exchange",
description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Agent chats instead return the latest 20 contiguous comments and oldestCommentId for backward cursor paging. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.",
request: { params: z.object({ id: z.string() }) },
responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
@@ -6091,8 +6091,8 @@ registry.registerPath({
path: "/api/issues/{id}/runs",
tags: ["activity"],
summary: "List runs for an issue",
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries.") }) },
responses: { 200: r.ok(), 401: r.unauthorized },
request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries."), runIds: z.string().optional().describe("Comma-separated list of 1 to 100 run UUIDs. Returns only those runs linked to this accessible issue.") }) },
responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound },
});
registry.registerPath({
server/src/secrets/local-encrypted-provider.ts
+2 -2
View File
@@ -94,8 +94,8 @@ function loadOrCreateMasterKey(allowCreate = true, env: LocalEncryptedKeyEnv = p
}
/** Derive a purpose-specific instance secret without exposing or reusing the vault key. */
export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env): string {
return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(true, env), Buffer.alloc(0),
export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env, options: { allowCreate?: boolean } = {}): string {
return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(options.allowCreate ?? true, env), Buffer.alloc(0),
`paperclip:instance-secret:v1\0${purpose}`, 32)).toString("base64url");
}
server/src/services/activity.ts
+3 -1
View File
@@ -428,7 +428,8 @@ export function activityService(db: Db) {
return limit === undefined ? query : query.limit(limit);
},
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number) => {
runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number, selectedRunIds?: string[]) => {
if (selectedRunIds && !selectedRunIds.length) return [];
scheduleRunLivenessBackfill(companyId, issueId);
const runsQuery = db
.select({
@@ -479,6 +480,7 @@ export function activityService(db: Db) {
.where(
and(
eq(heartbeatRuns.companyId, companyId),
...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []),
or(
eq(heartbeatRuns.issueId, issueId),
sql`exists (
server/src/services/agent-identity.ts
+16 -11
View File
@@ -5,6 +5,7 @@ import type { AgentPublicIdentity } from "@paperclipai/shared";
import { notFound } from "../errors.js";
import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js";
import { persistActivity } from "./activity-log.js";
import { createAgentIdentityRedactor } from "./agent-identity-redaction.js";
const publicColumns = {
algorithm: agentIdentityKeys.algorithm,
@@ -17,6 +18,16 @@ export function agentIdentityService(db: Db) {
const ownership = (companyId: string, agentId: string) => and(
eq(agentIdentityKeys.companyId, companyId), eq(agentIdentityKeys.agentId, agentId),
);
async function resolveIdentity(row: typeof agentIdentityKeys.$inferSelect) {
const privateKeyPem = await localEncryptedProvider.resolveVersion({ material: row.privateKeyMaterial, externalRef: null });
const privateKey = createPrivateKey(privateKeyPem);
const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
!derived.equals(stored) || fingerprint(stored) !== row.keyId)
throw new Error("Agent identity public and private keys do not match");
return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
}
return {
// Reading an identity must never provision it or decrypt private material.
@@ -24,6 +35,10 @@ export function agentIdentityService(db: Db) {
const [row] = await db.select(publicColumns).from(agentIdentityKeys).where(ownership(companyId, agentId));
return row ? { ...row, createdAt: row.createdAt.toISOString() } : null;
},
async createExistingLogRedactor(companyId: string, agentId: string) {
const [row] = await db.select().from(agentIdentityKeys).where(ownership(companyId, agentId));
return createAgentIdentityRedactor(row ? (await resolveIdentity(row)).privateKeyPem : undefined);
},
async ensureAgentIdentity(companyId: string, agentId: string) {
return db.transaction(async (tx) => {
@@ -49,18 +64,8 @@ export function agentIdentityService(db: Db) {
details: { algorithm: "Ed25519", keyId },
});
}
const privateKeyPem = await localEncryptedProvider.resolveVersion({
material: row.privateKeyMaterial, externalRef: null,
});
// Fail closed on corruption or a wrong master key; never replace a key.
const privateKey = createPrivateKey(privateKeyPem);
const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" });
const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" });
if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" ||
!derived.equals(stored) || fingerprint(stored) !== row.keyId) {
throw new Error("Agent identity public and private keys do not match");
}
return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem };
return resolveIdentity(row);
});
},
};
server/src/services/ai-connections.ts
+22 -14
View File
@@ -1105,7 +1105,7 @@ export function aiConnectionService(db: Db) {
async function refreshQuotaCredential(row: Parameters<typeof credential>[0], failedValue: string, signal: AbortSignal) {
return withAccountHomeSecretMutationLock(undefined, row.connection.companyId, async () => {
const recovery = quotaCredentialRecovery(row.connection.companyId, row.grant.id);
let savingReplacement = false;
let replacement: { value: string; secretId: string; version: number; epoch: number; base: string; grantUpdatedAt: number; versionSelector: "latest" | number } | undefined;
const save = () => db.transaction(async (tx) => {
// Do not wait on a transaction that might itself be waiting for the file
// lock. All database locks must be available before consuming OAuth tokens.
@@ -1117,11 +1117,20 @@ export function aiConnectionService(db: Db) {
if (!grant || grant.status !== "active") throw new Error("credentials_unavailable");
const ref = grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential");
if (!ref) throw new Error("credentials_unavailable");
await tx.select({ id: companySecrets.id }).from(companySecrets).where(and(
const [secret] = await tx.select({ id: companySecrets.id, version: companySecrets.latestVersion, epoch: companySecrets.aiSessionEpoch }).from(companySecrets).where(and(
eq(companySecrets.id, ref.secretId), eq(companySecrets.companyId, companyId),
)).for("update", { noWait: true });
if (!secret) throw new Error("credentials_unavailable");
const current = await aiConnectionService(tx as unknown as Db).credential({ connection: row.connection, grant });
signal.throwIfAborted();
if (replacement) {
// A lost commit response is reconciled from the authoritative secret.
// Reconnect, revocation and a competing saved generation always win.
if (current === replacement.value) return current;
if (ref.secretId !== replacement.secretId || secret.version !== replacement.version
|| secret.epoch !== replacement.epoch || current !== replacement.base
|| grant.updatedAt.getTime() !== replacement.grantUpdatedAt
|| (ref.versionSelector ?? "latest") !== replacement.versionSelector) return current;
} else signal.throwIfAborted();
let pending = await recovery.read();
if (pending && (pending.secretId !== ref.secretId || pending.connectionId !== row.connection.id || pending.baseHash !== quotaCredentialHash(current))) {
// Reconnect may reuse this grant. Once the current authorized token
@@ -1141,10 +1150,9 @@ export function aiConnectionService(db: Db) {
or(sql`${heartbeatRuns.contextSnapshot}->'aiConnection'->>'grantId' = ${grant.id}`,
sql`${agents.runtimeConfig}->'aiConnection'->>'provider' = 'openai'`, eq(agents.adapterType, "codex_local")),
)).limit(1);
if (active && !pending) throw new Error("provider_unavailable");
let refreshed = pending?.value;
if (active && !pending && !replacement) throw new Error("provider_unavailable");
let refreshed = replacement?.value ?? pending?.value;
if (!refreshed) {
await recovery.prepare();
const auth = JSON.parse(current);
const refreshToken = auth.tokens?.refresh_token;
if (typeof refreshToken !== "string" || !refreshToken) throw new Error("authentication_required");
@@ -1169,12 +1177,12 @@ export function aiConnectionService(db: Db) {
...(typeof tokens.refresh_token === "string" && tokens.refresh_token ? { refresh_token: tokens.refresh_token } : {}),
...(typeof tokens.id_token === "string" && tokens.id_token ? { id_token: tokens.id_token } : {}),
}, last_refresh: new Date().toISOString() });
// This fsynced encrypted journal survives transaction rollback and host
// restart. Never exchange the consumed token again after a failed save.
await recovery.write({ companyId, grantId: grant.id, connectionId: row.connection.id,
secretId: ref.secretId, baseHash: quotaCredentialHash(current), value: refreshed });
}
savingReplacement = true;
// Only this logical refresh owns the replacement. Save retries must not
// consume another single-use token. Process loss before commit can require
// signing in again in v1; no secondary credential vault is created.
replacement ??= { value: refreshed, secretId: ref.secretId, version: secret.version, epoch: secret.epoch, base: current,
grantUpdatedAt: grant.updatedAt.getTime(), versionSelector: ref.versionSelector ?? "latest" };
// Persist the rotated token even if the quota deadline expires after the
// successful exchange; losing it would strand all subsequent executions.
await secretService(tx).rotate(ref.secretId, { value: refreshed, preserveAiSessionEpoch: true }, { userId: grant.subjectUserId });
@@ -1191,9 +1199,9 @@ export function aiConnectionService(db: Db) {
await recovery.clear().catch((error) => logger.warn({ err: error, grantId: row.grant.id }, "Saved quota credential recovery cleanup pending"));
return value;
} catch (error) {
if (!savingReplacement || attempt >= 2) throw error;
// Saving retries have no provider side effect. Persistent failures
// retain the journal for the next quota poll, including after restart.
if (!replacement || attempt >= 2) throw error;
// Retry only the returned value. Required database reads still fail
// closed; exhaustion is an error, never a successful refresh.
}
}
});
server/src/services/customer-success-inspection.ts
+3 -3
View File
@@ -20,7 +20,7 @@ import {
import { deriveBundleState } from "./agent-instructions.js";
import { readInstructionBytes } from "./agent-instruction-files.js";
import { skillPackageStore } from "./skill-package-store.js";
import { getRunLogStore } from "./run-log-store.js";
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
// This is a reviewed catalog of readers, not a proxy for existing GET routes.
// Select credentials out of queries; never load encrypted identity records.
@@ -311,9 +311,9 @@ export async function readCustomerSuccessResource(
}
if (q.operation === "runs.log") {
const run = await ownRow(tx, schema.heartbeatRuns, q.companyId!, q.resourceId!);
if (!run.logRef || run.logStore !== "local_file") return unavailable("run_log_unavailable");
if (!run.logRef || !isRunLogStoreType(run.logStore)) return unavailable("run_log_unavailable");
const result = await getRunLogStore().read(
{ store: "local_file", logRef: String(run.logRef) },
{ store: run.logStore, logRef: String(run.logRef) },
{
offset: q.range?.start ?? q.offset ?? 0,
limitBytes: q.range ? q.range.end - q.range.start + 1 : 256000,
server/src/services/delivery-work-notifications.ts
+2
View File
@@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
/** Topics for existing durable queues; register intent before their writes. */
export const DELIVERY_QUEUES = {
runEvidenceMigration: "run-evidence-migration",
providerTraceRetention: "provider-trace-retention",
nativeRunnerEvidenceRetention: "native-runner-evidence-retention",
skillVersionRetention: "skill-version-retention",
databaseBackups: "database-backups",
server/src/services/execution-preparation-timing.test.ts
+11 -7
View File
@@ -1,6 +1,6 @@
import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming } from "./execution-preparation-timing.js";
import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming, withRunResponsePresentationTiming } from "./execution-preparation-timing.js";
import { ROOT_CONTEXT, trace, SpanStatusCode, type Context } from "@opentelemetry/api";
import { createHash } from "node:crypto";
@@ -45,6 +45,7 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
try {
await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42);
await expect(withRunWorkspaceFinalizationTiming("run", () => 42)).resolves.toBe(42);
await expect(withRunResponsePresentationTiming("run", () => 42)).resolves.toBe(42);
for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation",
"skills.inventory", "skills.inventory.wait", "skills.cache.acquire", "skills.version.materialize", "skills.consume",
"workspace_sync.seed_read_copy", "workspace_sync.seed_read_hash", "workspace_sync.seed_read_touch"] as const) {
@@ -69,7 +70,10 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async
}
});
it("parents concurrent deferred copyback phases to their own finalization roots and preserves failure", async () => {
it.each([
[withRunWorkspaceFinalizationTiming, "task.workspace_finalization", "workspace_sync.restore"],
[withRunResponsePresentationTiming, "task.response_presentation", "response.transaction"],
] as const)("parents concurrent deferred phases to their own %s root and preserves failure", async (withTiming, rootName, phase) => {
const spans: Array<{ name: string; attributes: Record<string, unknown>; context: Context;
span: { end: ReturnType<typeof vi.fn>; setStatus: ReturnType<typeof vi.fn> } }> = [];
handles.startActiveSpan.mockImplementation((name, options, context, callback) => {
@@ -77,16 +81,16 @@ it("parents concurrent deferred copyback phases to their own finalization roots
spans.push({ name, attributes: options.attributes, context, span });
return callback(span);
});
await Promise.all(["first", "second"].map(runId => withRunWorkspaceFinalizationTiming(runId, async () => {
await Promise.all(["first", "second"].map(runId => withTiming(runId, async () => {
await Promise.resolve();
await measureRunPhase("workspace_sync.restore", async () => {});
await measureRunPhase(phase, async () => {});
})));
for (const runId of ["first", "second"]) {
const hash = createHash("sha256").update(runId).digest("hex").slice(0, 12);
const owned = spans.filter(span => span.attributes["paperclip.task.run.run_id"] === hash);
expect(owned).toHaveLength(2);
const root = owned.find(span => span.name === "task.workspace_finalization")!;
const child = owned.find(span => span.name === "workspace_sync.restore")!;
const root = owned.find(span => span.name === rootName)!;
const child = owned.find(span => span.name === phase)!;
expect(root.context).toBe(ROOT_CONTEXT);
expect(trace.getSpan(child.context)).toBe(root.span);
expect(root.span.end).toHaveBeenCalledOnce();
@@ -94,7 +98,7 @@ it("parents concurrent deferred copyback phases to their own finalization roots
}
const original = new Error("private finalization failure");
const work = vi.fn(() => { throw original; });
await expect(withRunWorkspaceFinalizationTiming("failed", work)).rejects.toBe(original);
await expect(withTiming("failed", work)).rejects.toBe(original);
expect(work).toHaveBeenCalledOnce();
expect(spans.at(-1)!.span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR });
expect(spans.at(-1)!.span.end).toHaveBeenCalledOnce();
server/src/services/execution-preparation-timing.ts
+8
View File
@@ -199,6 +199,14 @@ export function withRunWorkspaceFinalizationTiming<T>(runId: string, work: () =>
return measure(runId, "task.workspace_finalization", work, true);
}
export function withRunResponsePresentationTiming<T>(runId: string, work: () => T | Promise<T>): Promise<T> {
const owned = ownership.getStore();
if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) {
return (async () => work())();
}
return measure(runId, "task.response_presentation", work, true);
}
export async function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
const owned = ownership.getStore();
if (!endpointEnabled() || !owned || owned.ended) return work();
server/src/services/feedback.ts
+7 -5
View File
@@ -47,7 +47,8 @@ import {
sanitizeFeedbackValue,
sha256Digest,
} from "./feedback-redaction.js";
import { getRunLogStore } from "./run-log-store.js";
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
import { getOperatorSettingDefaults } from "./setting-defaults.js";
const FEEDBACK_SCHEMA_VERSION = "paperclip-feedback-envelope-v2";
@@ -318,17 +319,18 @@ async function findMatchingFile(
return search(rootDir, 0);
}
async function readFullRunLog(run: {
async function readFullRunLog(db: Db, run: {
logStore: string | null;
logRef: string | null;
}) {
if (run.logStore !== "local_file" || !run.logRef) return null;
if (!isRunLogStoreType(run.logStore) || !run.logRef) return null;
const store = getRunLogStore();
if (run.logStore === "s3_segments") await recoverSettledNativeRunLog(db, store, run.logRef).catch(() => undefined);
let offset = 0;
let combined = "";
while (true) {
const result = await store.read({ store: "local_file", logRef: run.logRef }, {
const result = await store.read({ store: run.logStore, logRef: run.logRef }, {
offset,
limitBytes: 512_000,
}).catch(() => null);
@@ -1526,7 +1528,7 @@ async function buildFeedbackTraceBundleFromRow(
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, run.id))
.orderBy(asc(heartbeatRunEvents.seq));
const logText = await readFullRunLog(run);
const logText = await readFullRunLog(db, run);
const logEntries = parseRunLogEntries(logText);
const stdoutText = logEntries
.filter((entry) => entry.stream === "stdout")
server/src/services/heartbeat.ts
+16 -4
View File
@@ -25,7 +25,10 @@ import { createHeartbeatPreparationTiming } from "./heartbeat/preparation-timing
import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkingCopy } from "./heartbeat/instruction-working-copy.js";
import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js";
import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js";
import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js";
import { createHeartbeatRunOutput } from "./heartbeat/run-output.js";
import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.js";
import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability";
import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js";
import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js";
import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js";
@@ -370,7 +373,7 @@ import {
type NativeSessionBackend,
} from "../vendor/paperclip-runner/index.js";
import { getRunLogStore } from "./run-log-store.js";
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
import {
providerTraceStore,
} from "./provider-trace-store.js";
@@ -2075,6 +2078,7 @@ export function heartbeatService(
}
try {
providerTraceCapture = await traceStore.prepare({
assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
runId: run.id,
companyId: run.companyId,
provider:
@@ -3116,6 +3120,8 @@ export function heartbeatService(
const runOutput = createHeartbeatRunOutput(db, {
run, runId, issueId, runLogStore, identityRedactor, secretKeys,
appendRunEvent, getCurrentUserRedactionOptions,
assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner),
runtimeLogSecretValues: observabilitySecretValues(observability),
});
const goalCheckpointSession: {
current: {
@@ -3201,7 +3207,7 @@ export function heartbeatService(
message: "run started",
});
const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent } =
const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onPersistedNativeLog, onAdapterMeta, onAdapterEvent } =
await timePreparation("run_output", () => runOutput.begin(currentRun));
if (runScopedMentionedSkillKeys.length > 0) {
await onLog(
@@ -3435,6 +3441,7 @@ export function heartbeatService(
identityRedactor,
appendIdentityRedactedLog,
onLog,
onPersistedNativeLog,
onAdapterEvent,
onAdapterMeta,
},
@@ -4055,6 +4062,9 @@ export function heartbeatService(
companyId: string;
logStore: string | null;
logRef: string | null;
runtimeMode?: string | null;
status?: string;
logBytes?: number | null;
},
opts?: { offset?: number; limitBytes?: number; alignStartToRecord?: boolean; signal?: AbortSignal },
) => {
@@ -4067,12 +4077,14 @@ export function heartbeatService(
if (!run) throw notFound("Heartbeat run not found");
if (!run.logStore || !run.logRef) throw notFound("Run log not found");
const result = await runLogStore.read(
if (!isRunLogStoreType(run.logStore)) throw notFound("Run log backend not supported");
const result = await readRecoverableRunLog(db, runLogStore,
{
store: run.logStore as "local_file",
store: run.logStore,
logRef: run.logRef,
},
opts,
run,
);
return {
server/src/services/heartbeat/run-completion.test.ts
+53
View File
@@ -1,3 +1,5 @@
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
import type { Db } from "@paperclipai/db";
import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
@@ -158,6 +160,57 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)(
expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object));
});
it("settles the accepted native exact rate-card price without replacing it with zero", async () => {
const f = await fixture({ runtimeMode: "native" });
const runnerInstanceId = randomUUID();
const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
provider: { kind: "codex", model: "gpt-6.1-sol" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
const event = (sourceSeq: number, eventType: string, payload: Record<string, unknown>): PrpEvent => ({
sourceKind: "runner", sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `rate-card-${sourceSeq}`,
turnId: "price-turn", eventType, emittedAt: new Date().toISOString(), payload,
} as PrpEvent);
await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
await delivery.capture(event(2, "turn.completed", {}));
const accepted = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
expect(accepted?.costStatus).toBe("estimated");
const exact = accepted!.costUsdExact;
f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...accepted, usageComplete: accepted!.complete };
await f.completion.completeRun(f.input);
const saved = await read(f.run.id);
expect(saved).toMatchObject({ status: "succeeded", costAccountingPending: false, costAccountedAt: expect.any(Date),
usageJson: { costUsdExact: exact, costStatus: "estimated" } });
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
expect(ledger).toHaveLength(1);
expect(ledger[0].costCents).toBe(100.2);
});
it("retains newer accepted native usage when completion consumed an older receipt", async () => {
const f = await fixture({ runtimeMode: "native", usageJson: { sessionRotated: true, configFreshness: { version: 0 } } });
const runnerInstanceId = randomUUID();
const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId,
provider: { kind: "codex", model: "test-model" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
const event = (sourceSeq: number, eventType: string, inputTokens = 100): PrpEvent => ({
sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `racing-${sourceSeq}`, turnId: "turn-1",
emittedAt: new Date().toISOString(), eventType, payload: eventType === "usage.reported"
? { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens, outputTokens: 10, providerCostUsd: inputTokens / 1000 } } : {},
} as PrpEvent);
await delivery.capture(event(1, "usage.reported"));
await delivery.capture(event(2, "turn.completed"));
const acceptedA = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id });
await delivery.capture(event(3, "usage.reported", 120));
f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...acceptedA, usageComplete: acceptedA!.complete };
await f.completion.completeRun(f.input);
expect(await read(f.run.id)).toMatchObject({ status: "succeeded", costAccountingPending: true, costAccountedAt: null,
usageJson: { inputTokens: 120, costUsd: 0.12, accountingReceiptSequence: 3, accountingReceiptReady: false, sessionRotated: false, configFreshness: { version: 1 } } });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
await delivery.capture(event(4, "turn.completed", 120));
expect(await (await import("../run-cost-accounting.js")).accountRunCost(db, f.run.id)).toBe(true);
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
expect(ledger).toHaveLength(1);
expect(ledger[0].costCents).toBe(12);
});
it.each([
{ timedOut: true, expected: "timed_out", errorCode: "timeout" },
{ signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" },
server/src/services/heartbeat/run-completion.ts
+2 -1
View File
@@ -1,3 +1,4 @@
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js";
import { measureRunPhase } from "../execution-preparation-timing.js";
import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js";
@@ -686,7 +687,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC
cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null,
),
usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) },
usageJson: preserveAcceptedRunUsageProjectionSql({ ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) }),
finishedAt:
persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt,
updatedAt: new Date(),
server/src/services/heartbeat/run-lifecycle.ts
+2
View File
@@ -1,3 +1,4 @@
import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js";
@@ -1202,6 +1203,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
.set({
status,
...patch,
...(patch?.usageJson ? { usageJson: preserveAcceptedRunUsageProjectionSql(patch.usageJson) } : {}),
executionStatusDeliveryId: randomUUID(),
updatedAt: new Date(),
})
server/src/services/heartbeat/run-log-authority.ts
+65
View File
@@ -0,0 +1,65 @@
import { and, eq } from "drizzle-orm";
import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { legacyControllerBootId } from "../legacy-controller-lease.js";
import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js";
export function createRunLogWriterAuthority(
db: Db,
admitted: typeof heartbeatRuns.$inferSelect,
nativeLeaseOwner?: string,
) {
return async () => {
const [run] = await db
.select({
id: heartbeatRuns.id,
runtimeMode: heartbeatRuns.runtimeMode,
controllerBootId: heartbeatRuns.controllerBootId,
leaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt,
logStore: heartbeatRuns.logStore,
logRef: heartbeatRuns.logRef,
})
.from(heartbeatRuns)
.where(
and(
eq(heartbeatRuns.id, admitted.id),
eq(heartbeatRuns.companyId, admitted.companyId),
eq(heartbeatRuns.agentId, admitted.agentId),
eq(heartbeatRuns.status, "running"),
),
);
if (!run) throw new Error("run_log_writer_authority_lost");
if (run.runtimeMode !== "native") {
if (
run.controllerBootId !== legacyControllerBootId ||
!run.leaseExpiresAt ||
run.leaseExpiresAt <= new Date()
)
throw new Error("run_log_writer_authority_lost");
return;
}
const [owner] = await db
.select({
leaseOwner: nativeRunFinalizations.leaseOwner,
bootId: nativeRunFinalizations.controllerBootId,
expiresAt: nativeRunFinalizations.leaseExpiresAt,
})
.from(nativeRunFinalizations)
.where(
and(
eq(nativeRunFinalizations.runId, admitted.id),
eq(nativeRunFinalizations.companyId, admitted.companyId),
),
);
if (!owner && !run.logStore && !run.logRef && !admitted.logStore && !admitted.logRef) return;
const identity = await currentNativeControllerIdentity();
if (
!owner ||
owner.bootId !== identity.bootId ||
!owner.expiresAt ||
owner.expiresAt <= new Date() ||
(nativeLeaseOwner !== undefined && owner.leaseOwner !== nativeLeaseOwner)
)
throw new Error("run_log_writer_authority_lost");
};
}
server/src/services/heartbeat/run-output.test.ts
+149 -9
View File
@@ -7,6 +7,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createDurableRunLogStore } from "../run-log-store.js";
import type { StorageProvider } from "../../storage/types.js";
import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js";
import { generateKeyPairSync } from "node:crypto";
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
@@ -45,15 +48,24 @@ function createRunRowDb(row: Partial<Run>) {
return {
update: () => ({
set: (values: Partial<Run>) => ({
where: async () => { Object.assign(row, values); },
where: async () => {
Object.assign(row, values);
},
}),
}),
} as unknown as Db;
}
const newRunRow = {
id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running",
logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0, lastOutputAt: null,
id: "run-1",
companyId: "company-1",
agentId: "agent-1",
status: "running",
logStore: null,
logRef: null,
logBytes: null,
lastOutputSeq: 0,
lastOutputAt: null,
} as Partial<Run>;
async function beginAttempt(store: ReturnType<typeof createDurableRunLogStore>, row: Partial<Run>) {
@@ -81,11 +93,26 @@ afterEach(async () => {
describe("heartbeat run output log recovery", () => {
it("treats only a run row with no recorded log or output as new", () => {
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(true);
expect(isNewRunLog({ logStore: "local_file", logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
expect(isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false);
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(false);
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(false);
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(
true,
);
expect(
isNewRunLog({
logStore: "local_file",
logRef: "c/a/r.ndjson",
logBytes: null,
lastOutputSeq: 0,
}),
).toBe(false);
expect(
isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 }),
).toBe(false);
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(
false,
);
expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(
false,
);
});
it("makes no remote log recovery request for a new run", async () => {
@@ -116,8 +143,121 @@ describe("heartbeat run output log recovery", () => {
await second.onLog("stdout", "attempt two\n");
await restartedStore.finalize(second.output.handle!);
const lines = objects.get(row.logRef!)!.toString().trim().split("\n").map((line) => JSON.parse(line));
const lines = objects
.get(row.logRef!)!
.toString()
.trim()
.split("\n")
.map((line) => JSON.parse(line));
expect(lines.map((line) => line.chunk)).toEqual(["attempt one\n", "attempt two\n"]);
expect(lines.map((line) => line.seq)).toEqual([1, 2]);
});
});
describe("object-backed diagnostic recording", () => {
it("applies identity redaction to a native key-bearing event before S3 admission", async () => {
const privateKeyPem = generateKeyPairSync("ed25519")
.privateKey.export({ type: "pkcs8", format: "pem" })
.toString();
const row = { ...newRunRow, runtimeMode: "native" } as Run;
const db = createRunRowDb(row) as Db & { select: () => unknown };
db.select = (() => ({
from: () => ({ where: () => ({ orderBy: () => ({ limit: async () => [] }) }) }),
})) as typeof db.select;
const chunks: string[] = [];
const output = createHeartbeatRunOutput(db, {
run: row,
runId: row.id,
issueId: null,
secretKeys: new Set(),
identityRedactor: createAgentIdentityRedactor(privateKeyPem),
runLogStore: {
begin: async () => ({
store: "s3_segments",
logRef: "company/agent/run",
initialSourceCursor: 0,
}),
append: async () => 0,
enqueue: async (_handle, event) => {
chunks.push(event.chunk);
return { committed: Promise.resolve(100) };
},
flush: async () => undefined,
},
appendRunEvent: vi.fn(),
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
});
const handlers = await output.begin(row);
await handlers.onPersistedNativeLog!(
1,
formatNativeRunLogEvent({
schema: "paperclip.prp.event.v1",
schemaVersion: 1,
runId: row.id,
sourceEventId: "source:1",
sourceInstanceId: "source",
sourceKind: "runner",
sourceSeq: 1,
normalizedSessionId: "session",
eventType: "harness.diagnostic",
priority: 1,
emittedAt: new Date().toISOString(),
payload: { text: privateKeyPem },
}),
);
expect(chunks).toHaveLength(1);
expect(chunks[0]).not.toContain(privateKeyPem.split("\n")[1]!);
expect(chunks[0]).toContain("REDACTED");
});
it("admits a burst without serial network waits, publishes after receipts, and coalesces progress", async () => {
const { publishLiveEvent } = await import("../live-events.js");
vi.mocked(publishLiveEvent).mockClear();
const row = { ...newRunRow };
const writes: Array<Partial<Run>> = [];
const db = {
update: () => ({
set: (values: Partial<Run>) => ({
where: async () => {
writes.push(values);
Object.assign(row, values);
},
}),
}),
} as unknown as Db;
const commit: Array<() => void> = [];
const recorder = createHeartbeatRunOutput(db, {
run: row as Run,
runId: row.id!,
issueId: null,
runLogStore: {
begin: async () => ({
store: "s3_segments",
logRef: "company/agent/run",
initialBytes: 0,
initialSequence: 10,
}),
append: () => new Promise<number>((resolve) => commit.push(() => resolve(100))),
},
identityRedactor: { chunk: (_stream, chunk) => chunk, redact: (value) => value },
secretKeys: new Set(),
appendRunEvent: vi.fn(),
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
});
const { onLog } = await recorder.begin(row as Run);
for (let i = 0; i < 100; i++) await onLog("stdout", `line ${i}\n`);
expect(commit).toHaveLength(100);
expect(publishLiveEvent).not.toHaveBeenCalled();
for (const resolve of commit) resolve();
await vi.waitFor(() =>
expect(
vi
.mocked(publishLiveEvent)
.mock.calls.filter(([event]) => event.type === "heartbeat.run.log"),
).toHaveLength(100),
);
await recorder.flushOutputProgress({ force: true });
expect(row.lastOutputSeq).toBe(110);
expect(row.lastOutputBytes).toBe(10000);
expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2);
});
});
server/src/services/heartbeat/run-output.ts
+189 -80
View File
@@ -5,9 +5,15 @@ import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../l
import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { touchHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js";
import { publishLiveEvent } from "../live-events.js";
import type { RunLogHandle, RunLogStore } from "../run-log-store.js";
import { isRunLogStoreType, type RunLogHandle, type RunLogStore } from "../run-log-store.js";
import { appendExcerpt, compactRunLogChunk } from "./run-log.js";
import { isHeartbeatRunRuntimeStatusActive, publishHeartbeatRunRuntimeProgress, type createHeartbeatLifecycle } from "./run-lifecycle.js";
import { createNativeRunLogExporter } from "../native-runtime/native-run-log-export.js";
import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
import {
isHeartbeatRunRuntimeStatusActive,
publishHeartbeatRunRuntimeProgress,
type createHeartbeatLifecycle,
} from "./run-lifecycle.js";
const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024;
export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000;
@@ -19,11 +25,13 @@ export interface HeartbeatRunOutputInput {
run: Run;
runId: string;
issueId: string | null;
runLogStore: Pick<RunLogStore, "begin" | "append">;
runLogStore: Pick<RunLogStore, "begin" | "append" | "enqueue" | "flush">;
identityRedactor: Pick<ReturnType<typeof createAgentIdentityRedactor>, "chunk" | "redact">;
secretKeys: Set<string>;
appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
getCurrentUserRedactionOptions: () => Promise<CurrentUserRedactionOptions>;
assertWriterAuthority?: () => Promise<void>;
runtimeLogSecretValues?: readonly string[];
}
/**
@@ -35,15 +43,26 @@ export interface HeartbeatRunOutputInput {
export function isNewRunLog(
run: Pick<Run, "logStore" | "logRef" | "logBytes" | "lastOutputSeq">,
): boolean {
return run.logStore == null
&& run.logRef == null
&& run.logBytes == null
&& Number(run.lastOutputSeq ?? 0) === 0;
return (
run.logStore == null &&
run.logRef == null &&
run.logBytes == null &&
Number(run.lastOutputSeq ?? 0) === 0
);
}
/** One recorder per execution attempt; the log store itself stays service-scoped. */
export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) {
const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input;
const {
run,
runId,
issueId,
runLogStore,
identityRedactor,
secretKeys,
appendRunEvent,
getCurrentUserRedactionOptions,
} = input;
let handle: RunLogHandle | null = null;
let stdoutExcerpt = "";
let stderrExcerpt = "";
@@ -59,7 +78,14 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
} | null;
} = { pending: null };
let persistedLogBytes = Number(run.logBytes ?? 0);
let progressWrite: Promise<void> | null = null;
let recordingFailureReported = false;
const flushOutputProgress = async (opts?: { force?: boolean }) => {
if (progressWrite) {
await progressWrite;
if (opts?.force) return flushOutputProgress(opts);
return;
}
const pendingOutputProgress = outputProgressState.pending;
if (!pendingOutputProgress) return;
const shouldFlush =
@@ -68,7 +94,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >=
ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS;
if (!shouldFlush) return;
await db
progressWrite = db
.update(heartbeatRuns)
.set({
lastOutputAt: pendingOutputProgress.at,
@@ -77,111 +103,181 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
lastOutputBytes: pendingOutputProgress.bytes,
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, run.id));
lastOutputFlushAt = pendingOutputProgress.at;
outputProgressState.pending = null;
.where(eq(heartbeatRuns.id, run.id))
.then(() => {
lastOutputFlushAt = pendingOutputProgress.at;
if (outputProgressState.pending === pendingOutputProgress)
outputProgressState.pending = null;
})
.finally(() => {
progressWrite = null;
});
await progressWrite;
};
// Bind handlers to the run row persisted at execution start. Progress writes
// above use only the immutable run ID from the initial row.
async function begin(currentRun: Run) {
const run = currentRun;
if (run.logStore != null && !isRunLogStoreType(run.logStore))
throw new Error("Run log backend is not supported");
handle = await runLogStore.begin({
companyId: run.companyId,
agentId: run.agentId,
runId,
newRun: isNewRunLog(run),
assertWriterAuthority: input.assertWriterAuthority,
previousStore: run.logStore ?? undefined,
});
outputSeq = handle.initialSequence ?? outputSeq;
persistedLogBytes = handle.initialBytes ?? persistedLogBytes;
await db
.update(heartbeatRuns)
.set({
logStore: handle.store,
logRef: handle.logRef,
...(handle.store === "s3_segments" ? { logBytes: null, logSha256: null } : {}),
updatedAt: new Date(),
})
.where(eq(heartbeatRuns.id, runId));
const currentUserRedactionOptions =
await getCurrentUserRedactionOptions();
const appendIdentityRedactedLog = async (stream: "stdout" | "stderr", chunk: string) => {
const currentUserRedactionOptions = await getCurrentUserRedactionOptions();
if (
handle.store === "s3_segments" &&
run.runtimeMode === "native" &&
input.runtimeLogSecretValues?.length
) {
const registry = createRunSecretRedactionRegistry(db);
for (const value of new Set(input.runtimeLogSecretValues))
await registry.register(run.companyId, runId, value);
}
const reportRecordingFailure = async () => {
if (recordingFailureReported) return;
recordingFailureReported = true;
await appendRunEvent(currentRun, {
eventType: "lifecycle",
stream: "system",
level: "warn",
message: "Run log persistence is incomplete",
payload: { reason: "object_log_persistence_failed" },
}).catch(() => undefined);
};
const appendIdentityRedactedLog = async (
stream: "stdout" | "stderr",
chunk: string,
waitForCommit = true,
sourceCursor?: number,
) => {
const sanitizedChunk = compactRunLogChunk(
redactCurrentUserText(chunk, currentUserRedactionOptions),
);
if (stream === "stdout")
stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
if (stream === "stderr")
stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk);
if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk);
const ts = new Date().toISOString();
outputSeq += 1;
const chunkSeq = outputSeq;
let appendedBytes = 0;
let committed: Promise<number> = Promise.resolve(0);
if (handle) {
appendedBytes = await runLogStore.append(handle, {
const event = {
stream,
chunk: sanitizedChunk,
ts,
seq: chunkSeq,
});
sourceCursor,
};
committed = runLogStore.enqueue
? (await runLogStore.enqueue(handle, event)).committed
: runLogStore.append(handle, event);
}
const recorded = committed.then(async (appendedBytes) => {
persistedLogBytes += appendedBytes;
}
outputProgressState.pending = {
at: new Date(ts),
seq: chunkSeq,
stream,
bytes: persistedLogBytes,
};
await flushOutputProgress();
// Streamed CLI output is real run activity: keep the in-memory
// runtime status ("Working... / X ago") fresh between structured
// events so sandbox runs with mid-run log streaming never show a
// minutes-stale timestamp. Throttled to avoid churning the live
// event stream on every 250ms tail chunk.
const logActivityAt = new Date(ts);
if (
isHeartbeatRunRuntimeStatusActive(run.status) &&
logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
) {
lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
const touchedStatus = touchHeartbeatRunRuntimeStatus({
companyId: run.companyId,
issueId,
agentId: run.agentId,
runId: run.id,
at: logActivityAt,
});
if (touchedStatus)
publishHeartbeatRunRuntimeProgress(touchedStatus);
}
const payloadChunk =
sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
? sanitizedChunk.slice(
sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES,
)
: sanitizedChunk;
publishLiveEvent({
companyId: run.companyId,
type: "heartbeat.run.log",
payload: {
runId: run.id,
agentId: run.agentId,
issueId,
ts,
outputProgressState.pending = {
at: new Date(ts),
seq: chunkSeq,
stream,
chunk: payloadChunk,
logRef: handle?.logRef,
logStore: handle?.store,
truncated: payloadChunk.length !== sanitizedChunk.length,
},
bytes: persistedLogBytes,
};
await flushOutputProgress();
// Streamed CLI output is real run activity: keep the in-memory
// runtime status ("Working... / X ago") fresh between structured
// events so sandbox runs with mid-run log streaming never show a
// minutes-stale timestamp. Throttled to avoid churning the live
// event stream on every 250ms tail chunk.
const logActivityAt = new Date(ts);
if (
isHeartbeatRunRuntimeStatusActive(run.status) &&
logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >=
ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS
) {
lastLogRuntimeStatusTouchMs = logActivityAt.getTime();
const touchedStatus = touchHeartbeatRunRuntimeStatus({
companyId: run.companyId,
issueId,
agentId: run.agentId,
runId: run.id,
at: logActivityAt,
});
if (touchedStatus) publishHeartbeatRunRuntimeProgress(touchedStatus);
}
const payloadChunk =
sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES
? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES)
: sanitizedChunk;
publishLiveEvent({
companyId: run.companyId,
type: "heartbeat.run.log",
payload: {
runId: run.id,
agentId: run.agentId,
issueId,
ts,
seq: chunkSeq,
stream,
chunk: payloadChunk,
logRef: handle?.logRef,
logStore: handle?.store,
truncated: payloadChunk.length !== sanitizedChunk.length,
},
});
});
if (waitForCommit) await recorded;
else void recorded.catch(reportRecordingFailure);
};
const onLog = (stream: "stdout" | "stderr", chunk: string) =>
appendIdentityRedactedLog(stream, identityRedactor.chunk(stream, chunk));
const onLog = (stream: "stdout" | "stderr", chunk: string) => {
const external = handle?.store === "s3_segments";
const recorded = appendIdentityRedactedLog(
stream,
identityRedactor.chunk(stream, chunk),
!external,
);
return external ? recorded.catch(reportRecordingFailure) : recorded;
};
const nativeExporter =
handle.store === "s3_segments" && run.runtimeMode === "native"
? createNativeRunLogExporter({
db,
companyId: run.companyId,
runId,
afterCursor: handle.initialSourceCursor ?? 0,
enqueue: (cursor, chunk) =>
appendIdentityRedactedLog(
"stdout",
identityRedactor.chunk("stdout", chunk),
false,
cursor,
),
flush: () => runLogStore.flush!(handle!),
})
: undefined;
if (nativeExporter) await nativeExporter.recover();
const onPersistedNativeLog = nativeExporter
? (cursor: number, chunk: string) =>
nativeExporter.record(cursor, chunk).catch(reportRecordingFailure)
: undefined;
const onAdapterMeta = async (meta: AdapterInvocationMeta) => {
meta = identityRedactor.redact(meta);
if (meta.env && secretKeys.size > 0) {
@@ -214,16 +310,29 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
});
};
return { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent };
return {
currentUserRedactionOptions,
appendIdentityRedactedLog,
onLog,
onPersistedNativeLog,
onAdapterMeta,
onAdapterEvent,
};
}
return {
begin,
get handle() { return handle; },
get handle() {
return handle;
},
outputProgressState,
flushOutputProgress,
// Finalization awaits I/O and can receive more output before persisting excerpts.
get stdoutExcerpt() { return stdoutExcerpt; },
get stderrExcerpt() { return stderrExcerpt; },
get stdoutExcerpt() {
return stdoutExcerpt;
},
get stderrExcerpt() {
return stderrExcerpt;
},
};
}
server/src/services/heartbeat/run-state.ts
+3
View File
@@ -355,6 +355,9 @@ const heartbeatRunLogAccessColumns = {
...heartbeatRunReadAccessColumns,
logStore: heartbeatRuns.logStore,
logRef: heartbeatRuns.logRef,
runtimeMode: heartbeatRuns.runtimeMode,
status: heartbeatRuns.status,
logBytes: heartbeatRuns.logBytes,
} as const;
const heartbeatRunIssueControlColumns = {
server/src/services/heartbeat/runtime-execution.test.ts
+35 -1
View File
@@ -1,8 +1,10 @@
import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
import { agents, budgetPolicies, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db";
import { eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { trace } from "@opentelemetry/api";
@@ -213,6 +215,38 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () =
expectCleanup(input);
});
it("returns the accepted exact native rate-card price and preserves its live budget hold", async () => {
const { input, run } = await fixture(true);
input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home };
input.runtime.nativeExecution!.provider = { kind: "codex", model: "gpt-6.1-sol" };
await db.insert(budgetPolicies).values({ companyId: run.companyId, scopeType: "agent", scopeId: input.agent.id,
metric: "billed_cents", windowKind: "lifetime", amount: 100, notifyEnabled: false });
let exact: string | undefined;
vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => {
expect(native.onUsage).toBeUndefined();
const delivery = await createNativeUsageDelivery(db, { companyId: run.companyId, runId: run.id,
runnerInstanceId: native.runnerInstanceId, provider: native.execution.provider,
billing: { provider: "openai", biller: "openai", billingType: "metered_api" } });
const event = (sourceSeq: number, eventType: string, payload: Record<string, unknown>): PrpEvent => ({
sourceKind: "runner", sourceInstanceId: native.runnerInstanceId, runId: run.id, sourceSeq, sourceEventId: `priced-${sourceSeq}`,
turnId: "priced-turn", eventType, emittedAt: new Date().toISOString(), payload,
} as PrpEvent);
await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true,
runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } }));
await delivery.capture(event(2, "turn.completed", {}));
const saved = (await read(run.id)).usageJson!;
exact = saved.costUsdExact as string;
expect(Number(exact)).toBe(1.002);
expect(saved.budgetExposure).toMatchObject({ costCents: "100.2000000" });
expect((await db.select().from(agents).where(eq(agents.id, input.agent.id)))[0].lifecycleHolds).toContain("budget");
return { exitCode: 0, signal: null, timedOut: false, usageComplete: true, costUsd: null, model: "gpt-6.1-sol" };
});
const result = await executeHeartbeatRuntime(db, input);
if (!result.dispatched) throw new Error("expected native dispatch");
expect(result.adapterResult).toMatchObject({ costUsdExact: exact, costStatus: "estimated", usageComplete: true,
pricingProvenance: { source: "rate_card" }, usage: { inputTokens: 1000, outputTokens: 100000 } });
});
it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => {
const { input } = await fixture(true);
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId,
server/src/services/heartbeat/runtime-execution.ts
+20 -7
View File
@@ -1,3 +1,4 @@
import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js";
import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
import { writeExecutionControl } from "../execution-control-notifications.js";
import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js";
@@ -33,7 +34,7 @@ import {
NativeWorkspaceFinalizationOwnershipLostError,
type NativeWorkspaceFinalizationOwnership,
} from "../native-runtime/native-workspace-finalization-ownership.js";
import { createRunUsageRecorder } from "../usage-receipts.js";
import { createRunUsageRecorder, readAcceptedRunUsageCheckpoint } from "../usage-receipts.js";
import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result";
import { hasWorkspaceRestoreFailure } from "@paperclipai/shared";
import { isConversation } from "../agent-conversations.js";
@@ -172,6 +173,7 @@ export interface HeartbeatRuntimeExecutionInput {
>;
};
output: Pick<AdapterExecutionContext, "onLog"> & {
onPersistedNativeLog?: (cursor: number, chunk: string) => Promise<void>;
onAdapterEvent: NonNullable<AdapterExecutionContext["onEvent"]>;
onAdapterMeta: AdapterExecutionContext["onMeta"];
identityRedactor: ReturnType<typeof createAgentIdentityRedactor>;
@@ -495,9 +497,13 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
adapterFinalizeOutcome = status;
};
const usageRecorder = await measureRunPhase("execution.usage_startup", () =>
const producerOwnsUsage = usesNativeUsageDelivery({
runnerd: nativeRuntimeResolution.kind === "native" && agent.adapterType === "paperclip_runner",
remote: executionTarget?.kind === "remote", provider: nativeExecution?.provider,
});
const usageRecorder = producerOwnsUsage ? undefined : await measureRunPhase("execution.usage_startup", () =>
createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType }));
input.effects.onUsageCaptureReady(usageRecorder.persistFailure);
input.effects.onUsageCaptureReady(usageRecorder?.persistFailure ?? (async () => {}));
let adapterResult: AdapterExecutionResult;
const runGoalControlRequestId = readNonEmptyString(
context.goalControlRequestId,
@@ -629,10 +635,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
goalCheckpointSession.current = { params, displayId };
},
onLog,
onPersistedNativeLog: input.output.onPersistedNativeLog,
onEvent: onAdapterEvent,
instructionWorkingCopy: nativeInstructionWorkingCopy(),
onUsage: async receipt => { await usageRecorder.capture(receipt); },
onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
preparationSpans: nativeRunnerPreparationSpans,
// Bootstrap with executable/home discovery while keeping
// configured provider values and the server-selected
@@ -834,7 +841,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
onLog,
onMeta: onAdapterMeta,
onEvent: onAdapterEvent,
onUsage: async receipt => { await usageRecorder.capture(receipt); },
onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined,
onExecutionPhase: executionControl.phases.enter,
startupTraceContext: getStartupTraceContext(),
onRuntimeProgress: async (progress) => {
@@ -913,8 +920,14 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave };
if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) {
const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
if (usageRecorder) {
const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult));
adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete };
} else {
const accepted = await measureRunPhase("execution.usage", () => readAcceptedRunUsageCheckpoint(db, { companyId: run.companyId, runId: run.id }));
adapterResult = { ...adapterResult, ...accepted, usageComplete: accepted?.complete ?? false,
...(!accepted ? { costStatus: "unpriced" as const } : {}) };
}
} else {
// Stop may already own the terminal result. Preserve its metadata
// while durably recording the proof needed to release admission.
server/src/services/issues.ts
+11 -5
View File
@@ -189,7 +189,8 @@ import {
resolveIssueGoalId,
resolveNextIssueGoalId,
} from "./issue-goal-fallback.js";
import { getRunLogStore } from "./run-log-store.js";
import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js";
import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js";
import { getDefaultCompanyGoal } from "./goals.js";
import { assertAssignableAgent } from "./agent-assignability.js";
import {
@@ -6712,13 +6713,14 @@ export async function readIssueCommentRunLogText(run: {
logStore: string | null;
logRef: string | null;
logBytes: number | null;
}) {
if (run.logStore !== "local_file" || !run.logRef) return "";
}, db?: Db) {
if (!isRunLogStoreType(run.logStore) || !run.logRef) return "";
// A timed-out finalization leaves size unknown even when earlier entries
// exist. Read those logs within the same byte budget as a known-size log.
if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return "";
const logRef = run.logRef;
const storeType = run.logStore;
const store = getRunLogStore();
let offset = 0;
let content = "";
@@ -6727,6 +6729,8 @@ export async function readIssueCommentRunLogText(run: {
let readTimer: NodeJS.Timeout | undefined;
const readChunks = async () => {
if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal)
.catch(() => { controller.signal.throwIfAborted(); });
while (nextOffset !== undefined) {
controller.signal.throwIfAborted();
const remainingBytes =
@@ -6734,7 +6738,7 @@ export async function readIssueCommentRunLogText(run: {
Buffer.byteLength(content, "utf8");
if (remainingBytes <= 0) break;
const chunk = await store.read(
{ store: "local_file", logRef },
{ store: storeType, logRef },
{
offset,
limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes),
@@ -7259,7 +7263,7 @@ export function issueService(db: Db) {
);
await Promise.all(
batch.map(async (run) => {
logByRunId.set(run.runId, await readIssueCommentRunLogText(run));
logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db));
}),
);
}
@@ -12587,6 +12591,7 @@ export function issueService(db: Db) {
order?: "asc" | "desc";
limit?: number | null;
humanOnly?: boolean;
companyId?: string;
},
) => {
const order = opts?.order === "asc" ? "asc" : "desc";
@@ -12597,6 +12602,7 @@ export function issueService(db: Db) {
: null;
const conditions = [eq(issueComments.issueId, issueId)];
if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId));
if (opts?.humanOnly) conditions.push(or(
eq(issueComments.authorType, "user"),
and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)),
server/src/services/native-runtime/__fixtures__/usage-receiver.ts
+45
View File
@@ -0,0 +1,45 @@
import { resolve } from "node:path";
import { createDb } from "@paperclipai/db";
import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint } from "../../../vendor/paperclip-runner/index.js";
import { createNativeUsageDelivery } from "../native-usage-delivery.js";
type Start = {
kind: "start"; requestId: number; connectionString: string; directory: string; port?: number;
identity: ConstructorParameters<typeof DurablePrpControlPlane>[0]["identity"];
input: Parameters<typeof createNativeUsageDelivery>[1];
checkpoint?: [string, string][];
};
let core: DurablePrpControlPlane | undefined;
let checkpoint: DurablePrpCheckpoint | undefined;
process.on("message", async (message: Start | { kind: "command"; requestId: number; type: string; commandId: string } | { kind: "stop"; requestId: number }) => {
try {
if (message.kind === "start") {
const delivery = await createNativeUsageDelivery(createDb(message.connectionString), message.input);
if (message.checkpoint) decodeDurablePrpCheckpoint(new Map(message.checkpoint)).restoreSessionDirectory(message.directory);
core = new DurablePrpControlPlane({ stateDirectory: resolve(message.directory, "control-plane"), identity: message.identity,
expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"c".repeat(64)}`,
onCommittedEvent: async event => {
try { await delivery.capture(event); }
catch (error) { process.send?.({ kind: "capture-failed", cause: error instanceof Error ? error.message : String(error) }); throw error; }
},
});
core.setCheckpointPersistence(async saved => { checkpoint = saved; });
await core.start(message.port);
const ticket = message.checkpoint ? undefined : core.issueBootstrapTicket();
await core.flushCheckpoint();
process.send?.({ requestId: message.requestId, connectUrl: core.connectUrl, ticket });
} else if (message.kind === "command") {
core!.queueCommand(message.type, {}, message.commandId, true);
process.send?.({ requestId: message.requestId });
} else {
await core?.stop();
await core?.drainPendingConnectionProcessing();
await core?.flushCheckpoint();
process.send?.({ requestId: message.requestId, checkpoint: [...checkpoint!.records] }, () => process.exit(0));
}
} catch (error) {
process.send?.({ requestId: message.requestId, error: error instanceof Error ? error.message : String(error) });
}
});
process.send?.({ kind: "ready" });
server/src/services/native-runtime/native-github-access.test.ts
+10 -3
View File
@@ -3,6 +3,7 @@ import { spawn, execFile } from "node:child_process";
import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";
import { promisify } from "node:util";
import type { CommandManagedDuplexChannel, CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime";
import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability";
import { startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -39,7 +40,7 @@ function localDuplexChannel(command: readonly string[], cwd: string): CommandMan
async function broker(resolveCredentials = vi.fn(async (binding: ReturnType<typeof run>) => ({
status: "available", env: { GH_TOKEN: `fixture-${binding.runId}` },
})), remote = false, bridgeUnavailable = false, onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>,
duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean; recorder?: DuplexObservabilityRecorder }) {
const root = await mkdtemp(path.join(tmpdir(), "native-github-reuse-"));
cleanups.push(() => rm(root, { recursive: true, force: true }));
const bin = path.join(root, "real-bin");
@@ -61,7 +62,7 @@ duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) {
});
const target = remote ? { kind: "remote" as const, transport: "sandbox" as const, providerKey: "test",
remoteCwd: root, runner: { execute, ...(duplex ? { openDuplexChannel } : {}) }, streamRunLogs: false,
...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, effectiveCapabilities: {
...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, duplexObservabilityRecorder: duplex.recorder, effectiveCapabilities: {
reusableLeases: true, nativeSyncIn: false, nativeSyncOut: false, persistentProcessSessions: false,
independentControlCommands: true, incrementalSessionOutput: false, concurrentSyncOperations: false,
duplexCommandStream: duplex.capability, runnerWebSocketIngress: false,
@@ -122,9 +123,13 @@ it.each([false, true])("resolves managed GitHub credentials from OpenCode shell
expect(childEnvironment.githubTokenPresent).toBe(false);
}, 30000);
it("uses the selected duplex transport across credential rotation without file polling", async () => {
const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true });
const recorder = { recordSpan: vi.fn(), incrementCounter: vi.fn(), emitEvent: vi.fn() };
const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true, recorder });
expect(b.bridgeModes).toEqual(["http2_v1"]);
expect(b.openDuplexChannel).toHaveBeenCalledOnce();
expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.channel_open", dimensions: {
provider: "other", transport: "http2", outcome: "ok",
} }));
const commandsAfterStartup = b.execute.mock.calls.length;
expect((await request(b)).status).toBe(403);
const releaseA = b.activate(run("a"));
@@ -135,6 +140,8 @@ it("uses the selected duplex transport across credential rotation without file p
releaseA();
expect(await (await request(b)).json()).toMatchObject({ env: { GH_TOKEN: "fixture-b" } });
expect(b.resolveCredentials.mock.calls.map(([binding]) => binding.runId)).toEqual(["a", "b"]);
expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.request" }));
expect(recorder.emitEvent).not.toHaveBeenCalledWith(expect.objectContaining({ dimensions: expect.objectContaining({ transport: "file" }) }));
expect(b.execute.mock.calls).toHaveLength(commandsAfterStartup);
releaseB();
await b.stop();
server/src/services/native-runtime/native-github-access.ts
+2
View File
@@ -3,6 +3,7 @@ import { createServer } from "node:http";
import {
cleanupGitHubOperationLaunchers,
adapterExecutionTargetEnablesSandboxDuplexBridge,
adapterExecutionTargetDuplexObservabilityRecorder,
githubOperationLauncherDirectory,
prepareGitHubOperationLaunchers,
startAdapterExecutionTargetPaperclipBridge,
@@ -107,6 +108,7 @@ export async function createNativeGitHubAccess(input: {
hostApiToken: token,
hostApiUrl: url,
enableSandboxDuplexBridge: duplexRequested,
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(input.target),
onLog: input.onLog,
});
if (input.target?.kind === "remote" && !bridge) {
server/src/services/native-runtime/native-response-presentation-work.ts
+14 -5
View File
@@ -7,6 +7,7 @@ import type { NativeResponsePresentationScope } from "../native-response-present
import { onCommittedChatResponseAuthorityRegistered } from "../durable-chat-wakeup.js";
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
import { recoverCommittedNativeChatResponse } from "./native-run-finalizer.js";
import { measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
export function registerNativeResponsePresentationWork(db: Db,
coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, canRun: () => boolean) {
@@ -20,27 +21,35 @@ export function nativeResponsePresentationWork(db: Db) {
const query = nativeResponsePresentationCandidates(scope, after, runId).as("pending_native_response");
return db.select().from(query).orderBy(asc(query.id));
};
async function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise<Entry[]> {
const work = () => loadWithinTrace(scope, signal, canRun);
return scope?.runId ? withRunResponsePresentationTiming(scope.runId, work) : work();
}
async function loadWithinTrace(scope: NativeResponsePresentationScope | undefined, signal: AbortSignal | undefined,
canRun: () => boolean): Promise<Entry[]> {
const entries: Entry[] = [];
let after: string | undefined;
while (canRun() && !signal?.aborted) {
const rows = await candidates(scope, after);
const rows = await measureRunPhase("response.resolve", () => candidates(scope, after));
for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, agentId: row.agentId, dueAt: row.nextAttemptAt?.getTime() ?? 0 });
if (rows.length < 100) return entries;
after = rows[rows.length - 1].id;
}
throw new Error("Native response presentation discovery stopped");
}
async function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
function check(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
return withRunResponsePresentationTiming(entry.id, () => checkWithinTrace(entry, signal));
}
async function checkWithinTrace(entry: Entry, signal?: AbortSignal): Promise<Entry | null> {
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
const scope = { companyId: entry.companyId, issueId: entry.issueId };
const [current] = await candidates(scope, undefined, entry.id);
const [current] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
if (!current) return null;
if (current.nextAttemptAt && current.nextAttemptAt.getTime() > Date.now()) return { ...entry, dueAt: current.nextAttemptAt.getTime() };
if (signal?.aborted) throw new Error("Native response presentation recovery stopped");
const recoveryStartedAt = Date.now();
await recoverCommittedNativeChatResponse(db, { ...scope, runId: entry.id, onlyWhenDue: true });
const [remaining] = await candidates(scope, undefined, entry.id);
const [remaining] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id));
return remaining?.nextAttemptAt && remaining.nextAttemptAt.getTime() > recoveryStartedAt
? { ...entry, dueAt: remaining.nextAttemptAt.getTime() } : null;
}
server/src/services/native-runtime/native-run-finalizer.ts
+14 -2
View File
@@ -67,6 +67,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
import { issueService } from "../issues.js";
import { publishChatPublicationCommitSignal } from "../chat-publication-reconciliation.js";
import { nativeSha256 } from "./canonical.js";
import { beginRunPhase, measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js";
import {
readNativeBoardResponseWaitOrigin,
readNativeBoardResponseWaitSource,
@@ -736,11 +737,20 @@ export async function repairCommittedNativeChatResponse(
export async function recoverCommittedNativeChatResponse(
db: Db,
input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
): Promise<boolean> {
return withRunResponsePresentationTiming(input.runId, () => recoverCommittedNativeChatResponseInScope(db, input));
}
async function recoverCommittedNativeChatResponseInScope(
db: Db,
input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean },
): Promise<boolean> {
let agentId: string | null = null;
let materialized = false;
const entered = beginRunPhase("response.transaction.entry");
try {
materialized = await db.transaction(async (transaction) => {
materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => {
entered();
const tx = transaction as unknown as Db;
@@ -1022,10 +1032,12 @@ export async function recoverCommittedNativeChatResponse(
await projectCommittedRun({ db: tx, run, coordinator });
agentId = run.agentId;
return true;
});
}));
} catch (error) {
if (error instanceof CommittedChatResponseAuthorizationError) return false;
throw error;
} finally {
entered();
}
if (materialized && agentId)
publishChatPublicationCommitSignal({
server/src/services/native-runtime/native-run-log-export.integration.test.ts
+474
View File
@@ -0,0 +1,474 @@
import { randomUUID, createHash, generateKeyPairSync } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import {
createDb,
heartbeatRunEvents,
heartbeatRuns,
agentIdentityKeys,
type Db,
} from "@paperclipai/db";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
S3Client,
CreateBucketCommand,
DeleteBucketCommand,
DeleteObjectsCommand,
ListObjectsV2Command,
} from "@aws-sdk/client-s3";
import { createS3StorageProvider } from "../../storage/s3-provider.js";
import { createObjectRunLogStore } from "../object-run-log-store.js";
import { createNativeRunLogExporter, formatNativeRunLogEvent } from "./native-run-log-export.js";
import { recoverSettledNativeRunLog, readRecoverableRunLog } from "./native-run-log-recovery.js";
import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js";
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
import { createNativeRunLogTestTables } from "../../__tests__/helpers/native-run-log-database.js";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import type { StorageProvider } from "../../storage/types.js";
import { appendNativeCredentialUpdate } from "./native-credential-updates.js";
import { appendRetainedNativeCleanupEvent } from "./native-session-executor.js";
import { idleWorkSnapshot } from "../task-admission.js";
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
import { localEncryptedProvider } from "../../secrets/local-encrypted-provider.js";
import { createHeartbeatRunOutput } from "../heartbeat/run-output.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { ObjectJournalConflictError } from "../object-byte-journal.js";
const url = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL");
const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT;
describe.skipIf(!url || !endpoint)("native run log source recovery", () => {
const schema = `native_log_${randomUUID().replaceAll("-", "")}`,
bucket = `pc-native-log-${randomUUID()}`;
let db: Db,
client: S3Client,
bucketCreated = false;
beforeAll(async () => {
db = createDb(url!, { maxConnections: 1, idleTimeoutSeconds: 0 });
await db.execute(sql.raw(`create schema ${schema}`));
await db.execute(sql.raw(`set search_path=${schema}`));
await createNativeRunLogTestTables(db);
client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true });
await client.send(new CreateBucketCommand({ Bucket: bucket }));
bucketCreated = true;
});
afterEach(() => vi.unstubAllEnvs());
afterAll(async () => {
if (client && bucketCreated) {
for (;;) {
const page = await client.send(new ListObjectsV2Command({ Bucket: bucket }));
if (!page.Contents?.length) break;
await client.send(
new DeleteObjectsCommand({
Bucket: bucket,
Delete: { Objects: page.Contents.map((item) => ({ Key: item.Key! })) },
}),
);
}
await client.send(new DeleteBucketCommand({ Bucket: bucket }));
client.destroy();
}
if (db) {
await db.execute(sql.raw(`drop schema ${schema} cascade`));
await db.$client.end();
}
});
async function fixture() {
const binding = {
companyId: randomUUID(),
runId: randomUUID(),
agentId: randomUUID(),
sessionId: randomUUID(),
issueId: randomUUID(),
sourceInstanceId: randomUUID(),
controlPlaneSourceInstanceId: randomUUID(),
completionContractId: randomUUID(),
completionContractSha256: "test",
};
await db.execute(
sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${binding.runId},${binding.companyId},${binding.agentId},'running')`,
);
const provider = createS3StorageProvider({
endpoint,
bucket,
region: "us-east-1",
forcePathStyle: true,
});
const store = (storage: StorageProvider = provider) =>
createObjectRunLogStore({
provider: storage,
keyPrefix: "logs",
flushMs: 30_000,
assertWriterAuthority: async (input) => {
await input.assertWriterAuthority?.();
},
});
const begin = { companyId: binding.companyId, agentId: binding.agentId, runId: binding.runId };
const port = new PaperclipControlPlanePort(db, binding);
const event = (seq: number): PrpEvent => ({
schema: "paperclip.prp.event.v1",
schemaVersion: 1,
sourceInstanceId: binding.sourceInstanceId,
sourceEventId: `${binding.sourceInstanceId}:${seq}`,
sourceSeq: seq,
sourceKind: "runner",
runId: binding.runId,
normalizedSessionId: binding.sessionId,
eventType: "item.delta",
priority: 2,
emittedAt: new Date().toISOString(),
payload: { text: `step ${seq} 再開` },
});
return { binding, store, begin, port, event, provider };
}
it("replays acknowledged native records into a new writer, repairs gaps, and ignores late duplicates", async () => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin);
await db.execute(
sql`update heartbeat_runs set log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
);
const events = [f.event(1), f.event(2), f.event(3)];
const cursors = await Promise.all(events.slice(0, 2).map((event) => f.port.appendEvent(event)));
const receipt = await a.enqueue!(h, {
stream: "stdout",
chunk: formatNativeRunLogEvent(events[0]!),
ts: "then",
seq: 1,
sourceCursor: cursors[0]!.cursor,
});
await a.flush!(h);
await receipt.committed;
const b = f.store(),
successor = await b.begin(f.begin);
let seq = successor.initialSequence!;
const exporter = createNativeRunLogExporter({
db,
companyId: f.binding.companyId,
runId: f.binding.runId,
afterCursor: successor.initialSourceCursor!,
enqueue: async (sourceCursor, chunk) => {
const queued = await b.enqueue!(successor, {
stream: "stdout",
chunk,
ts: "now",
seq: ++seq,
sourceCursor,
});
void queued.committed.catch(() => undefined);
},
flush: () => b.flush!(successor),
});
const third = await f.port.appendEvent(events[2]!);
await exporter.record(third.cursor, formatNativeRunLogEvent(events[2]!));
await exporter.record(cursors[0]!.cursor, formatNativeRunLogEvent(events[0]!));
await b.finalize(successor);
const entries = (await b.read(successor)).content
.trim()
.split("\n")
.map((line) => JSON.parse(JSON.parse(line).chunk).event);
expect(entries.map((event) => event.sourceSeq)).toEqual([1, 2, 3]);
expect(
await db
.select()
.from(heartbeatRunEvents)
.where(eq(heartbeatRunEvents.runId, f.binding.runId)),
).toHaveLength(3);
});
it("repairs a settled run on demand after its controller lease expires, without extending source retention", async () => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin);
await f.port.appendEvent(f.event(1));
await db.execute(
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
);
await db.execute(
sql`insert into native_run_finalizations values(${f.binding.runId},${f.binding.companyId},now()+interval '1 minute')`,
);
const b = f.store();
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
expect((await b.read(h)).content).toBe("");
await db.execute(
sql`update native_run_finalizations set lease_expires_at=now()-interval '1 second' where run_id=${f.binding.runId}`,
);
await Promise.all(
[1, 2].map(() =>
recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })),
),
);
expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
const [run] = await db
.select({ bytes: heartbeatRuns.logBytes })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, f.binding.runId));
expect(run!.bytes).toBeGreaterThan(0);
});
it.each(["hash", "payload", "identity"])(
"rejects a corrupt canonical source %s without marking the partial archive complete",
async (kind) => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin);
await f.port.appendEvent(f.event(1));
if (kind === "hash")
await db.execute(
sql`update heartbeat_run_events set source_payload_sha256='wrong' where run_id=${f.binding.runId}`,
);
else if (kind === "payload")
await db.execute(
sql`update heartbeat_run_events set payload='{}'::jsonb where run_id=${f.binding.runId}`,
);
else
await db.execute(
sql`update heartbeat_run_events set source_instance_id='wrong' where run_id=${f.binding.runId}`,
);
await db.execute(
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
);
await expect(
recoverSettledNativeRunLog(db, f.store(), h.logRef, async () => ({ enabled: false })),
).rejects.toThrow("integrity");
const [run] = await db
.select({ bytes: heartbeatRuns.logBytes })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, f.binding.runId));
expect(run!.bytes).toBeNull();
},
);
it("ignores credential and retained cleanup projections when replaying canonical log sources", async () => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin);
await f.port.appendEvent(f.event(1));
const credential = (sourceSeq: number): PrpEvent => ({
...f.event(sourceSeq),
schema: "paperclip.prp.event.v3",
schemaVersion: 3,
priority: 0,
eventType: "auth.credentials.updated",
payload: {
schema: "paperclip.auth.credentials.updated.v1",
provider: "codex",
bindingId: "test-auth",
revision: 1,
detection: "credential_store",
},
});
await f.port.appendEvent(credential(2));
await db.execute(sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef},
native_session_id=${f.binding.sessionId}, runner_instance_id=${f.binding.sourceInstanceId} where id=${f.binding.runId}`);
await appendNativeCredentialUpdate(db, {
companyId: f.binding.companyId,
agentId: f.binding.agentId,
normalizedSessionId: f.binding.sessionId,
runnerInstanceId: f.binding.sourceInstanceId,
event: credential(20),
});
await appendRetainedNativeCleanupEvent(db, {
companyId: f.binding.companyId,
agentId: f.binding.agentId,
runId: f.binding.runId,
nativeSessionId: f.binding.sessionId,
runnerInstanceId: f.binding.sourceInstanceId,
requestId: "test-cleanup",
event: f.event(21),
});
await appendHeartbeatRunEvent(db, {
companyId: f.binding.companyId,
agentId: f.binding.agentId,
runId: f.binding.runId,
eventType: "workspace.diff.recorded",
payload: { complete: true },
nativeSource: {
sourceInstanceId: "provider-trace-reproject:test",
sourceEventId: "provider-trace-reproject:test:change",
sourceSeq: 1,
protocolSchemaVersion: 1,
canonicalPayload: { complete: true },
},
});
const b = f.store();
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
const entries = (await b.read(h)).content.trim().split("\n");
expect(entries).toHaveLength(2);
expect(JSON.parse(JSON.parse(entries[0]!).chunk).event.sourceSeq).toBe(1);
expect(JSON.parse(JSON.parse(entries[1]!).chunk).event.eventType).toBe(
"auth.credentials.updated",
);
const [run] = await db
.select({ bytes: heartbeatRuns.logBytes })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, f.binding.runId));
expect(run!.bytes).toBeGreaterThan(0);
});
it("aborts in-flight archive writes when its last reader cancels, then resumes without loss", async () => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin),
baseline = idleWorkSnapshot().active;
await f.port.appendEvent(f.event(1));
await db.execute(
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`,
);
let entered = false,
aborted = false;
const blocked: StorageProvider = {
...f.provider,
async putObjectConditional(input) {
if (input.objectKey.includes("/objects/")) {
entered = true;
await new Promise<void>((_resolve, reject) =>
input.signal!.addEventListener(
"abort",
() => {
aborted = true;
reject(input.signal!.reason);
},
{ once: true },
),
);
}
return f.provider.putObjectConditional!(input);
},
};
const controller = new AbortController();
const repairing = recoverSettledNativeRunLog(
db,
f.store(blocked),
h.logRef,
async () => ({ enabled: false }),
controller.signal,
);
const rejected = expect(repairing).rejects.toThrow("reader cancelled");
const lastReader = new AbortController();
const joined = recoverSettledNativeRunLog(
db,
f.store(blocked),
h.logRef,
async () => ({ enabled: false }),
lastReader.signal,
);
const joinedRejected = expect(joined).rejects.toThrow("last reader cancelled");
await vi.waitFor(() => expect(entered).toBe(true));
controller.abort(new Error("reader cancelled"));
await rejected;
expect(aborted).toBe(false);
lastReader.abort(new Error("last reader cancelled"));
await joinedRejected;
await vi.waitFor(() => {
expect(aborted).toBe(true);
expect(idleWorkSnapshot().active).toBe(baseline);
});
const b = f.store();
await vi.waitFor(async () => {
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
});
expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1);
});
it("redacts the existing agent identity and registered secrets before publishing recovered native output", async () => {
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "e".repeat(64));
const f = await fixture();
const keys = generateKeyPairSync("ed25519"),
privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString();
const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString();
const keyId = `sha256:${createHash("sha256")
.update(keys.publicKey.export({ type: "spki", format: "der" }))
.digest("base64url")}`;
const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem });
await db.insert(agentIdentityKeys).values({
agentId: f.binding.agentId,
companyId: f.binding.companyId,
algorithm: "Ed25519",
keyId,
publicKeyPem,
privateKeyMaterial: encrypted.material,
});
const secret = "registered-native-fixture-secret";
const redactor = createAgentIdentityRedactor(privateKeyPem);
redactor.values.push(secret);
const run = {
id: f.binding.runId,
companyId: f.binding.companyId,
agentId: f.binding.agentId,
status: "running",
runtimeMode: "native",
logStore: null,
logRef: null,
logBytes: null,
lastOutputSeq: 0,
lastOutputAt: null,
} as typeof heartbeatRuns.$inferSelect;
const output = createHeartbeatRunOutput(db, {
run,
runId: run.id,
issueId: null,
runLogStore: f.store(),
identityRedactor: redactor,
secretKeys: new Set(),
runtimeLogSecretValues: [secret],
appendRunEvent: async () => undefined,
getCurrentUserRedactionOptions: async () => ({ enabled: false }),
});
await output.begin(run);
const h = output.handle!;
await f.port.appendEvent({ ...f.event(1), payload: { text: `${privateKeyPem}\n${secret}` } });
await db.execute(
sql`update heartbeat_runs set status='succeeded', log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
);
const b = f.store();
await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false }));
const content = (await b.read(h)).content;
expect(content).not.toContain(privateKeyPem.split("\n")[1]!);
expect(content).not.toContain(secret);
expect(content).toContain("REDACTED");
const [snapshot] = await db
.select({ context: heartbeatRuns.contextSnapshot })
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, f.binding.runId));
expect(JSON.stringify(snapshot!.context)).not.toContain(privateKeyPem.split("\n")[1]!);
});
it("serves the published snapshot when another writer wins a repair claim", async () => {
const f = await fixture(),
a = f.store(),
h = await a.begin(f.begin);
const event = f.event(1),
receipt = await f.port.appendEvent(event);
const queued = await a.enqueue!(h, {
stream: "stdout",
chunk: formatNativeRunLogEvent(event),
ts: "then",
seq: 1,
sourceCursor: receipt.cursor,
});
await a.flush!(h);
await queued.committed;
await f.port.appendEvent(f.event(2));
await db.execute(
sql`update heartbeat_runs set status='succeeded',log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`,
);
let enter!: () => void, release!: () => void;
const entered = new Promise<void>((resolve) => {
enter = resolve;
}),
blocked = new Promise<void>((resolve) => {
release = resolve;
});
const store = {
...f.store(),
async begin() {
enter();
await blocked;
throw new ObjectJournalConflictError();
},
};
const repairing = recoverSettledNativeRunLog(db, store, h.logRef, async () => ({
enabled: false,
}));
const rejected = expect(repairing).rejects.toBeInstanceOf(ObjectJournalConflictError);
await entered;
const read = readRecoverableRunLog(db, store, h);
release();
await rejected;
expect((await read).content.trim().split("\n")).toHaveLength(1);
await a.finalize(h);
});
});
server/src/services/native-runtime/native-run-log-export.ts
+210
View File
@@ -0,0 +1,210 @@
import { and, asc, desc, eq, gt, isNotNull, lte, not, or, sql } from "drizzle-orm";
import { StringDecoder } from "node:string_decoder";
import { heartbeatRunEvents, type Db } from "@paperclipai/db";
import { validatePrpEvent, type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { nativeSha256 } from "./canonical.js";
export function formatNativeRunLogEvent(event: PrpEvent) {
return `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`;
}
export function nativeRunLogSourceWhere(companyId: string, runId: string) {
return and(
eq(heartbeatRunEvents.companyId, companyId),
eq(heartbeatRunEvents.runId, runId),
isNotNull(heartbeatRunEvents.sourceEventId),
not(
sql`coalesce((${or(
and(
eq(heartbeatRunEvents.eventType, "auth.credentials.updated"),
sql`${heartbeatRunEvents.sourceEventId} like 'credentials:%'`,
sql`${heartbeatRunEvents.sourceInstanceId} like '%:credentials'`,
),
and(
eq(heartbeatRunEvents.eventType, "native.cleanup.event"),
sql`${heartbeatRunEvents.sourceEventId} like 'cleanup:%'`,
sql`${heartbeatRunEvents.sourceInstanceId} like '%:cleanup:%'`,
),
and(
eq(heartbeatRunEvents.eventType, "workspace.diff.recorded"),
sql`${heartbeatRunEvents.sourceEventId} like 'provider-trace-reproject:%'`,
sql`${heartbeatRunEvents.sourceInstanceId} like 'provider-trace-reproject:%'`,
),
)}), false)`,
),
);
}
async function sourcePage(
db: Db,
companyId: string,
runId: string,
after: number,
through?: number,
signal?: AbortSignal,
) {
signal?.throwIfAborted();
const rows = await db
.select({
seq: heartbeatRunEvents.seq,
payload: heartbeatRunEvents.payload,
sourceInstanceId: heartbeatRunEvents.sourceInstanceId,
sourceEventId: heartbeatRunEvents.sourceEventId,
sourceSeq: heartbeatRunEvents.sourceSeq,
sha256: heartbeatRunEvents.sourcePayloadSha256,
})
.from(heartbeatRunEvents)
.where(
and(
nativeRunLogSourceWhere(companyId, runId),
gt(heartbeatRunEvents.seq, after),
through === undefined ? undefined : lte(heartbeatRunEvents.seq, through),
),
)
.orderBy(asc(heartbeatRunEvents.seq))
.limit(16);
signal?.throwIfAborted();
return rows.map((row) => {
const validated = validatePrpEvent(row.payload?.prpEvent);
if (
!validated.ok ||
validated.event.runId !== runId ||
validated.event.sourceInstanceId !== row.sourceInstanceId ||
validated.event.sourceEventId !== row.sourceEventId ||
validated.event.sourceSeq !== row.sourceSeq ||
nativeSha256(validated.event) !== row.sha256
)
throw new Error("native_run_log_source_integrity_error");
return { cursor: row.seq, event: validated.event };
});
}
export function createNativeRunLogCoverageVerifier(db: Db, companyId: string, runId: string) {
const decoder = new StringDecoder("utf8");
let text = "",
cursor = 0;
let page: Awaited<ReturnType<typeof sourcePage>> = [];
const unproven = () => new Error("native_log_source_coverage_unproven");
async function next() {
if (!page.length) page = await sourcePage(db, companyId, runId, cursor);
return page[0];
}
async function line(value: string) {
if (!value.trim()) return;
if (Buffer.byteLength(value) > 1024 * 1024) throw unproven();
let chunk: unknown;
try {
chunk = JSON.parse(value).chunk;
} catch {
throw unproven();
}
if (typeof chunk !== "string" || !chunk.startsWith('{"type":"paperclip.prp.event"')) return;
let event: PrpEvent;
try {
const parsed = validatePrpEvent(JSON.parse(chunk).event);
if (!parsed.ok) throw unproven();
event = parsed.event;
} catch {
throw unproven();
}
const expected = await next();
if (
!expected ||
event.runId !== runId ||
event.sourceInstanceId !== expected.event.sourceInstanceId ||
event.sourceEventId !== expected.event.sourceEventId ||
event.sourceSeq !== expected.event.sourceSeq ||
event.normalizedSessionId !== expected.event.normalizedSessionId ||
event.schemaVersion !== expected.event.schemaVersion ||
event.eventType !== expected.event.eventType
)
throw unproven();
cursor = expected.cursor;
page.shift();
}
return {
async write(bytes: Buffer) {
text += decoder.write(bytes);
let boundary: number;
while ((boundary = text.indexOf("\n")) !== -1) {
await line(text.slice(0, boundary));
text = text.slice(boundary + 1);
}
if (Buffer.byteLength(text) > 1024 * 1024) throw unproven();
},
async finish() {
text += decoder.end();
if (text) await line(text);
if (await next()) throw unproven();
return cursor;
},
};
}
export function createNativeRunLogExporter(input: {
db: Db;
companyId: string;
runId: string;
afterCursor: number;
enqueue: (cursor: number, chunk: string) => Promise<void>;
flush: () => Promise<void>;
signal?: AbortSignal;
}) {
let cursor = input.afterCursor;
let queue = Promise.resolve();
let queuedBytes = 0,
queuedRecords = 0;
const sourceWhere = nativeRunLogSourceWhere(input.companyId, input.runId);
async function replay(through?: number) {
input.signal?.throwIfAborted();
if (through === undefined) {
const [last] = await input.db
.select({ seq: heartbeatRunEvents.seq })
.from(heartbeatRunEvents)
.where(sourceWhere)
.orderBy(desc(heartbeatRunEvents.seq))
.limit(1);
through = last?.seq ?? cursor;
}
while (cursor < through) {
const rows = await sourcePage(
input.db,
input.companyId,
input.runId,
cursor,
through,
input.signal,
);
if (!rows.length) break;
for (const row of rows) {
input.signal?.throwIfAborted();
await input.enqueue(row.cursor, formatNativeRunLogEvent(row.event));
cursor = row.cursor;
}
await input.flush();
}
}
return {
recover: () => replay(),
async record(sourceCursor: number, chunk: string) {
if (!Number.isSafeInteger(sourceCursor) || sourceCursor < 1)
throw new Error("invalid_native_run_log_cursor");
const bytes = Buffer.byteLength(chunk);
if (queuedRecords >= 32 || queuedBytes + bytes > 8 * 1024 * 1024)
throw new Error("native_run_log_admission_capacity_exceeded");
queuedRecords++;
queuedBytes += bytes;
const work = queue
.then(async () => {
if (sourceCursor <= cursor) return;
if (sourceCursor > cursor + 1) await replay(sourceCursor - 1);
await input.enqueue(sourceCursor, chunk);
cursor = sourceCursor;
})
.finally(() => {
queuedRecords--;
queuedBytes -= bytes;
});
queue = work.catch(() => undefined);
await work;
},
};
}
server/src/services/native-runtime/native-run-log-recovery.ts
+223
View File
@@ -0,0 +1,223 @@
import { and, eq, gt, inArray, isNull, sql } from "drizzle-orm";
import {
heartbeatRuns,
heartbeatRunEvents,
nativeRunFinalizations,
type Db,
} from "@paperclipai/db";
import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../log-redaction.js";
import type { RunLogHandle, RunLogReadOptions, RunLogStore } from "../run-log-store.js";
import { instanceSettingsService } from "../instance-settings.js";
import { compactRunLogChunk } from "../heartbeat/run-log.js";
import { beginIdleTrackedWork } from "../task-admission.js";
import { agentIdentityService } from "../agent-identity.js";
import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js";
import { createAgentIdentityRedactor } from "../agent-identity-redaction.js";
import { logger } from "../../middleware/logger.js";
import { createNativeRunLogExporter, nativeRunLogSourceWhere } from "./native-run-log-export.js";
interface Recovery {
work: Promise<void>;
controller: AbortController;
readers: number;
}
const recoveries = new WeakMap<Db, Map<string, Recovery>>();
function awaitRecovery(recovery: Recovery, signal?: AbortSignal) {
signal?.throwIfAborted();
recovery.readers++;
return new Promise<void>((resolve, reject) => {
let finished = false;
const finish = () => {
if (finished) return false;
finished = true;
signal?.removeEventListener("abort", abort);
recovery.readers--;
return true;
};
const abort = () => {
if (!finish()) return;
if (!recovery.readers) recovery.controller.abort(signal!.reason);
reject(signal!.reason);
};
signal?.addEventListener("abort", abort, { once: true });
recovery.work.then(
() => {
if (finish()) resolve();
},
(error) => {
if (finish()) reject(error);
},
);
});
}
export async function readRecoverableRunLog(
db: Db,
store: RunLogStore,
handle: RunLogHandle,
options?: RunLogReadOptions,
known?: { runtimeMode?: string | null; status?: string; logBytes?: number | null },
) {
if (
handle.store === "s3_segments" &&
known?.runtimeMode !== "legacy" &&
known?.status !== "running" &&
(known?.logBytes === undefined || known.logBytes === null)
)
await recoverSettledNativeRunLog(db, store, handle.logRef, undefined, options?.signal).catch(
() => {
options?.signal?.throwIfAborted();
logger.warn(
{ runId: handle.logRef.split("/").at(-1) },
"Native run log repair is incomplete; serving the published prefix",
);
},
);
return store.read(handle, options);
}
export async function recoverSettledNativeRunLog(
db: Db,
store: RunLogStore,
logRef: string,
getRedactionOptions: () => Promise<CurrentUserRedactionOptions> = async () => ({
enabled: (await instanceSettingsService(db).getGeneral()).censorUsernameInLogs,
}),
requestSignal?: AbortSignal,
) {
requestSignal?.throwIfAborted();
const binding = /^([a-f0-9-]+)\/([a-f0-9-]+)\/([a-f0-9-]+)$/.exec(logRef);
if (!binding) throw new Error("invalid_native_run_log_reference");
let pending = recoveries.get(db);
if (!pending) {
pending = new Map();
recoveries.set(db, pending);
}
const existing = pending.get(logRef);
if (existing) return awaitRecovery(existing, requestSignal);
const controller = new AbortController();
const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]);
const finishWork = beginIdleTrackedWork();
const runId = binding[3]!,
companyId = binding[1]!;
const eligible = () =>
and(
eq(heartbeatRuns.id, runId),
eq(heartbeatRuns.companyId, companyId),
eq(heartbeatRuns.agentId, binding[2]!),
eq(heartbeatRuns.logStore, "s3_segments"),
eq(heartbeatRuns.logRef, logRef),
eq(heartbeatRuns.runtimeMode, "native"),
inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out"]),
isNull(heartbeatRuns.logBytes),
sql`not exists (select 1 from ${nativeRunFinalizations} where ${and(
eq(nativeRunFinalizations.runId, runId),
eq(nativeRunFinalizations.companyId, companyId),
gt(nativeRunFinalizations.leaseExpiresAt, new Date()),
)})`,
);
const recover = (async () => {
signal.throwIfAborted();
const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eligible());
signal.throwIfAborted();
if (!run) return;
const [redaction, identityRedactor, secrets] = await Promise.all([
getRedactionOptions(),
agentIdentityService(db).createExistingLogRedactor(companyId, binding[2]!),
createRunSecretRedactionRegistry(db).redactionValuesForRun(companyId, runId),
]);
for (const value of secrets)
identityRedactor.values.push(...createAgentIdentityRedactor(value).values);
identityRedactor.values.sort((left, right) => right.length - left.length);
const handle = await store.begin({
companyId,
agentId: binding[2]!,
runId,
previousStore: "s3_segments",
signal,
assertWriterAuthority: async () => {
signal.throwIfAborted();
const [current] = await db
.select({ id: heartbeatRuns.id })
.from(heartbeatRuns)
.where(eligible());
if (!current) throw new Error("native_run_log_recovery_authority_lost");
},
});
let sequence = handle.initialSequence ?? 0;
let lastSourceCursor = handle.initialSourceCursor ?? 0;
const receipts: Promise<number>[] = [];
let failure: unknown;
const exporter = createNativeRunLogExporter({
db,
companyId,
runId,
afterCursor: handle.initialSourceCursor ?? 0,
signal,
enqueue: async (sourceCursor, chunk) => {
const receipt = await store.enqueue!(handle, {
stream: "stdout",
ts: new Date().toISOString(),
seq: ++sequence,
sourceCursor,
chunk: compactRunLogChunk(
redactCurrentUserText(identityRedactor.chunk("stdout", chunk), redaction),
),
});
receipts.push(
receipt.committed.catch((error) => {
failure ??= error;
return 0;
}),
);
lastSourceCursor = sourceCursor;
},
flush: async () => {
await store.flush!(handle);
await Promise.all(receipts.splice(0));
if (failure) throw failure;
},
});
try {
await exporter.recover();
} catch (error) {
failure ??= error;
} finally {
const summary = await store.finalize(handle, { computeDigest: false });
if (summary.bytes === null) failure ??= new Error("native_run_log_recovery_incomplete");
if (!failure && summary.bytes !== null) {
signal.throwIfAborted();
const updated = await db
.update(heartbeatRuns)
.set({
logBytes: summary.bytes,
logSha256: summary.sha256,
lastOutputSeq: sequence,
lastOutputBytes: summary.bytes,
})
.where(
and(
eligible(),
sql`not exists (select 1 from ${heartbeatRunEvents} where ${and(
nativeRunLogSourceWhere(companyId, runId),
gt(heartbeatRunEvents.seq, lastSourceCursor),
)})`,
),
)
.returning({ id: heartbeatRuns.id });
if (!updated.length) throw new Error("native_run_log_recovery_source_changed");
}
}
if (failure) throw failure;
})();
const recovery: Recovery = {
controller,
readers: 0,
work: recover.finally(() => {
pending.delete(logRef);
finishWork();
}),
};
pending.set(logRef, recovery);
await awaitRecovery(recovery, requestSignal);
}
server/src/services/native-runtime/native-session-executor.test.ts
+47 -5
View File
@@ -110,6 +110,10 @@ vi.mock("../usage-receipts.js", async (importOriginal) => ({
createRunUsageRecorder: usageRecorder.create,
}));
vi.mock("./native-usage-delivery.js", () => ({
createNativeUsageDelivery: vi.fn(async () => ({ capture: async () => undefined })),
}));
const githubAccess = vi.hoisted(() => ({
activate: vi.fn((_binding: { runId: string }) => vi.fn()),
stop: vi.fn(async () => undefined),
@@ -140,6 +144,7 @@ type BackendFactoryOptions = {
};
type RunnerTransportOptions = {
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
environment?: NodeJS.ProcessEnv;
acpxRuntimeDirectory?: string;
adoptExistingRunner?: { pid: number; isAlive: () => Promise<boolean> | boolean };
@@ -249,8 +254,8 @@ vi.mock("./paperclip-control-plane-port.js", async importOriginal => {
return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort {
constructor(...args: ConstructorParameters<typeof original.PaperclipControlPlanePort>) {
super(...args);
accountingEvents.committed = args[2]?.onCommittedEvent;
accountingEvents.duplicate = args[2]?.onDuplicateEvent;
accountingEvents.committed = args[2]?.onCommittedEvent ? event => args[2]!.onCommittedEvent!(event, 1) : undefined;
accountingEvents.duplicate = args[2]?.onDuplicateEvent ? event => args[2]!.onDuplicateEvent!(event, 1) : undefined;
}
} };
});
@@ -7154,6 +7159,36 @@ describe("native terminal-turn accounting", () => {
expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true }));
expect(usageRecorder.create).not.toHaveBeenCalled();
});
it("binds retained warm transport usage to the successor run authority", async () => {
const directory = await mkdtemp(join(tmpdir(), "native-warm-usage-"));
const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR;
process.env.PAPERCLIP_RUNNER_STATE_DIR = directory;
const first = { ...execution, binding: { ...execution.binding, runId: "first-warm-usage", executionWorkspaceId: "warm-usage-workspace" },
session: { ...execution.session, normalizedSessionId: "warm-usage-session", lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60000 } } } as NativeExecutionInputV1;
const second = { ...first, binding: { ...first.binding, runId: "second-warm-usage" } };
const firstCapture = vi.fn(async (_event: PrpEvent) => {}), secondCapture = vi.fn(async (_event: PrpEvent) => {});
try {
await createRunnerdBackend({ db: leaseDb(first), execution: first, runnerInstanceId: "warm-producer", onAccountingEvent: firstCapture });
state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
const retainedHook = state.createTransport.mock.calls.at(-1)![0].onAccountingEvent!;
const firstReport = { eventType: "usage.reported", runId: first.binding.runId, sourceInstanceId: "warm-producer" } as PrpEvent;
await retainedHook(firstReport);
expect(firstCapture).toHaveBeenCalledExactlyOnceWith(firstReport);
await rm(state.createTransport.mock.calls.at(-1)![0].stateDirectory!, { recursive: true, force: true });
await createRunnerdBackend({ db: leaseDb(second), execution: second, runnerInstanceId: "warm-producer", onAccountingEvent: secondCapture });
await expect(retainedHook({ ...firstReport, eventType: "run.attached" })).resolves.toBeUndefined();
expect(secondCapture).not.toHaveBeenCalled();
const secondReport = { ...firstReport, runId: second.binding.runId };
await retainedHook(secondReport);
expect(secondCapture).toHaveBeenCalledExactlyOnceWith(secondReport);
expect(firstCapture).toHaveBeenCalledTimes(1);
await expect(retainedHook(firstReport)).rejects.toThrow("native_usage_run_binding_mismatch");
} finally {
if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous;
await rm(directory, { recursive: true, force: true });
}
});
it("uses the canonical recorder for partial and completed usage when no onUsage callback is supplied", async () => {
const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
state.execute.mockReset().mockImplementationOnce(async () => {
@@ -12341,7 +12376,14 @@ describe("runnerd provider runtime wiring", () => {
}
});
it.each(["github", "assigned"] as const)("cleans run-owned access after overlapping executor failure: %s", async failure => {
it.each((["codex", "opencode"] as const).flatMap(provider =>
(["github", "assigned"] as const).map(failure => ({ provider, failure })),
))("cleans run-owned access after overlapping $provider executor failure: $failure", async ({ provider, failure }) => {
const current = {
...execution,
provider: provider === "codex" ? execution.provider : { kind: "opencode", model: null },
session: { ...execution.session, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server" },
} as NativeExecutionInputV1;
const issueService = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({
update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })),
} as unknown as ReturnType<typeof issueServiceModule.issueService>);
@@ -12367,7 +12409,7 @@ describe("runnerd provider runtime wiring", () => {
state.createBackend.mockClear();
let settled = false;
const running = executePaperclipNativeSession({
db: leaseDb(execution), execution, runnerInstanceId: "runner-overlap-cleanup",
db: leaseDb(current), execution: current, runnerInstanceId: "runner-overlap-cleanup",
useRunnerd: true, managedGitHub: true,
runnerEnvironment: {
PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned",
@@ -12393,7 +12435,7 @@ describe("runnerd provider runtime wiring", () => {
expect(state.execute).not.toHaveBeenCalled();
expect(state.createBackend).not.toHaveBeenCalled();
if (failure === "assigned") {
expect(activate).toHaveBeenCalledWith(execution.binding);
expect(activate).toHaveBeenCalledWith(current.binding);
expect(deactivate).toHaveBeenCalledOnce();
expect(stop).toHaveBeenCalledOnce();
}
server/src/services/native-runtime/native-session-executor.ts
+47 -189
View File
@@ -1,4 +1,8 @@
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js";
export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js";
import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js";
import { formatNativeRunLogEvent } from "./native-run-log-export.js";
import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
import { notifyNativeResultCommit } from "../native-result-commit-notifications.js";
import { releaseRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js";
@@ -107,7 +111,7 @@ import type {
AdapterExecutionResult,
AdapterRuntimeEvent,
} from "../../adapters/index.js";
import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared";
import type { NativeFinalizationResult } from "@paperclipai/shared";
import type {
DurablePrpControlPlane,
HarnessRuntimeRequestResolution,
@@ -1752,12 +1756,13 @@ export async function synchronizeCompletedProviderPlan(input: {
};
}
class SessionToolAuthorityEpoch {
class SessionAuthorityEpoch {
readonly runId: string;
#authority: PaperclipRunnerToolAuthority;
#revoked = false;
constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) {
constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace,
private readonly onAccountingEvent?: (event: PrpEvent) => Promise<void>) {
this.runId = runId;
this.#authority = authority;
}
@@ -1773,6 +1778,14 @@ class SessionToolAuthorityEpoch {
}
}
async acceptAccountingEvent(event: PrpEvent) {
if (!isNativeUsageAccountingEvent(event)) return;
this.#assertCurrent();
if (event.runId !== this.runId) throw new Error("native_usage_run_binding_mismatch");
await this.onAccountingEvent?.(event);
this.#assertCurrent();
}
definitions() {
this.#assertCurrent();
return this.#authority.definitions();
@@ -1793,7 +1806,7 @@ class SessionToolAuthorityEpoch {
}
}
const sessionToolAuthorityEpochs = new Map<string, SessionToolAuthorityEpoch>();
const sessionToolAuthorityEpochs = new Map<string, SessionAuthorityEpoch>();
const initializingSessionToolAuthorities = new Set<string>();
const executingRunnerdSessionScopes = new Map<string, string>();
const executingNativeOwnerScopes = new Map<string, symbol>();
@@ -7820,6 +7833,7 @@ export async function executePaperclipNativeSession(input: {
/** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
onPersistedNativeLog?: (cursor: number, chunk: string) => Promise<void>;
onEvent?: (event: AdapterRuntimeEvent) => Promise<void>;
/** Only this run's registered private instruction entry.
* Probe at terminal; persist only after owned shutdown. */
@@ -8559,7 +8573,15 @@ async function executePaperclipNativeSessionWithinScope(
for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent);
}
let fallbackUsageRecorder: ReturnType<typeof createRunUsageRecorder> | undefined;
const producerOwnsUsage = usesNativeUsageDelivery({ runnerd: input.useRunnerd === true,
remote: input.runnerExecutionTarget?.kind === "remote", provider: input.execution.provider });
const producerUsage = producerOwnsUsage ? await createNativeUsageDelivery(input.db, {
companyId: input.execution.binding.companyId, runId: input.execution.binding.runId,
runnerInstanceId: effectiveRunnerInstanceId, provider: input.execution.provider,
billing: resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity),
}) : undefined;
const persistAccountingUsage = async (usage: Record<string, unknown> | null, complete: boolean) => {
if (producerOwnsUsage) return;
const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" });
const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined;
complete = complete && accountingUsage !== undefined;
@@ -8653,7 +8675,7 @@ async function executePaperclipNativeSessionWithinScope(
attributes:{eventKind:event.eventType}});
}
},
onCommittedEvent: async (event) => {
onCommittedEvent: async (event, cursor) => {
if (event.eventType === "run.result.proposed") {
const at = Date.now();
await trace.record({name:"completion.tool_accepted",parentName:"native.session.execute",startedAtMs:at,endedAtMs:at});
@@ -8806,7 +8828,8 @@ async function executePaperclipNativeSessionWithinScope(
});
trace.activate(taskSettleScope);
}
if (input.onLog)
if (input.onPersistedNativeLog) await input.onPersistedNativeLog(cursor, formatNativeRunLogEvent(event));
else if (input.onLog)
await input.onLog(
"stdout",
`${JSON.stringify({ type: "paperclip.prp.event", event })}\n`,
@@ -8885,12 +8908,10 @@ async function executePaperclipNativeSessionWithinScope(
},
);
},
onDuplicateEvent: async (event) => {
onDuplicateEvent: async (event, cursor) => {
await input.onPersistedNativeLog?.(cursor, formatNativeRunLogEvent(event));
if (!firstTokenRecorded) observeDeliverableText(event);
await observeAccountingEvent(event);
// A crash can happen after the event commit but before its callback
// finishes. Recover only idempotent durable projections here; activity,
// publication, logging, trace, and metric effects remain committed-only.
await liveQuestions.observe(event);
await projectSessionGoalEvent(event);
providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event);
@@ -9165,7 +9186,8 @@ async function executePaperclipNativeSessionWithinScope(
return input.runnerEnvironment;
};
const overlapGitHubPreparation = needsCoreGitHubAccess(input) && input.useRunnerd && input.backend === undefined
&& input.runnerExecutionTarget?.kind === "remote" && input.execution.provider.kind === "codex"
&& input.runnerExecutionTarget?.kind === "remote"
&& (input.execution.provider.kind === "codex" || input.execution.provider.kind === "opencode")
&& input.execution.schema !== "paperclip.native-execution-input.v6"
&& input.execution.schema !== "paperclip.native-execution-input.v8";
if (!overlapGitHubPreparation) await prepareRunnerEnvironment();
@@ -9178,6 +9200,7 @@ async function executePaperclipNativeSessionWithinScope(
input.useRunnerd && input.backend === undefined
? await timeController("backend_setup", () => createRunnerdBackend({
...input,
onAccountingEvent: producerUsage?.capture,
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? {
bootId: controller.bootId,
runnerArtifact: restoredController ? { version: restoredController.runnerVersion, digest: restoredController.runnerDigest } : undefined,
@@ -9217,7 +9240,7 @@ async function executePaperclipNativeSessionWithinScope(
persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution,
nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot());
persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote");
runnerdBackend = await createRunnerdBackend({ ...input, runnerInstanceId: effectiveRunnerInstanceId,
runnerdBackend = await createRunnerdBackend({ ...input, onAccountingEvent: producerUsage?.capture, runnerInstanceId: effectiveRunnerInstanceId,
controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? { bootId: controller.bootId } : undefined,
durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, trace, toolTrace });
}
@@ -10261,151 +10284,6 @@ async function executePaperclipNativeSessionWithinScope(
return adapterResult;
}
function numericUsageField(
usage: Record<string, unknown> | null,
keys: string[],
): number | undefined {
if (!usage) return undefined;
for (const key of keys) {
const value = usage[key];
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
return value;
}
return undefined;
}
function nativeUsageMeasurement(usage: Record<string, unknown>) {
const nestedUsage = record(usage.usage);
// An explicit partial delta must not fall through to a full session total.
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
const candidates = [
record(usage.runDelta),
record(nestedUsage.runDelta),
record(usage.total),
record(nestedUsage.total),
record(usage.cumulative),
record(nestedUsage.cumulative),
nestedUsage,
usage,
];
return (
candidates.find(
(candidate) =>
numericUsageField(candidate, [
"inputTokens",
"input",
"promptTokens",
"outputTokens",
"output",
"completionTokens",
]) !== undefined,
) ?? usage
);
}
export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
if (provider.kind === "acpx") {
if (provider.agent === "cursor") return "cursor";
if (provider.agent === "copilot") return "github";
if (provider.agent === "pi") return "openrouter";
}
return "openai";
}
function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
const sequences = new Map<string, number>();
let currentTurn: string | undefined;
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
const snapshot = () => {
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
for (const [turnId, turn] of turns) {
const normalized = normalizeNativeUsage(turn.usage);
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
outputTokens += normalized?.outputTokens ?? 0;
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
const price = nativeUsageCostUsd(turn.usage, provider);
if (price === undefined) costIncomplete = true;
else { cost += usdToUnits(price); costKnown = true; }
}
// A free empty attempt contributes no price evidence for later paid work.
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
// still receive an estimate. Preserve positive known spend and true zero totals.
const retainCost = costKnown && (!costIncomplete || cost > 0n);
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
const current = currentTurn ? turns.get(currentTurn) : undefined;
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
};
return {
observe(event: PrpEvent) {
if (!event) return false;
const payload = record(event.payload);
const terminal = terminalTypes.includes(event.eventType);
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
// Historical replay can revisit an earlier report after restoration.
// Re-persist its aggregate, but never rewind or count that report twice.
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
sequences.set(event.sourceInstanceId, event.sourceSeq);
}
const key = event.turnId ?? "unattributed";
const turn = turns.get(key) ?? { usage: null, terminal: false };
if (event.eventType === "turn.started") {
currentTurn = event.turnId;
turn.usage = null;
turn.terminal = false;
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
if (payload.kind === "usage") {
const usage = record(payload.usage);
turn.usage = Object.hasOwn(usage, "runDelta")
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
}
if (terminal) turn.terminal = true;
turns.set(key, turn);
return true;
},
snapshot,
finish(turnId: string | null) {
const turn = turnId ? turns.get(turnId) : undefined;
if (turn) turn.terminal = true;
return snapshot();
},
};
}
export function nativeUsageCostUsd(
usage: Record<string, unknown> | null,
provider?: NativeExecutionInput["provider"],
) {
// The ACP normalization contract fills absent per-turn cost with zero and
// reports actual cost cumulatively. Until it carries an authoritative run
// delta with provenance, neither value is a candidate's billed USD receipt.
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
if (!usage) return undefined;
const measurement = nativeUsageMeasurement(usage);
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
// PRP currently defaults an absent providerCostUsd to zero. That zero is
// not proof of a free request when the runtime observed token consumption.
if (direct !== undefined && direct > 0) return direct;
const normalized = normalizeNativeUsage(usage);
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
const currency =
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
if (currency !== "USD") return undefined;
return numericUsageField(cost, ["amount", "total"]);
}
/** Billing identity follows the selected runtime and resolved credentials.
* Unknown authentication remains unknown; a provider name does not prove billing mode. */
export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick<AdapterExecutionResult, "provider" | "biller" | "billingType">): Pick<AdapterExecutionResult, "provider" | "biller" | "billingType"> {
@@ -10438,33 +10316,6 @@ export function resolveNativeBilling(provider: NativeExecutionInput["provider"],
return { provider: "openai", biller: billingType === "metered_api" ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType };
}
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
const measurement = nativeUsageMeasurement(usage);
const cache = record(measurement.cache);
const count = (source: Record<string, unknown>, keys: string[]) => {
for (const key of keys) {
if (source[key] === undefined) continue;
const value = source[key];
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
}
return undefined;
};
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
if (input === undefined || outputTokens === undefined
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
return {
inputTokens, outputTokens,
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
};
}
function processEnvironment(
environment: NodeJS.ProcessEnv,
@@ -11850,6 +11701,7 @@ export async function createRunnerdBackend(input: {
casesEnabled?: boolean;
db: Db;
controllerPersistence?: { bootId: string; runnerArtifact?: { version: string; digest: string } };
onAccountingEvent?: (event: PrpEvent) => Promise<void>;
execution: NativeExecutionInput;
runnerInstanceId: string;
chatAttachmentReadScope?: NativeChatAttachmentReadScope;
@@ -12165,13 +12017,14 @@ async function createRunnerdBackendWithinSessionClaim(
syncIssueExternalObjects: input.syncIssueExternalObjects,
enqueueWakeup: input.enqueueWakeup,
});
const authorityEpoch = new SessionToolAuthorityEpoch(
const authorityEpoch = new SessionAuthorityEpoch(
input.execution.binding.runId,
authority,
input.toolTrace,
input.onAccountingEvent,
);
let dynamicTools: Awaited<
ReturnType<SessionToolAuthorityEpoch["definitions"]>
ReturnType<SessionAuthorityEpoch["definitions"]>
>;
try {
dynamicTools = await timeController("tool_authority", () => authorityEpoch.definitions());
@@ -12273,7 +12126,7 @@ async function createRunnerdBackendWithinSessionClaim(
authority.binding.readRemoteTaskFile = file => owner.readWorkspaceFile(file.contentRef, file.byteSize);
dynamicTools = await authorityEpoch.definitions();
}
const executeTool = (call: Parameters<SessionToolAuthorityEpoch["execute"]>[0]) => {
const executeTool = (call: Parameters<SessionAuthorityEpoch["execute"]>[0]) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
return current.execute(call);
@@ -12368,7 +12221,7 @@ async function createRunnerdBackendWithinSessionClaim(
};
const backendOptions = {
onSpawn: input.onSpawn, dynamicTools,
dynamicToolHandler: async (call: Parameters<SessionToolAuthorityEpoch["execute"]>[0]) => {
dynamicToolHandler: async (call: Parameters<SessionAuthorityEpoch["execute"]>[0]) => {
await assertExternalNativeRunAuthority(input.db, externalExecution); return authorityEpoch.execute(call);
},
completionFeedback: async (result: PrpStructuredRunResult) => {
@@ -13709,7 +13562,7 @@ async function createRunnerdBackendWithinSessionClaim(
markSpawned?.();
}
const executeCurrentToolAuthority = (
call: Parameters<SessionToolAuthorityEpoch["execute"]>[0],
call: Parameters<SessionAuthorityEpoch["execute"]>[0],
) => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_tool_authority_unavailable");
@@ -13757,6 +13610,11 @@ async function createRunnerdBackendWithinSessionClaim(
onSpawn: input.onSpawn,
...(input.execution.provider.kind === "codex" ? { disableCodexWebSearch: "webSearch" in input.execution.provider && input.execution.provider.webSearch === "disabled" } : {}),
homeMode: preconfiguredHome ? "preconfigured" : "isolated",
onAccountingEvent: input.onAccountingEvent ? async event => {
const current = sessionToolAuthorityEpochs.get(sessionScopeId);
if (!current) throw new Error("native_session_authority_unavailable");
await current.acceptAccountingEvent(event);
} : undefined,
onCredentialsUpdated: (event) => appendNativeCredentialUpdate(input.db, {
companyId: input.execution.binding.companyId,
agentId: input.execution.binding.agentId,
server/src/services/native-runtime/native-usage-accounting.test.ts
+43
View File
@@ -0,0 +1,43 @@
import { describe, expect, it } from "vitest";
import type { PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage, usesNativeUsageDelivery } from "./native-usage-accounting.js";
const provider = { kind: "opencode" as const, model: "test-model" };
const event = (sourceSeq: number, eventType: string, turnId = "turn-1", payload: Record<string, unknown> = {}) => ({
sourceInstanceId: "producer", sourceSeq, eventType, turnId, payload,
} as PrpEvent);
describe("native turn accounting", () => {
it("selects producer delivery only for supported remote Runner execution", () => {
expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider })).toBe(true);
expect(usesNativeUsageDelivery({ runnerd: false, remote: true, provider })).toBe(false);
expect(usesNativeUsageDelivery({ runnerd: true, remote: false, provider })).toBe(false);
expect(usesNativeUsageDelivery({ runnerd: true, remote: true })).toBe(false);
expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider: { kind: "muse", model: null } as never })).toBe(false);
});
it("reconstructs cumulative turn usage without counting repeated producer events twice", () => {
const accounting = createNativeTurnAccounting(provider);
const usage = { kind: "usage", usage: { runDeltaComplete: true, runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: 0.25 } } };
const history = [event(1, "turn.started"), event(2, "item.completed", "turn-1", usage), event(3, "turn.completed"),
event(4, "turn.started", "turn-2"), event(5, "item.completed", "turn-2", usage), event(6, "turn.completed", "turn-2")];
history.forEach(value => accounting.observe(value));
history.forEach(value => accounting.observe(value));
const snapshot = accounting.snapshot();
expect(snapshot.complete).toBe(true);
expect(normalizeNativeUsage(snapshot.usage)).toEqual({ inputTokens: 240, outputTokens: 20, cachedInputTokens: 0, cacheWriteTokens: 0 });
expect(nativeUsageCostUsd(snapshot.usage, provider)).toBe(0.5);
});
it("keeps an incomplete provider report pending instead of trusting placeholder zeroes", () => {
const accounting = createNativeTurnAccounting(provider);
accounting.observe(event(1, "turn.started"));
accounting.observe(event(2, "item.completed", "turn-1", { kind: "usage", usage: {
runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 },
} }));
accounting.observe(event(3, "turn.completed"));
expect(accounting.snapshot().complete).toBe(false);
expect(accounting.snapshot().usage?.accountingCostIncomplete).toBe(true);
expect(normalizeNativeUsage({ runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0 } })).toBeUndefined();
});
});
server/src/services/native-runtime/native-usage-accounting.ts
+187
View File
@@ -0,0 +1,187 @@
import { usdToUnits } from "@paperclipai/shared";
import type { NativeExecutionInput, PrpEvent } from "../../vendor/paperclip-runner/index.js";
const record = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
function numericUsageField(
usage: Record<string, unknown> | null,
keys: string[],
): number | undefined {
if (!usage) return undefined;
for (const key of keys) {
const value = usage[key];
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
return value;
}
return undefined;
}
function nativeUsageMeasurement(usage: Record<string, unknown>) {
const nestedUsage = record(usage.usage);
// An explicit partial delta must not fall through to a full session total.
if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta);
if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta);
const candidates = [
record(usage.runDelta),
record(nestedUsage.runDelta),
record(usage.total),
record(nestedUsage.total),
record(usage.cumulative),
record(nestedUsage.cumulative),
nestedUsage,
usage,
];
return (
candidates.find(
(candidate) =>
numericUsageField(candidate, [
"inputTokens",
"input",
"promptTokens",
"outputTokens",
"output",
"completionTokens",
]) !== undefined,
) ?? usage
);
}
export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string {
if (provider.kind === "acpx") {
if (provider.agent === "cursor") return "cursor";
if (provider.agent === "copilot") return "github";
if (provider.agent === "pi") return "openrouter";
}
return "openai";
}
export function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) {
const turns = new Map<string, { usage: Record<string, unknown> | null; terminal: boolean }>();
const sequences = new Map<string, number>();
let currentTurn: string | undefined;
const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"];
const snapshot = () => {
if (turns.size === 0) return { usage: null, turnId: undefined, complete: false };
let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
let cost = 0n, costKnown = false, costIncomplete = false, complete = true;
for (const [turnId, turn] of turns) {
const normalized = normalizeNativeUsage(turn.usage);
complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined;
inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0);
outputTokens += normalized?.outputTokens ?? 0;
cacheReadTokens += normalized?.cachedInputTokens ?? 0;
cacheWriteTokens += normalized?.cacheWriteTokens ?? 0;
const price = nativeUsageCostUsd(turn.usage, provider);
if (price === undefined) costIncomplete = true;
else { cost += usdToUnits(price); costKnown = true; }
}
// A free empty attempt contributes no price evidence for later paid work.
// Keep an incomplete zero subtotal absent so complete aggregate tokens can
// still receive an estimate. Preserve positive known spend and true zero totals.
const retainCost = costKnown && (!costIncomplete || cost > 0n);
const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens,
...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) };
const current = currentTurn ? turns.get(currentTurn) : undefined;
return { usage: { runDelta, accountingCostIncomplete: costIncomplete,
...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) },
turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete };
};
return {
observe(event: PrpEvent) {
if (!event) return false;
const payload = record(event.payload);
const terminal = terminalTypes.includes(event.eventType);
if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false;
// Historical replay can revisit an earlier report after restoration.
// Re-persist its aggregate, but never rewind or count that report twice.
if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) {
if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true;
sequences.set(event.sourceInstanceId, event.sourceSeq);
}
const key = event.turnId ?? "unattributed";
const turn = turns.get(key) ?? { usage: null, terminal: false };
if (event.eventType === "turn.started") {
currentTurn = event.turnId;
turn.usage = null;
turn.terminal = false;
} else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId;
if (payload.kind === "usage") {
const usage = record(payload.usage);
turn.usage = Object.hasOwn(usage, "runDelta")
? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null;
}
if (terminal) turn.terminal = true;
turns.set(key, turn);
return true;
},
snapshot,
finish(turnId: string | null) {
const turn = turnId ? turns.get(turnId) : undefined;
if (turn) turn.terminal = true;
return snapshot();
},
};
}
export function nativeUsageCostUsd(
usage: Record<string, unknown> | null,
provider?: NativeExecutionInput["provider"],
) {
// The ACP normalization contract fills absent per-turn cost with zero and
// reports actual cost cumulatively. Until it carries an authoritative run
// delta with provenance, neither value is a candidate's billed USD receipt.
if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined;
if (!usage) return undefined;
const measurement = nativeUsageMeasurement(usage);
const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta");
const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])
?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]));
// PRP currently defaults an absent providerCostUsd to zero. That zero is
// not proof of a free request when the runtime observed token consumption.
if (direct !== undefined && direct > 0) return direct;
const normalized = normalizeNativeUsage(usage);
if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0;
const cost = record(hasRunDelta ? measurement.cost : usage.cost);
const currency =
typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD";
if (currency !== "USD") return undefined;
return numericUsageField(cost, ["amount", "total"]);
}
export function normalizeNativeUsage(usage: Record<string, unknown> | null, options: { inputIncludesCacheReads?: boolean } = {}) {
if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined;
const measurement = nativeUsageMeasurement(usage);
const cache = record(measurement.cache);
const count = (source: Record<string, unknown>, keys: string[]) => {
for (const key of keys) {
if (source[key] === undefined) continue;
const value = source[key];
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN;
}
return undefined;
};
const input = count(measurement, ["inputTokens", "input", "promptTokens"]);
const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]);
const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]);
const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]);
if (input === undefined || outputTokens === undefined
|| ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined;
const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0)
+ (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0);
if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined;
return {
inputTokens, outputTokens,
...(cachedInputTokens === undefined ? {} : { cachedInputTokens }),
...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }),
};
}
export function usesNativeUsageDelivery(input: {
runnerd: boolean; remote: boolean; provider?: NativeExecutionInput["provider"] | null;
}) {
return input.runnerd && input.remote && input.provider !== undefined && input.provider !== null
&& ["codex", "opencode", "acpx", "claude_managed", "aws_agentcore"].includes(input.provider.kind);
}
export function isNativeUsageAccountingEvent(event: Pick<PrpEvent, "eventType">) {
return ["usage.reported", "turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType);
}
server/src/services/native-runtime/native-usage-delivery.integration.test.ts
+283
View File
@@ -0,0 +1,283 @@
import { compareCents, usdToUnits } from "@paperclipai/shared";
import { spawn, execFileSync, type ChildProcess } from "node:child_process";
import { once } from "node:events";
import { randomUUID } from "node:crypto";
import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db";
import { type PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
import { accountRunCost } from "../run-cost-accounting.js";
import { usageReceiptSpoolPath } from "../usage-receipts.js";
import { createNativeUsageDelivery } from "./native-usage-delivery.js";
const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url));
const fixtureProfile = process.env.CI ? "release" : "debug";
const billing = { provider: "openai", biller: "openai", billingType: "metered_api" as const };
describe("producer-owned native usage", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: Db;
beforeAll(async () => {
execFileSync("cargo", ["build", ...(fixtureProfile === "release" ? ["--release"] : []), "--locked",
"--manifest-path", resolve(runnerRoot, "Cargo.toml"), "-p", "paperclip-runner-core", "--example", "recovery-fixture"],
{ timeout: 600_000, stdio: "pipe" });
database = await startEmbeddedPostgresTestDatabase("native-usage-delivery-");
db = createDb(database.connectionString);
}, 660_000);
afterAll(async () => { await database?.cleanup(); });
const seed = async () => {
const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "Usage delivery", issuePrefix: randomUUID().slice(0, 8) });
await db.insert(agents).values({ id: agentId, companyId, name: "Producer", adapterType: "paperclip_runner" });
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "running", runtimeMode: "native", costAccountingPending: true });
return { companyId, agentId, runId, runnerInstanceId: randomUUID(), provider: { kind: "codex" as const, model: "test-model" }, billing };
};
const event = (input: Pick<Awaited<ReturnType<typeof seed>>, "runId" | "runnerInstanceId">, sequence: number, type: string, turnId = "turn-1", amount = 0.25): PrpEvent => ({
schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: input.runnerInstanceId,
sourceEventId: `usage-${sequence}`, sourceSeq: sequence, priority: 0, runId: input.runId,
normalizedSessionId: "session-1", turnId, emittedAt: new Date().toISOString(), eventType: type,
payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {},
} as PrpEvent);
it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => {
const input = await seed();
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
const root = mkdtempSync(resolve(tmpdir(), "native-usage-runner-"));
const receiverA = resolve(root, "receiver-a"), receiverB = resolve(root, "receiver-b");
const identity = { runnerInstanceId: input.runnerInstanceId, environmentLeaseId: randomUUID(), runId: input.runId,
normalizedSessionId: randomUUID(), turnId: "turn-1", itemId: "item-1" };
const expectedRunnerVersion = "0.3.0", expectedRunnerDigest = `sha256:${"c".repeat(64)}`;
let receiver: ChildProcess | undefined, child: ChildProcess | undefined;
let exit: Promise<unknown[]> | undefined, failed = 0, lastCaptureError = "";
const receiptFile = resolve(root, "producer.json"), releaseFile = resolve(root, "release"), scriptFile = resolve(root, "script.json");
writeFileSync(scriptFile, JSON.stringify({ steps: 3, waitAfter: 2, retentionLimit: 8, receiptFile, releaseFile, accounting: true }));
const receipts = () => db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId));
try {
await db.execute(sql`CREATE FUNCTION fail_native_usage_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'receipt database unavailable'; END $$`);
await db.execute(sql`CREATE TRIGGER fail_native_usage BEFORE INSERT ON run_usage_receipts FOR EACH ROW EXECUTE FUNCTION fail_native_usage_receipt()`);
const startReceiver = () => new Promise<ChildProcess>((resolve, reject) => {
const process = spawn(globalThis.process.execPath, ["--import", fileURLToPath(new URL("../../../node_modules/tsx/dist/loader.mjs", import.meta.url)), fileURLToPath(new URL("./__fixtures__/usage-receiver.ts", import.meta.url))], {
cwd: fileURLToPath(new URL("../../../../", import.meta.url)), stdio: ["ignore", "ignore", "pipe", "ipc"], env: { ...globalThis.process.env },
});
let diagnostics = "";
process.stderr!.on("data", bytes => { diagnostics = (diagnostics + String(bytes)).slice(-16000); });
const timer = setTimeout(() => { process.kill("SIGKILL"); reject(new Error(`receiver startup timeout: ${diagnostics}`)); }, 60_000);
process.once("exit", code => { clearTimeout(timer); reject(new Error(`receiver exited ${code}: ${diagnostics}`)); });
process.on("message", message => {
const kind = (message as { kind?: string }).kind;
if (kind === "capture-failed") { failed++; lastCaptureError = String((message as { cause?: string }).cause); }
if (kind === "ready") { clearTimeout(timer); resolve(process); }
});
});
let requestId = 0;
const request = (target: ChildProcess, value: Record<string, unknown>) => new Promise<Record<string, unknown>>((resolve, reject) => {
const id = ++requestId;
const timer = setTimeout(() => { target.off("message", listener); reject(new Error("receiver response timeout")); }, 30_000);
const listener = (message: unknown) => {
const result = message as Record<string, unknown>;
if (result.requestId !== id) return;
clearTimeout(timer); target.off("message", listener);
if (result.error) reject(new Error(String(result.error))); else resolve(result);
};
target.on("message", listener);
target.send({ ...value, requestId: id });
});
receiver = await startReceiver();
const firstReceiver = await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverA, identity, input });
const connectUrl = String(firstReceiver.connectUrl), port = Number(new URL(connectUrl).port), ticket = String(firstReceiver.ticket);
child = spawn(resolve(runnerRoot, `target/${fixtureProfile}/examples/recovery-fixture`), ["--fixture-script", scriptFile, "--connect-url", connectUrl,
"--state-dir", resolve(root, "producer"), "--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId,
"--run-id", identity.runId, "--session-id", identity.normalizedSessionId, "--turn-id", identity.turnId, "--item-id", identity.itemId,
"--runner-version", expectedRunnerVersion, "--runner-digest", expectedRunnerDigest, "--max-outbox-bytes", "65536", "--p0-reserve-bytes", "16384", "--reconnect-grace-ms", "120000"],
{ env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: ticket }, stdio: ["ignore", "ignore", "pipe"] });
exit = once(child, "exit");
await request(receiver, { kind: "command", type: "turn.start", commandId: "start-once" });
writeFileSync(releaseFile, "produce");
await vi.waitFor(() => expect(failed).toBeGreaterThan(0), { timeout: 10_000 });
expect(await receipts()).toHaveLength(0);
const producerState = () => JSON.parse(readFileSync(resolve(root, "producer/runner-state.json"), "utf8")) as {
outbox: Array<{ eventType: string; envelope: { payload: { sourceEventId: string } } }>;
};
const pendingUsage = producerState().outbox.find(report => report.eventType === "usage.reported");
expect(pendingUsage).toBeDefined();
const pendingSourceEventId = pendingUsage!.envelope.payload.sourceEventId;
const stoppedReceiver = await request(receiver, { kind: "stop" });
rmSync(receiverA, { recursive: true, force: true });
await db.execute(sql`DROP TRIGGER fail_native_usage ON run_usage_receipts`);
await db.execute(sql`DROP FUNCTION fail_native_usage_receipt()`);
receiver = await startReceiver();
await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverB, identity, input, port, checkpoint: stoppedReceiver.checkpoint });
await vi.waitFor(async () => expect((await receipts()).filter(row => row.complete)).toHaveLength(1), { timeout: 15_000 });
await vi.waitFor(() => expect(producerState().outbox).toEqual([]), { timeout: 10_000 });
expect((await receipts()).some(row => (row.receiptJson.producerEvent as Record<string, unknown>).sourceEventId === pendingSourceEventId)).toBe(true);
expect((await receipts()).filter(row => row.receiptJson.producerEvent && (row.receiptJson.producerEvent as Record<string, unknown>).eventType === "usage.reported")).toHaveLength(1);
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(true);
expect(await accountRunCost(db, input.runId)).toBe(false);
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId));
expect(ledger).toHaveLength(1);
expect(ledger[0].costCents).toBe(25);
const failedBeforeLateReport = failed;
await request(receiver, { kind: "command", type: "turn.steer", commandId: "release-late-usage" });
await vi.waitFor(() => expect(failed).toBeGreaterThan(failedBeforeLateReport), { timeout: 10_000 });
expect(lastCaptureError).toContain("native_usage_after_finalization_requires_reconciliation");
const retainedLate = producerState().outbox.find(report => report.eventType === "usage.reported");
expect(retainedLate).toBeDefined();
expect(retainedLate!.envelope.payload).toMatchObject({ runId: input.runId, sourceInstanceId: input.runnerInstanceId });
expect((await receipts()).filter(row => (row.receiptJson.producerEvent as Record<string, unknown>)?.eventType === "usage.reported")).toHaveLength(1);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1);
expect(existsSync(usageReceiptSpoolPath())).toBe(false);
} finally {
await db.execute(sql`DROP TRIGGER IF EXISTS fail_native_usage ON run_usage_receipts`);
await db.execute(sql`DROP FUNCTION IF EXISTS fail_native_usage_receipt()`);
if (receiver && receiver.exitCode === null && receiver.signalCode === null) receiver.kill("SIGKILL");
if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
await exit;
rmSync(root, { force: true, recursive: true });
}
}, 240_000);
it("preserves source takeover fencing, budget holds, and missing-usage debt", async () => {
const input = await seed();
const first = await createNativeUsageDelivery(db, input);
await db.insert(budgetPolicies).values({ companyId: input.companyId, scopeType: "agent", scopeId: input.agentId,
metric: "billed_cents", windowKind: "lifetime", amount: 10, notifyEnabled: false });
await first.capture(event(input, 1, "usage.reported"));
const [agent] = await db.select().from(agents).where(eq(agents.id, input.agentId));
expect(agent.lifecycleHolds).toContain("budget");
const replacement = { ...input, runnerInstanceId: randomUUID() };
const second = await createNativeUsageDelivery(db, replacement);
await second.capture(event(replacement, 1, "usage.reported", "turn-2", 0.5));
await first.capture(event(input, 2, "turn.completed"));
const [pending] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
expect(pending.usageJson).toMatchObject({ costUsd: 0.5, accountingReceiptReady: false });
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(false);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
await second.capture(event(replacement, 2, "turn.completed", "turn-2"));
expect(await accountRunCost(db, input.runId)).toBe(true);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(50);
});
it.each(["tokens", "cost", "incomplete"])("accepts equal finalized snapshots and rejects changed %s without repricing", async change => {
const input = await seed();
const delivery = await createNativeUsageDelivery(db, input);
await delivery.capture(event(input, 1, "usage.reported"));
await delivery.capture(event(input, 2, "turn.completed"));
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(true);
await delivery.capture(event(input, 3, "usage.reported"));
const late = event(input, 4, "usage.reported");
(late.payload as Record<string, unknown>).runDelta = { inputTokens: change === "tokens" ? 150 : 120, outputTokens: change === "tokens" ? 15 : 10, providerCostUsd: change === "incomplete" ? 0.25 : 0.5 };
if (change === "incomplete") (late.payload as Record<string, unknown>).runDeltaAvailable = false;
await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25);
});
it("accepts equal finalized rate-card reports without requiring terminal-only derived pricing", async () => {
const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } };
const delivery = await createNativeUsageDelivery(db, input);
const report = (sequence: number) => {
const reported = event(input, sequence, "usage.reported");
reported.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 1000, outputTokens: 100000 } };
return reported;
};
await delivery.capture(report(1));
await delivery.capture(event(input, 2, "turn.completed"));
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect(await accountRunCost(db, input.runId)).toBe(true);
await delivery.capture(report(3));
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
expect(run.usageJson).toMatchObject({ pricingProvenance: { source: "rate_card" }, accountingReceiptReady: true });
expect(usdToUnits(run.usageJson!.costUsdExact as string)).toBe(usdToUnits("1.002"));
const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId));
expect(ledger).toHaveLength(1);
expect(ledger[0].costCents).toBe(100.2);
const correction = report(4);
(correction.payload as Record<string, unknown>).runDelta = { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 2 };
await expect(delivery.capture(correction)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation");
});
it("keeps successor exposure pending after prior-turn accounting invalidates its live snapshot", async () => {
const input = await seed();
const delivery = await createNativeUsageDelivery(db, input);
const paid = event(input, 1, "usage.reported");
paid.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 50, outputTokens: 5, providerCostUsd: 0.5 } };
await delivery.capture(paid);
const invalidated = event(input, 2, "usage.reported");
invalidated.payload = { runDeltaBasis: "run", runDeltaAvailable: false, runDelta: {}, reason: "accounting_reconciliation_required",
reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120, outputTokens: 12 }, priorOrigin: { runId: "old-run" } } };
await delivery.capture(invalidated);
await delivery.capture(event(input, 3, "turn.failed"));
await db.update(heartbeatRuns).set({ status: "failed", errorCode: "codex_usage_accounting_integrity", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
expect(run).toMatchObject({ costAccountingPending: true, costAccountedAt: null,
usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } });
expect(compareCents((run.usageJson?.budgetExposure as { costCents: string }).costCents, "50")).toBe(0);
const [required] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 2)));
expect(required.receiptJson.producerEvent).toMatchObject({ usage: { reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120 }, priorOrigin: { runId: "old-run" } } } });
expect(await accountRunCost(db, input.runId)).toBe(false);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
});
it("retains unmarked legacy Codex reports as incomplete unpriced debt", async () => {
const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } };
const delivery = await createNativeUsageDelivery(db, input);
const legacy = event(input, 1, "usage.reported");
delete (legacy.payload as Record<string, unknown>).runDeltaBasis;
await delivery.capture(legacy);
await delivery.capture(event(input, 2, "turn.completed"));
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId));
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)))[0]).toMatchObject({
costAccountingPending: true, costAccountedAt: null, usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } });
expect(await accountRunCost(db, input.runId)).toBe(false);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0);
});
it("replays partial and final usage once after a lost commit response and restores turn aggregation", async () => {
const input = { ...await seed(), provider: { kind: "opencode" as const, model: "test-model" } };
let loseResponse = false;
const failingDb = new Proxy(db, { get(target, property) {
if (property === "transaction") return async (...args: Parameters<Db["transaction"]>) => {
const result = await db.transaction(...args);
if (loseResponse) { loseResponse = false; throw new Error("lost commit response"); }
return result;
};
const value = Reflect.get(target, property);
return typeof value === "function" ? value.bind(target) : value;
} }) as Db;
let delivery = await createNativeUsageDelivery(failingDb, input);
const first = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed");
loseResponse = true;
await expect(delivery.capture(first)).rejects.toThrow("lost commit response");
await delivery.capture(first);
await delivery.capture(done);
delivery = await createNativeUsageDelivery(db, input);
await delivery.capture(first);
await delivery.capture(done);
const nextTurn = event(input, 3, "turn.started", "turn-1");
nextTurn.payload = { providerTurnId: "provider-turn-2" };
await delivery.capture(nextTurn);
const partial = event(input, 4, "usage.reported", "turn-1", 0.5);
(partial.payload as Record<string, unknown>).runDeltaAvailable = false;
await delivery.capture(partial);
const final = event(input, 5, "usage.reported", "turn-1", 0.5);
await delivery.capture(final);
const terminal = event(input, 6, "turn.completed", "turn-1");
terminal.payload = { providerTurnId: "provider-turn-2" };
await delivery.capture(terminal);
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
expect(run.usageJson).toMatchObject({ inputTokens: 240, outputTokens: 20, accountingReceiptReady: true, costUsd: 0.75 });
expect(run.usageJson?.accountingCaptureFailed).not.toBe(true);
expect(await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)))).toHaveLength(1);
await expect(delivery.capture({ ...final, payload: { ...final.payload as object, runDeltaAvailable: false } })).rejects.toThrow("native_usage_source_conflict");
});
});
server/src/services/native-runtime/native-usage-delivery.ts
+109
View File
@@ -0,0 +1,109 @@
import { createHash } from "node:crypto";
import { and, asc, eq, isNull, sql } from "drizzle-orm";
import { heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db";
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import type { NativeExecutionInput, PrpEvent } from "../../vendor/paperclip-runner/index.js";
import { priceAnthropicReceipt } from "../anthropic-pricing.js";
import { priceCodexReceipt } from "../codex-pricing.js";
import { liveRunUsageCostCents } from "../budget-live-usage.js";
import { receiptFingerprint } from "../receipt-fingerprint.js";
import { persistUsageReceipt, type UsageReceiptEnvelope } from "../usage-receipts.js";
import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage } from "./native-usage-accounting.js";
const terminal = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]);
const uuid = (identity: string) => {
const hex = createHash("sha256").update(identity).digest("hex");
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
};
export async function createNativeUsageDelivery(db: Db, input: {
companyId: string; runId: string; runnerInstanceId: string;
provider: NativeExecutionInput["provider"];
billing: Pick<AdapterUsageCheckpoint, "provider" | "biller" | "billingType">;
}) {
const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`);
const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql`
coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb ||
case when ${heartbeatRuns.usageJson}->>'accountingReceiptSourceId' = ${sourceId}
then '{}'::jsonb else ${JSON.stringify({ accountingReceiptSourceId: sourceId, accountingReceiptSequence: 0, accountingReceiptReady: false })}::jsonb end`
}).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId),
isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id });
if (!bound) throw new Error("native_usage_run_not_available");
const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and(
eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId),
eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence));
const accepted = new Map<string, UsageReceiptEnvelope>();
const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null;
let observedUsage: Record<string, unknown> | null = null;
let observedTurn: string | undefined;
let activeProviderTurn: string | undefined;
const observe = (envelope: UsageReceiptEnvelope) => {
const fact = envelope.producerEvent!;
if (fact.eventType === "turn.started") activeProviderTurn = fact.providerTurnId ?? fact.sourceEventId;
const accountingTurn = fact.providerTurnId ?? activeProviderTurn ?? fact.turnId;
const interpretedUsage = fact.usage ? { ...fact.usage, runDeltaComplete: fact.usage.runDeltaComplete === true
&& (input.provider.kind !== "codex" || fact.usage.runDeltaBasis === "run") } : undefined;
const event = { sourceInstanceId: input.runnerInstanceId, sourceSeq: envelope.sequence,
turnId: accountingTurn, eventType: fact.eventType,
payload: interpretedUsage ? { kind: "usage", usage: interpretedUsage } : {} } as PrpEvent;
if (turnAccounting) {
turnAccounting.observe(event);
return turnAccounting.snapshot();
}
if (fact.eventType === "turn.started") observedTurn = undefined;
if (interpretedUsage) {
observedUsage = interpretedUsage;
observedTurn = normalizeNativeUsage(observedUsage, { inputIncludesCacheReads: input.provider.kind === "codex" }) ? accountingTurn : undefined;
}
return { usage: observedUsage, complete: terminal.has(fact.eventType) && observedTurn !== undefined && observedTurn === accountingTurn };
};
for (const row of saved) {
const envelope = row.receipt as UsageReceiptEnvelope;
if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer");
accepted.set(envelope.producerEvent.sourceEventId, envelope);
observe(envelope);
}
let pending: UsageReceiptEnvelope | undefined;
let chain = Promise.resolve();
const capture = async (event: PrpEvent) => {
if (event.eventType !== "usage.reported" && event.eventType !== "turn.started" && !terminal.has(event.eventType)) return;
if (event.runId !== input.runId || event.sourceInstanceId !== input.runnerInstanceId) throw new Error("native_usage_source_mismatch");
const payload = event.payload as Record<string, unknown>;
const fact: NonNullable<UsageReceiptEnvelope["producerEvent"]> = {
sourceEventId: event.sourceEventId, turnId: event.turnId, eventType: event.eventType,
...(typeof payload.providerTurnId === "string" ? { providerTurnId: payload.providerTurnId } : {}),
...(event.eventType === "usage.reported" ? { usage: { runDelta: payload.runDelta, runDeltaComplete: payload.runDeltaAvailable === true, runDeltaBasis: payload.runDeltaBasis,
...(payload.reconciliation && typeof payload.reconciliation === "object" ? { reconciliation: payload.reconciliation } : {}) } } : {}),
};
const previous = accepted.get(event.sourceEventId) ?? (pending?.producerEvent?.sourceEventId === event.sourceEventId ? pending : undefined);
if (previous && (previous.sequence !== event.sourceSeq || previous.receivedAt !== event.emittedAt
|| receiptFingerprint(previous.producerEvent) !== receiptFingerprint(fact))) throw new Error("native_usage_source_conflict");
if (accepted.has(event.sourceEventId)) return;
if (pending && pending.producerEvent?.sourceEventId !== event.sourceEventId) throw new Error("native_usage_report_pending");
if (!pending) {
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1",
id: uuid(`${input.runId}:${input.runnerInstanceId}:${event.sourceEventId}`), companyId: input.companyId,
runId: input.runId, adapterType: "paperclip_runner", sourceId, sequence: event.sourceSeq,
receivedAt: event.emittedAt, producerEvent: fact, receipt: { complete: false } };
const snapshot = observe(envelope);
const usage = normalizeNativeUsage(snapshot.usage, { inputIncludesCacheReads: input.provider.kind === "codex" });
envelope.receipt = priceAnthropicReceipt(priceCodexReceipt({ ...input.billing,
model: input.provider.model ?? "unknown", usage, usageBasis: "per_run", complete: snapshot.complete && usage !== undefined,
costUsd: usage ? nativeUsageCostUsd(snapshot.usage, input.provider) ?? null : null,
costUsdExact: typeof snapshot.usage?.accountingCostUsdExact === "string" ? snapshot.usage.accountingCostUsdExact : undefined,
costStatus: usage === undefined || snapshot.usage?.accountingCostIncomplete === true ? "unpriced"
: input.provider.kind === "aws_agentcore" ? "estimated" : undefined,
}));
envelope.liveCostCents = liveRunUsageCostCents(envelope.receipt as AdapterUsageCheckpoint);
pending = envelope;
}
await persistUsageReceipt(db, pending);
accepted.set(event.sourceEventId, pending);
pending = undefined;
};
return { capture(event: PrpEvent) {
const result = chain.then(() => capture(event));
chain = result.catch(() => undefined);
return result;
} };
}
server/src/services/native-runtime/paperclip-control-plane-port.ts
+6 -6
View File
@@ -88,8 +88,8 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
readonly #assertRunAdmissionActive?: () => void;
readonly #onResultCommitted?: () => Promise<void>;
readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
readonly #onCommittedEvent?: (event: PrpEvent) => Promise<void>;
readonly #onDuplicateEvent?: (event: PrpEvent) => Promise<void>;
readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
constructor(
db: Db,
@@ -103,8 +103,8 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
assertRunAdmissionActive?: () => void;
onResultCommitted?: () => Promise<void>;
onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise<void>;
onCommittedEvent?: (event: PrpEvent) => Promise<void>;
onDuplicateEvent?: (event: PrpEvent) => Promise<void>;
onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise<void>;
} = {},
) {
this.#db = db;
@@ -313,13 +313,13 @@ export class PaperclipControlPlanePort implements ControlPlanePort {
});
} catch {
}
await this.#onCommittedEvent?.(event);
await this.#onCommittedEvent?.(event, persisted.row.seq);
} else {
// A recovered runner may replay the event whose durable side effects
// parked the prior attempt. Do not repeat those effects, but let the
// embedding runtime refresh observational state before appendEvent
// returns to its synchronous governed-wait boundary.
await this.#onDuplicateEvent?.(event);
await this.#onDuplicateEvent?.(event, persisted.row.seq);
}
return {
cursor: persisted.row.seq,
server/src/services/object-byte-journal.test.ts
+326
View File
@@ -0,0 +1,326 @@
import { createHash } from "node:crypto";
import { Readable } from "node:stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { StorageProvider } from "../storage/types.js";
import { createObjectByteJournal, ObjectJournalConflictError } from "./object-byte-journal.js";
async function readBody(body: AsyncIterable<Uint8Array>) {
const chunks: Buffer[] = [];
for await (const bytes of body) chunks.push(Buffer.from(bytes));
return Buffer.concat(chunks);
}
function storage() {
const objects = new Map<string, Buffer>();
const calls = { get: 0, put: 0, maxPut: 0 };
let loseHeadResponse = false;
const etag = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex");
const missing = () => Object.assign(new Error("missing"), { name: "NoSuchKey" });
const conflict = () => Object.assign(new Error("conflict"), { name: "PreconditionFailed" });
const provider: StorageProvider = {
id: "s3",
async getObject({ objectKey, signal }) {
signal?.throwIfAborted();
calls.get++;
const bytes = objects.get(objectKey);
if (!bytes) throw missing();
return { stream: Readable.from([bytes]), etag: etag(bytes), contentLength: bytes.length };
},
async putObject() {
throw new Error("unconditional PUT forbidden");
},
async putObjectConditional(input) {
const old = objects.get(input.objectKey);
if (input.ifNoneMatch ? old : !old || etag(old) !== input.ifMatch) throw conflict();
const bytes = Buffer.isBuffer(input.body) ? input.body : await readBody(input.body);
objects.set(input.objectKey, Buffer.from(bytes));
calls.put++;
calls.maxPut = Math.max(calls.maxPut, bytes.length);
if (loseHeadResponse && input.objectKey.endsWith("/head.json")) {
loseHeadResponse = false;
throw new Error("response lost after PUT");
}
return { etag: etag(bytes) };
},
async headObject({ objectKey }) {
return { exists: objects.has(objectKey) };
},
async deleteObject({ objectKey }) {
objects.delete(objectKey);
},
async listObjects({ prefix }) {
return { keys: [...objects.keys()].filter((k) => k.startsWith(prefix)) };
},
};
return {
provider,
objects,
calls,
loseResponse: () => {
loseHeadResponse = true;
},
};
}
const authority = async () => undefined;
afterEach(() => vi.useRealTimers());
describe("object byte journal", () => {
it("keeps a shared capability probe alive for another writer when one claim cancels", async () => {
const s = storage();
const put = s.provider.putObjectConditional!;
let release!: () => void, started!: () => void, admitted!: () => void;
let probeSignal: AbortSignal | undefined;
const probing = new Promise<void>((resolve) => {
started = resolve;
});
const joined = new Promise<void>((resolve) => {
admitted = resolve;
});
s.provider.putObjectConditional = async (input) => {
if (input.objectKey.includes("conditional-probe-") && !probeSignal) {
probeSignal = input.signal;
await new Promise<void>((resolve, reject) => {
release = resolve;
input.signal!.addEventListener("abort", () => reject(input.signal!.reason), {
once: true,
});
started();
});
}
return put(input);
};
const controller = new AbortController();
const a = createObjectByteJournal({ provider: s.provider, prefix: "company/a/log" });
const first = a.claim({
expectedOwner: null,
assertAuthority: authority,
signal: controller.signal,
});
void first.catch(() => undefined);
await probing;
const b = createObjectByteJournal({ provider: s.provider, prefix: "company/b/log" });
const second = b.claim({
expectedOwner: null,
assertAuthority: async () => {
admitted();
},
});
await joined;
await Promise.resolve();
controller.abort();
await expect(first).rejects.toMatchObject({ name: "AbortError" });
expect(probeSignal!.aborted).toBe(false);
release();
const writer = await second;
expect((await b.head())!.head.owner).toBe(writer.head.owner);
});
it("recovers committed bytes and stable ranges with no local filesystem", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "company/run/log" });
let writer = await a.claim({ expectedOwner: null, assertAuthority: authority });
const expected = Buffer.from("first\n再開\n".repeat(20));
writer = await a.append(writer, expected.subarray(0, 17));
writer = await a.append(writer, expected.subarray(17));
const b = createObjectByteJournal({ provider: s.provider, prefix: "company/run/log" });
const snapshot = (await b.head())!;
expect(await b.readRange(snapshot.head, 0, expected.length)).toEqual(expected);
expect(await b.readRange(snapshot.head, 7, 23)).toEqual(expected.subarray(7, 30));
expect(await b.readRange(snapshot.head, expected.length, 100)).toEqual(Buffer.alloc(0));
});
it("fences stale appends/finalization and preserves a pinned reader snapshot", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let old = await a.claim({ expectedOwner: null, assertAuthority: authority });
old = await a.append(old, Buffer.from("before"));
const snapshot = old.head;
let next = await a.claim({ expectedOwner: old.head.owner, assertAuthority: authority });
next = await a.append(next, Buffer.from("after"));
await expect(a.append(old, Buffer.from("lost"))).rejects.toBeInstanceOf(
ObjectJournalConflictError,
);
await expect(a.close(old)).rejects.toBeInstanceOf(ObjectJournalConflictError);
expect(await a.readRange(snapshot, 0, 100)).toEqual(Buffer.from("before"));
expect(await a.readRange(next.head, 0, 100)).toEqual(Buffer.from("beforeafter"));
});
it("reconciles an uncertain manifest commit without duplicating output", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await a.claim({ expectedOwner: null, assertAuthority: authority });
s.loseResponse();
writer = await a.append(writer, Buffer.from("once"));
expect(writer.head.end).toBe(4);
expect(await a.readRange((await a.head())!.head, 0, 100)).toEqual(Buffer.from("once"));
});
it("keeps index objects bounded over multiple tree levels", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await a.claim({ expectedOwner: null, assertAuthority: authority });
for (let i = 0; i < 180; i++)
writer = await a.append(writer, Buffer.from(String(i).padStart(4, "0")));
expect(await a.readRange(writer.head, 252, 16)).toEqual(Buffer.from("0063006400650066"));
expect(await a.readRange(writer.head, 700, 20)).toEqual(Buffer.from("01750176017701780179"));
expect(s.calls.maxPut).toBeLessThan(20_000);
expect(s.calls.put).toBeLessThan(180 * 5);
const reachable = new Set<string>();
function visit(ref: { key: string }, level: number) {
reachable.add(ref.key);
if (level === 0) return;
const node = JSON.parse(s.objects.get(ref.key)!.toString());
for (const child of node.children) visit(child, level - 1);
}
for (const root of writer.head.roots!) visit(root, root.level);
expect([...reachable].sort()).toEqual(
[...s.objects.keys()].filter((key) => key.includes("/objects/")).sort(),
);
});
it("retries a transient head conflict without fencing its current writer", async () => {
const s = storage(),
journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority });
const original = s.provider.putObjectConditional!;
let conflicts = 0;
s.provider.putObjectConditional = async (input) => {
if (input.objectKey.endsWith("/head.json") && conflicts++ === 0)
throw Object.assign(new Error("in-flight conflict"), {
name: "ConditionalRequestConflict",
$metadata: { httpStatusCode: 409 },
});
return original(input);
};
writer = await journal.append(writer, Buffer.from("once"));
expect(await journal.readRange(writer.head, 0, 100)).toEqual(Buffer.from("once"));
expect(conflicts).toBe(2);
});
it("does not accept a provider's 409 response as proof of conditional enforcement", async () => {
const s = storage(),
original = s.provider.putObjectConditional!;
s.provider.putObjectConditional = async (input) => {
if (s.objects.has(input.objectKey))
throw Object.assign(new Error("conflict"), { $metadata: { httpStatusCode: 409 } });
return original(input);
};
const journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
await expect(
journal.claim({ expectedOwner: null, assertAuthority: authority }),
).rejects.toThrow("conflict");
expect(s.objects.size).toBe(0);
});
it("retries transient segment failures and acknowledges exactly one batch", async () => {
const s = storage(),
journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority });
const original = s.provider.putObjectConditional!;
let failures = 2;
s.provider.putObjectConditional = async (input) => {
if (input.objectKey.includes("/objects/") && failures-- > 0)
throw Object.assign(new Error("temporary outage"), { $metadata: { httpStatusCode: 503 } });
return original(input);
};
writer = await journal.append(writer, Buffer.from("one batch"));
expect(await journal.readRange(writer.head, 0, 100)).toEqual(Buffer.from("one batch"));
});
it("rejects corrupt published data and cancelled reads", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await a.claim({ expectedOwner: null, assertAuthority: authority });
writer = await a.append(writer, Buffer.from("data"));
const data = [...s.objects].find(([, bytes]) => bytes.toString() === "data")!;
s.objects.set(data[0], Buffer.from("oops"));
await expect(a.readRange(writer.head, 0, 4)).rejects.toThrow("integrity");
await expect(a.readRange(writer.head, 0, 4, AbortSignal.abort())).rejects.toThrow();
});
it("retains a deletion tombstone and denies a subsequent claim", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await a.claim({ expectedOwner: null, assertAuthority: authority });
writer = await a.append(writer, Buffer.from("private"));
await a.close(writer);
await a.remove();
expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]);
expect((await a.head())!.head.deleted).toBe(true);
await expect(
a.claim({ expectedOwner: writer.head.owner, assertAuthority: authority }),
).rejects.toThrow();
});
it("does not claim ownership when execution authority rejects", async () => {
const s = storage(),
a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
await expect(
a.claim({
expectedOwner: null,
assertAuthority: async () => {
throw new Error("revoked");
},
}),
).rejects.toThrow("revoked");
expect(s.calls.put).toBe(0);
});
it("fences a pending first claim when deletion wins before journal creation", async () => {
const s = storage(),
journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let enter!: () => void, release!: () => void;
const entered = new Promise<void>((resolve) => {
enter = resolve;
});
const blocked = new Promise<void>((resolve) => {
release = resolve;
});
const pending = journal.claim({
expectedOwner: null,
assertAuthority: async () => {
enter();
await blocked;
},
});
await entered;
await journal.remove();
release();
await expect(pending).rejects.toBeInstanceOf(ObjectJournalConflictError);
expect((await journal.head())!.head.deleted).toBe(true);
expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]);
});
it("rejects a compatible provider which silently ignores conditional writes", async () => {
const s = storage();
s.provider.putObjectConditional = async (input) => {
const bytes = Buffer.from(input.body as Buffer);
s.objects.set(input.objectKey, bytes);
return { etag: "ignored-condition" };
};
const journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
await expect(
journal.claim({ expectedOwner: null, assertAuthority: authority }),
).rejects.toThrow("does not enforce conditional");
expect(s.objects.size).toBe(0);
});
it("fences an active writer before cleanup and preserves the retry obligation", async () => {
vi.useFakeTimers({ toFake: ["Date"] });
const s = storage(),
journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority });
writer = await journal.append(writer, Buffer.from("private"));
await expect(journal.remove()).rejects.toThrow("cleanup_pending");
expect((await journal.head())!.head.deleted).toBe(true);
await expect(journal.append(writer, Buffer.from("late"))).rejects.toBeInstanceOf(
ObjectJournalConflictError,
);
vi.setSystemTime(Date.now() + 30_001);
await journal.remove();
expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]);
});
it("does not publish a partial batch when a later segment fails", async () => {
const s = storage(),
journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" });
const writer = await journal.claim({ expectedOwner: null, assertAuthority: authority });
const original = s.provider.putObjectConditional!;
const failed = Buffer.alloc(1024 * 1024, 2);
const key = createHash("sha256").update(failed).digest("hex");
s.provider.putObjectConditional = (input) => {
if (input.objectKey.endsWith(key)) throw Error("second segment failed");
return original(input);
};
await expect(
journal.append(writer, Buffer.concat([Buffer.alloc(1024 * 1024, 1), failed])),
).rejects.toThrow("second segment failed");
expect((await journal.head())!.head.end).toBe(0);
});
});
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff. Show more