Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-30 21:49:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge current Runner D delivery

Browse Source 
Preserve the latest workflow coverage, repair eval-kernel build ordering, and type the transient ACP trace callback.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-23 20:41:14 +00:00
2 parents 2c5c2f0fce 5598e0b5ea
commit f042711b5d
244 files changed
+63659 -1710
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/pr.yml
+48 -1
View File
@@ -230,11 +230,56 @@ jobs:
pnpm test:run:general -- --group '${{ matrix.group }}'
fi
runner_workflow_evals:
name: Runner workflow evals
needs: [policy]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
- name: Restore regenerated PR lockfile (if policy uploaded one)
uses: actions/download-artifact@v8
continue-on-error: true
with:
name: pr-lockfile
path: .
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Run deterministic Runner workflow gate
run: |
pnpm test:runner-workflow-evals
pnpm report:runner-workflow-evals
- name: Upload Runner workflow report
if: always()
uses: actions/upload-artifact@v7
with:
name: runner-workflow-evals
path: packages/paperclip-runner/.paperclip-local/evals/workflows/
retention-days: 30
if-no-files-found: error
verify:
# Preserve the legacy required-check name while the underlying work runs in parallel.
name: verify
if: ${{ always() }}
needs: [typecheck_release_registry, general_tests, build]
needs: [typecheck_release_registry, general_tests, runner_workflow_evals, build]
runs-on: ubuntu-latest
timeout-minutes: 5
@@ -243,10 +288,12 @@ jobs:
env:
TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }}
GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }}
RUNNER_WORKFLOW_EVALS_RESULT: ${{ needs.runner_workflow_evals.result }}
BUILD_RESULT: ${{ needs.build.result }}
run: |
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
test "$GENERAL_TESTS_RESULT" = "success"
test "$RUNNER_WORKFLOW_EVALS_RESULT" = "success"
test "$BUILD_RESULT" = "success"
build:
.github/workflows/release-verify.yml
+47
View File
@@ -9,6 +9,12 @@ on:
type: string
jobs:
runner_chaos_evals:
name: Pre-release Runner chaos evals
uses: ./.github/workflows/runner-chaos-evals.yml
with:
ref: ${{ inputs.ref }}
typecheck:
name: Typecheck
runs-on: ubuntu-latest
@@ -155,6 +161,47 @@ jobs:
- name: Run serialized server test shard
run: pnpm test:run:serialized -- --shard-index ${{ matrix.shard_index }} --shard-count ${{ matrix.shard_count }}
runner_workflow_evals:
name: Runner workflow evals
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref }}
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Run deterministic Runner workflow gate
run: |
pnpm test:runner-workflow-evals
pnpm report:runner-workflow-evals
- name: Upload Runner workflow report
if: always()
uses: actions/upload-artifact@v7
with:
name: runner-workflow-evals-${{ github.run_id }}
path: packages/paperclip-runner/.paperclip-local/evals/workflows/
retention-days: 30
if-no-files-found: error
build:
name: Build
runs-on: ubuntu-latest
.github/workflows/runner-chaos-evals.yml
+83
View File
@@ -0,0 +1,83 @@
name: Runner Chaos Evals
on:
schedule:
- cron: "43 7 * * 0"
workflow_dispatch:
workflow_call:
inputs:
ref:
description: Commit SHA, branch, or tag to verify before release
required: false
type: string
concurrency:
group: runner-chaos-evals-${{ inputs.ref || github.ref }}
cancel-in-progress: true
jobs:
chaos_and_recovery:
name: Restart, replay, trace, and recovery faults
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v7
with:
ref: ${{ inputs.ref || github.sha }}
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build eval and Runner contracts
run: |
pnpm --filter @paperclipai/paperclip-eval-kernel build
pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals
- name: Run Runner fault and replay suites
run: |
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
src/eval/workflow-evals.test.ts \
src/native-session-runtime.test.ts \
src/live/live-session.test.ts \
src/live/turn-stream.test.ts \
src/protocol/replay-contract.test.ts \
src/drivers/opencode/mcp-bridge.test.ts \
src/drivers/acpx/acpx-runtime-host.test.ts
- name: Build server test dependencies
run: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps
- name: Run server finalization and recovery suites
run: |
pnpm --filter @paperclipai/server exec vitest run \
src/__tests__/native-finalization-recovery.test.ts \
src/__tests__/heartbeat-process-recovery.test.ts \
src/__tests__/heartbeat-comment-wake-batching.test.ts \
src/__tests__/heartbeat-dependency-scheduling.test.ts \
src/__tests__/provider-trace-store.test.ts \
src/services/issue-thread-interaction-resolution.test.ts \
src/services/recovery/successful-run-handoff.test.ts
- name: Upload chaos eval bundle
if: always()
uses: actions/upload-artifact@v7
with:
name: runner-chaos-evals-${{ github.run_id }}
path: packages/paperclip-runner/.paperclip-local/evals/workflows/
retention-days: 30
if-no-files-found: error
.github/workflows/runner-live-evals.yml
+74
View File
@@ -0,0 +1,74 @@
name: Runner Live Evals
on:
schedule:
- cron: "17 6 * * *"
workflow_dispatch:
concurrency:
group: runner-live-evals-${{ github.ref }}
cancel-in-progress: true
jobs:
live_matrix:
name: Balanced provider/model matrix
runs-on: ubuntu-latest
timeout-minutes: 180
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@v7
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Restore compatible seven-day baseline
uses: actions/cache@v5
with:
path: packages/paperclip-runner/.paperclip-local/evals/workflows/history
key: runner-live-eval-history-${{ github.ref_name }}-${{ github.run_id }}
restore-keys: |
runner-live-eval-history-${{ github.ref_name }}-
- name: Build provider-neutral eval kernel
run: pnpm --filter @paperclipai/paperclip-eval-kernel build
- name: Run trend-only live matrix
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
PAPERCLIP_EVAL_BASELINE_READY: "true"
PAPERCLIP_EVAL_RUNNER_BUILD: ${{ github.sha }}
PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD: "12"
PAPERCLIP_EVAL_SCHEDULE_SEED: runner-live-seven-day-v1
run: pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals
- name: Publish job summary
if: always()
run: |
summary=packages/paperclip-runner/.paperclip-local/evals/workflows/github-live-summary.md
if [ -f "$summary" ]; then
cat "$summary" >> "$GITHUB_STEP_SUMMARY"
fi
- name: Upload safe live eval bundle
if: always()
uses: actions/upload-artifact@v7
with:
name: runner-live-evals-${{ github.run_id }}
path: packages/paperclip-runner/.paperclip-local/evals/workflows/
retention-days: 30
if-no-files-found: error
doc/plans/2026-08-23-paperclip-runner-stress-campaign.md
+198
View File
@@ -0,0 +1,198 @@
# Paperclip Runner six-hour stress campaign
Started: 2026-08-23 05:48 America/Chicago
This is the durable evidence ledger for the provider-neutral Paperclip Runner
stress campaign. It records only non-secret configuration and links to the
authoritative local issues/runs/traces.
## Baseline
- Instance: `http://localhost:3100`, company `DOT`
- Branch: `PAP-16679-paperclip-runner` at `a70b0f70` with pre-existing local changes
- Server: `2026.722.0+816.git.a70b0f70.dirty`
- Running provider agents:
- Runner: `paperclip_runner`, Codex, `gpt-5.6-luna`, raw tracing enabled
- OpenCode Runner: `paperclip_runner`, OpenCode, `opencode/big-pickle`, raw tracing enabled
- Environment probes:
- Codex `gpt-5.6-luna`: pass
- OpenCode `opencode/big-pickle`: pass
- ACPX Claude `claude-sonnet-5`: pass after binding the existing encrypted company Anthropic secret; authenticated live turn completed
- ACPX Codex `gpt-5.6-sol`: pass
- ACPX Pi `openrouter/deepseek/deepseek-v4-flash-0731`: pass after importing `OPENROUTER_API_KEY` into the encrypted company secret store and binding it by reference
- Model substitutions:
- No substitution is currently required: the live OpenRouter catalog qualified Kimi Code, GLM, GPT-OSS, and GLM Flash configurations.
- Secret values are never persisted in agent configuration, this ledger, trace metadata, or command output.
- Canonical PRP action roster: 41 actions; 14 production bindings; remaining actions use safe isolated conformance fixtures where live execution is not authoritative or would be destructive.
## Live execution ledger
| Time | Provider/config | Scenario | Issue | Run | Expected | Actual | Trace | Result |
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
| 11:20Z | ACPX Claude / `claude-sonnet-5` | Ask-mode long answer (pre-fix control) | [DOT-102](http://localhost:3100/DOT/issues/DOT-102) | `faddb6ec-c80b-4297-a098-c400d8d30ff8` | Immediate commentary, long final, done | Rejected before provider: native runtime excluded ask mode; two bounded continuations, then blocked | Incomplete, 0 frames | FAIL |
| 11:20Z | OpenCode / `nemotron-3.5-lightning-free` | Ask-mode concise answer (pre-fix control) | [DOT-104](http://localhost:3100/DOT/issues/DOT-104) | `fd4e5a65-2646-46ed-87bf-4fcf0d63fb73` | Immediate commentary, concise final, done | Rejected before provider by the same ask-mode gate; bounded, not infinite | Incomplete, 0 frames | FAIL |
| 11:20Z | ACPX Codex / `gpt-5.6-sol` | Exact 11-byte file (pre-fix control) | [DOT-103](http://localhost:3100/DOT/issues/DOT-103) | `eb7fcbb1-f527-458d-bd0c-ae6ec30712d2` | File/tool stream, byte/hash evidence, done | Launch omitted ACPX runtime directory; recovery then failed closed on missing checkpoint | Capturing metadata, 0 frames | FAIL |
| 11:20Z | OpenCode / `nemotron-3-ultra-free` | Dependency-free module plus `node:test` | [DOT-105](http://localhost:3100/DOT/issues/DOT-105) | `20f193a6-6bab-43a9-a8cc-69f4333d5f67` | Commentary/reasoning/tool/file/test stream, exact final, done | 107 durable events; commands and writes visible; 8/8 tests; one final assistant comment; issue done | Complete, 507 frames / 328,229 bytes | PASS |
| 11:24Z | OpenCode / `nemotron-3.5-lightning-free` | Ask-mode concise answer (post-fix) | [DOT-106](http://localhost:3100/DOT/issues/DOT-106) | `c2173057-6243-484f-9fce-c8c6ad192913` | Direct answer, no file mutation, done | Admitted with a v2 ask envelope; provider turn in progress | Capturing | RUNNING |
| 11:24Z | ACPX Claude / `claude-sonnet-5` | Ask-mode long answer (post-fix) | [DOT-107](http://localhost:3100/DOT/issues/DOT-107) | `f487c4c0-7923-43bf-8468-4ce48015d06d` | Direct long answer | Ask envelope admitted, then provider emitted actionable `Authentication required` notice and `turn.failed`; bounded retry scheduled | Capturing, 0 frames | EXPECTED ENV FAILURE |
| 11:24Z | ACPX Codex / `gpt-5.6-sol` | Exact 11-byte file (post-fix) | [DOT-108](http://localhost:3100/DOT/issues/DOT-108) | `6ec04087-9c29-47ce-b038-7cb2598c6668` | Commentary/tool stream, byte/hash evidence, done | ACPX launched; streamed agent text/thinking and shell execution; verified 11 bytes and SHA-256 `e49c81…d78ee`; finalization pending | Capturing | RUNNING |
| 11:31Z | ACPX Pi / OpenRouter DeepSeek Flash | Exact JSON file plus runtime approval | [DOT-109](http://localhost:3100/DOT/issues/DOT-109) | `8bbba551-6b1a-4603-bc61-a098e6dc9b5e` | Immediate commentary, approval, exact file/evidence, done | Immediate agent message and rich tool status; write outside the bound workspace was correctly denied; two provider approval requests expired/cancelled before the API accepted a response; provider continued with task context | Capturing | RUNNING / APPROVAL RACE |
| 11:31Z | ACPX Claude / `claude-sonnet-5` | Authenticated ask-mode response | [DOT-110](http://localhost:3100/DOT/issues/DOT-110) | `22ddf545-50fa-460a-924d-bd896b5fb4e3` | Immediate streamed answer, semantic result, done | 48 events; authenticated turn streamed the requested answer, emitted accepted semantic result, materialized one response, issue done | Captured | PASS |
| 11:31Z | OpenCode / OpenRouter Kimi Code | Multi-file slug library and tests | [DOT-111](http://localhost:3100/DOT/issues/DOT-111) | `8ec8f8ac-1420-44a4-aa86-f8722ae93b9a` | Reasoning/tools/files/tests/final, done | 70 events; model identity correct, implementation and verification completed, accepted semantic result, issue done | Captured | PASS |
| 11:31Z | OpenCode / OpenRouter GLM | Planning workflow | [DOT-112](http://localhost:3100/DOT/issues/DOT-112) | `bb5d2541-a0e4-4250-af97-e9592071d5e6` | Canonical plan, human confirmation, no implementation before approval | 81 events; browser showed the canonical plan card and Plan tab. Confirm created a narrow continuation, which delegated implementation to DOT-115; parent became legitimately dependency-blocked instead of implementing prematurely | Captured | PASS |
| 11:31Z | OpenCode / OpenRouter GPT-OSS 20B | Ask-mode completion-tool robustness | [DOT-113](http://localhost:3100/DOT/issues/DOT-113) | `df8c4715-69e6-405c-b116-93f903a78afd` | Concise answer and valid semantic finish | Exact trace proved the cheap model used schema alias `paperclip_paperclip_finish`, disposition `completed`, and criterion `passed`. Normalization compatibility was added. A forced server restart then exposed a stale-active terminal checkpoint in recovery | Capturing | FIXED / LIVE RETEST PENDING |
| 11:31Z | OpenCode / OpenRouter GLM Flash | File task with intentionally unavailable verifier | [DOT-114](http://localhost:3100/DOT/issues/DOT-114) | `a5b9b353-ac6d-4d03-a355-2a9c724223c9` | `not_run/tool_unavailable` caveat, accepted low-risk completion, done | File work and semantic result completed, but the UI selected an opening progress sentence as the durable final reply because OpenCode labels every text part identically | Captured | FAIL / PRESENTATION |
| 11:37Z | OpenCode / OpenRouter GLM | Accepted-plan implementation subtask | [DOT-115](http://localhost:3100/DOT/issues/DOT-115) | `8c01a6f6-b4a1-474b-9d4c-e05118456fc1` | Implement the approved plan, test, return result to parent | Created the CSV parser and tests; 15/15 pass. Survived forced server restart and process-loss recovery, settled done, and retained the parent dependency lineage | Captured | PASS |
| 11:40Z | OpenCode / OpenRouter GLM Flash | Post-fix unavailable-verifier regression | [DOT-116](http://localhost:3100/DOT/issues/DOT-116) | `dd09ed68-de34-460c-9720-fd4f5c664aba` | Opening commentary during work, exact post-work result as final, accepted `not_run/tool_unavailable` caveat | Issue done; presentation selected the substantive post-work response, not opening commentary; caveat accepted without a retry | Captured | PASS |
| 12:37Z | OpenCode / OpenRouter GPT-OSS 20B | Alias normalization and recovered-session completion | [DOT-113](http://localhost:3100/DOT/issues/DOT-113) | `6edfa131-af84-4f13-a06d-71cc201223e3` | Accept known provider aliases, reuse the provider session, complete once | Issue done after the recovered same-session turn; normalized semantic result accepted with criterion satisfied and canonical verification | Complete, 1,736 frames / 514,978 bytes, digest verified | PASS |
| 12:40Z | OpenCode / OpenRouter GPT-OSS 20B | Invalid completion behavior and bounded exhaustion | [DOT-117](http://localhost:3100/DOT/issues/DOT-117) | `29a9b534-92b7-4edf-8024-307bfba5b449` | Prose cannot impersonate semantic authority; retries bounded; failure visibly owned | Model printed completion-shaped JSON as ordinary final text on all three same-run attempts. Paperclip correctly rejected it, preserved the trace, stopped after attempt 3, and did not mark work done. Live control exposed stale cross-run watchdog counts and a non-actionable agent-owned terminal wait | Captured | EXPECTED PROVIDER FAILURE / RECOVERY FIXED, RETEST PENDING |
| 12:56Z | OpenCode / OpenRouter GPT-OSS 20B | Post-fix result-less exhaustion ownership | [DOT-121](http://localhost:3100/DOT/issues/DOT-121) | `44f1e71b-b0cd-424a-afdc-9b95bca65cd3` | Three same-run attempts, named terminal failure, board-owned recovery, no extra provider run | Weak model again printed rather than invoked the result. Run ended `failed/native_session_retry_exhausted`; issue is `in_review`; active action is board-owned with return owner, attempt 1/3, and no wake. No blind retry followed | Complete, 1,480 frames / 405,060 bytes | PASS (EXPECTED PROVIDER FAILURE) |
| 12:56Z | Codex app-server / `gpt-5.6-luna` | Structured ask-user, human answer, same-task continuation, exact bytes | [DOT-122](http://localhost:3100/DOT/issues/DOT-122) | `5703966e-b516-4dc6-bdd0-f480b8cb2be1` → `af84fd0a-7e70-4dfc-afdc-0095efe863c5` | Ask before work; visible owned wait; answer Beta; resume and write exactly `Beta\n`; one settled response | Browser verified immediate commentary, question card, Beta answer, continuation commentary/tools, exact hex `426574610a`, and one primary final reply in chronological order. Issue done | Complete, 52 + 159 frames; correlated completion group has 4 frames → 15 stages → 6 PRP events | PASS |
| 12:57Z | OpenCode / OpenRouter Kimi Code | Queued steering after a nearly settled turn | [DOT-123](http://localhost:3100/DOT/issues/DOT-123) | `3f73546b-87e9-4704-b3f2-ddcdfb57745a` → `bc6711e9-4c59-4f58-a00c-8a962a5d3b53` | Preserve first result, place queued message at consumption point, narrow continuation, no redo | Browser showed original snake_case result, then queued board message, then a same-provider-session continuation changing only the requested format. Kimi reused files, updated code/tests/docs, and finished with 25/25 tests | Complete, 1,778 + 934 frames | PASS WITH PRESENTATION DEFECT |
| 12:59Z | OpenCode / OpenRouter GLM Flash | Agent debug tracing disabled for one run | [DOT-124](http://localhost:3100/DOT/issues/DOT-124) | `afb4af9e-6def-4f3a-b7f3-f82263469dc5` | Run succeeds while raw store remains empty; PRP/presentation inspector remains available; setting restored | Issue done; runner profile snapshot contains no provider trace; trace API returns null/0; browser Inspector says capture was off and offers re-run with trace while showing canonical PRP/presentation | Off (0 frames); agent tracing restored immediately after run snapshot | PASS |
| 13:01Z | OpenCode / OpenRouter Kimi Code | Substantive final followed by missing-semantic-result recovery | [DOT-125](http://localhost:3100/DOT/issues/DOT-125) | `3b1e8460-8c55-4ffb-bc3f-13fecfbdee69` | Preserve the real final while obtaining disposition only; no duplicate implementation | Same native session recovered, but presentation selected the recovery acknowledgement instead of the pre-recovery substantive final | Complete, 860 frames / 368,508 bytes | FAIL / FIXED |
| 13:06Z | OpenCode / OpenRouter Kimi Code | Fresh post-fix multi-file implementation | [DOT-126](http://localhost:3100/DOT/issues/DOT-126) | `b59ac6ba-280e-4eec-94d3-1c0ab969a343` | Normal completion without semantic recovery; substantive final; done | 13/13 tests, one accepted result, substantive response, issue done, no retry loop | Captured | PASS |
| 13:12Z | Codex app-server / `gpt-5.6-luna` | True same-turn steering plus idempotent replay | [DOT-127](http://localhost:3100/DOT/issues/DOT-127) | `b8f9448c-2007-4c48-b388-be640903014b` | Inject steering into the active turn, acknowledge once, do not restart, preserve causal UI order | Steering accepted in 200ms; replay was idempotent; agent explicitly incorporated the requirement without restart; 55/55 tests. Browser now splits the settled activity at the durable acknowledgement: pre-steer Worked → human steering bubble → post-steer Worked → final reply | Complete; acknowledgement maps to durable PRP and activity lineage | PASS |
| 13:17Z | Codex app-server / `gpt-5.6-luna` | Native web search and correlated presentation | [DOT-128](http://localhost:3100/DOT/issues/DOT-128) | `e3b29f15-96c2-4ece-b266-0a5ed391d430` | Immediate commentary, rich search status, cited final, done | Browser showed immediate commentary, `Searched once`, concise reasoning, and a cited final. Inspector correlated raw `webSearch` frames 50–51 through 10 mapping stages into four PRP events and the visible research component | 360 frames / 176.9 KiB | PASS |
| 13:20Z | OpenCode / OpenRouter Kimi Code | Real failed-tool recovery and test completion | [DOT-129](http://localhost:3100/DOT/issues/DOT-129) | `98678b97-e35d-4e3f-8474-6133c0f2f66e` | Surface the failure, recover without looping, verify, done | Provider recovered from the genuine failed tool, completed implementation, and passed 9/9 tests; issue settled with a substantive final and artifact | Captured | PASS |
| 13:25Z | OpenCode / OpenRouter GLM | Human-only completion review | [DOT-130](http://localhost:3100/DOT/issues/DOT-130) | `b9a48b05-23c6-420a-bdda-5a5977c82f0e` | External verification becomes a named human review, never a blind provider retry | Issue entered `in_review`; browser card shows `Assigned to You · board only`, evidence, and durable review interaction `9975a62b…` | Captured | PASS / REJECT CONTINUATION PENDING |
| 13:30Z | ACPX Pi / OpenRouter DeepSeek Flash | Result-less native exhaustion and cross-run recovery | [DOT-131](http://localhost:3100/DOT/issues/DOT-131) | `b696206c-8ab5-47ef-a7da-a47e713de015` | Three bounded attempts, board recovery, no fourth run | Terminal `native_session_retry_exhausted` at attempt 3; issue `in_review`, active recovery board-owned, and no generic stranded continuation raced it | Captured | PASS (EXPECTED PROVIDER FAILURE) |
| 13:34Z | ACPX Pi / OpenRouter DeepSeek Flash | Operator cancellation presentation | [DOT-132](http://localhost:3100/DOT/issues/DOT-132) | two cancelled runs | Cancel active work; no stuck-running UI artifacts | Browser showed compact `Stopped` terminal state; unfinished provider/tool activity normalized to Interrupted rather than Running | Captured | PASS |
| 14:16Z | ACPX Pi / OpenRouter DeepSeek Flash | Structured question followed by response-wake continuation | [DOT-136](http://localhost:3100/DOT/issues/DOT-136) | `268e7c5d-860b-4188-b749-12f514fc996e` → `4391cce2-b87f-46e7-b643-07554ca708bd` | Park immediately, resume on answer, runtime approval, exact file, done | Initial governed wait parked in 11s with no polling. Replacement ACPX session consumed Maple, but its command approval expired after 30s; model then retried denied writes and became idle. Operator cancelled the hung run | Captured | FAIL / FIXED IN FOLLOW-UP |
| 14:20Z | ACPX Pi / OpenRouter DeepSeek Flash | Direct native runtime approval route | [DOT-138](http://localhost:3100/DOT/issues/DOT-138) | `eaf5f1b8-903f-41e8-8bea-0083fc51fd91` | Resolve an in-process ACPX request and resume its provider turn | Resolution accepted in 115ms and emitted `runtime_request.resolved`; browser UI also successfully resolved a later live request. A relative nested path was incorrectly classified as absolute/outside workspace, exposing a Pi extension bug | Captured | ROUTING PASS / PATH POLICY FAIL |
| 14:24Z | ACPX Pi / OpenRouter DeepSeek Flash | Session permission, nested write, exact-byte finish | [DOT-139](http://localhost:3100/DOT/issues/DOT-139) | `f142fe16-be67-4ecf-9766-1b5628138ea4` | One approval grants the session, relative nested write succeeds, exact 9 bytes, one finish | Exactly one request was created/resolved; no second prompt; three tools ran; file is `61 70 70 72 6f 76 65 64 0a`; semantic result accepted and issue done in 15s | Captured | PASS |
| 14:25Z | OpenCode / OpenRouter GLM | Human completion-review rejection and narrow repair | [DOT-130](http://localhost:3100/DOT/issues/DOT-130) | `da584ad0-86f3-4c60-979a-7fbfc279482f` | Reviewer rejects with one verification request; agent verifies only, returns to review, then approval completes | Continuation read the rejection, used a read-only Node check without rewriting work, created a replacement human-only review, and settled done after approval | Captured | PASS |
| 14:28Z | OpenCode / OpenRouter GLM Flash | Native review materialization while the issue page remains open | [DOT-140](http://localhost:3100/DOT/issues/DOT-140) | `b02caacf-c642-4daf-80ed-ece5580b8bfe` → `3ec2d616-700f-4330-9288-fb9f9cb38fb5` | Review card appears live without reload; board approves; continuation finishes | Browser received the native interaction invalidation, rendered the named human-only card immediately, and approval completed the issue | Captured | PASS |
| 14:34Z | OpenCode / OpenRouter GLM | Plan reject → revise → accept → delegate → child return | [DOT-142](http://localhost:3100/DOT/issues/DOT-142) → [DOT-143](http://localhost:3100/DOT/issues/DOT-143) | `359270cb…` → `b1dbdaa2…` → `aab9a6ff…` → `0f41f4b1…` → `13c87242…` | Rejection wakes a revision-only turn; acceptance creates one child; parent resumes once and completes | Exact rejection note appeared causally; revision 2 incorporated all changes without implementation; acceptance created one child; child passed 52 tests; parent returned and completed. The run also exposed annotated evidence parsing and stale-finalizer races, both fixed below | Captured | PASS AFTER FIXES |
| 14:45Z | Kimi child → GLM parent | Parent/dependency wake coalescing and server-restart recovery | [DOT-144](http://localhost:3100/DOT/issues/DOT-144) → [DOT-145](http://localhost:3100/DOT/issues/DOT-145) | `4f15ab70…` → `b858bf14…` / `0b365fe7…` | One rich parent wake, no duplicate provider run, child summary survives restart | Child completed 7 tests. Exactly one parent wake intent used `issue_children_completed`; forced restart interrupted and recovered the parent. Pre-projection control carried a null summary and motivated the semantic-result fallback | Captured | PASS / CONTROL |
| 14:49Z | Kimi child → GLM parent | Semantic child summary at status-commit time | [DOT-146](http://localhost:3100/DOT/issues/DOT-146) → [DOT-147](http://localhost:3100/DOT/issues/DOT-147) | `3863e7fd…` → `19bd4f55…` | Parent wake contains accepted child summary before comment materialization | Wake snapshot contained the full accepted semantic summary and only one parent run. The provider still performed broad discovery because native execution input omitted the wake delta, exposing STRESS-028 | Captured | LINEAGE PASS / PROMPT FAIL |
| 14:53Z | Kimi ask child → GLM parent | Closed native prompt carries child handoff; no refetch | [DOT-148](http://localhost:3100/DOT/issues/DOT-148) → [DOT-149](http://localhost:3100/DOT/issues/DOT-149) | `7e857107…` → `95aedf07…` | Child summary appears in provider prompt; parent invokes only semantic finish and settles once | Both issues done in 24s total. Persisted native input contains `reason: issue_children_completed` and the child summary. Parent made no discovery/history/search/document/shell/file calls; browser shows one 8-second activity and one final reply | Captured | PASS |
| 14:56Z | GLM Flash child → Codex app-server parent | Cross-provider child return | [DOT-150](http://localhost:3100/DOT/issues/DOT-150) → [DOT-151](http://localhost:3100/DOT/issues/DOT-151) | captured | Exact child summary reaches a Codex parent without refetch or duplicate work | Parent emitted immediate reasoning/commentary, invoked only semantic finish, rendered one final reply, and both issues settled done | Captured | PASS |
| 14:58Z | GLM Flash child → ACPX Claude parent | Cross-provider child return through ACPX | [DOT-152](http://localhost:3100/DOT/issues/DOT-152) → [DOT-153](http://localhost:3100/DOT/issues/DOT-153) | captured | Protocol handoff succeeds; cheap child quality remains distinguishable from transport correctness | Parent received the exact wake and did not refetch. Child supplied an incorrect string reversal, correctly classified as model-quality/self-claim risk rather than a protocol failure | Captured | PROTOCOL PASS / MODEL QUALITY FAIL |
| 15:03Z | Codex app-server / `gpt-5.6-luna` | Governed suggested-task selection (pre-fix control) | [DOT-154](http://localhost:3100/DOT/issues/DOT-154) → [DOT-155](http://localhost:3100/DOT/issues/DOT-155) | `623cffe2…` → `cf44af4e…` | Create only the selected child, then resume the native session once | Selection UI created only DOT-155, but native input rejected `suggest_tasks`; a 75ms legacy failure then triggered an unnecessary review-path recovery | Captured | FAIL / FIXED |
| 15:08Z | Codex app-server / `gpt-5.6-luna` | Governed suggested-task selection (post-fix) | [DOT-156](http://localhost:3100/DOT/issues/DOT-156) → [DOT-157](http://localhost:3100/DOT/issues/DOT-157) | `f9d9926f…` → `8332344f…` | One selected child; one native response wake; commentary/tool status; authoritative done | Browser selected only calculator; exactly one unassigned DOT-157 was created. The original Codex session resumed natively, streamed commentary and search, reported the exact selection, and committed done. Exactly two total wakes (assignment + interaction), no recovery | Captured | PASS |
| 15:10Z | Codex app-server / `gpt-5.6-luna` | Native checkbox selection | [DOT-158](http://localhost:3100/DOT/issues/DOT-158) | captured | Tests-only selection resumes natively and completes once | Polished checkbox card accepted exactly `tests`; same-session continuation reported that exact ID and committed done | Captured | PASS |
| 15:14Z | Codex app-server / `gpt-5.6-luna` | Incremental item verdicts (clean post-fix replay) | [DOT-160](http://localhost:3100/DOT/issues/DOT-160) | `a43c1a52…` → `6adac669…` → final | Partial `alpha=approve` parks on the same interaction; final `beta=reject` completes | Partial wake settled as a governed wait in 942ms without replacement interaction or provider retry. Final wake reported both exact verdicts and committed done | Captured | PASS |
| 15:16Z | Codex app-server / `gpt-5.6-luna` | Yielded-wait presentation | [DOT-161](http://localhost:3100/DOT/issues/DOT-161) | `04751fab…` → final | Interaction card and compact wait state, no artificial assistant comment; final response after selection | Pending state had zero issue comments and presentation reason `yielded_control_plane_wait`; browser showed the card plus activity. Selecting blue produced one exact final `blue` reply and done | Captured | PASS |
| 15:27Z | OpenCode / OpenRouter GLM | Ask-mode canonical human input after disabling provider-native questions | [DOT-166](http://localhost:3100/DOT/issues/DOT-166) | `b32ae336…` → `a8f34a8c…` | Expose only Paperclip's durable question path; answer detailed; exact two-sentence final; done | OpenCode invoked `paperclip_request_human_input`, created one required single-select card, parked without a synthetic comment, consumed `detailed` in one continuation, rendered exactly two sentences, and committed done | Captured | PASS |
| 15:29Z | OpenCode / OpenRouter GLM | Governed-wait cancellation presentation | [DOT-167](http://localhost:3100/DOT/issues/DOT-167) | `f7c392f4…` → `ff21022e…` | A normal OpenCode `/abort` used to park the turn must not look like a provider failure | Initial run emitted `turn.cancelled` plus accepted yielded result, no error notice or failed turn. Browser showed only `Ran a tool` and the live question card; `energetic` resumed once and produced one exact sentence, issue done | Captured | PASS |
| 15:33Z | ACPX Claude / `claude-sonnet-5` | Canonical ask interaction plus final response (pre-fix control) | [DOT-168](http://localhost:3100/DOT/issues/DOT-168) | `1eb86101…` → `aa3e51c4…` | Ask, park, resume with bullets, show the actual bullet response | Interaction and continuation were correct, and raw PRP deltas contained all three DNS bullets, but ACPX never completed a final `agentMessage`; Paperclip therefore materialized the semantic status summary instead | Captured | FAIL / FIXED |
| 15:35Z | ACPX Claude / `claude-sonnet-5` | ACPX final-channel boundary (post-fix) | [DOT-169](http://localhost:3100/DOT/issues/DOT-169) | `460ad279…` | Preserve provider prose at semantic completion; exclude trailing acknowledgement; final-agent precedence | Presentation chose `final_agent_message`, materialized only the provider's three bullets, and kept the later acknowledgement inside collapsed activity. Browser shows one primary bullet response and issue done | Captured | PASS |
| 15:39Z | ACPX Codex / `gpt-5.6-sol` | Final-channel boundary plus semantic completion (pre-fix controls) | [DOT-170](http://localhost:3100/DOT/issues/DOT-170) → [DOT-175](http://localhost:3100/DOT/issues/DOT-175) | multiple bounded controls | Preserve final text and allow only runner-owned completion tools | DOT-170 proved the completed final-message boundary, but Codex-ACP routed its MCP approval through an advertised form-elicitation path that the headless runner did not own. Every control failed closed with `user cancelled MCP tool call`; retries were bounded and controls were cancelled before the next variant | Captured metadata | EXPECTED CONTROLS |
| 15:55Z | ACPX Codex / `gpt-5.6-sol` | Governed permission-channel semantic completion | [DOT-176](http://localhost:3100/DOT/issues/DOT-176) | `6942a12b…` | One approved Paperclip completion call; no external MCP auto-approval; accepted result; done | Removing the false generic-form capability forced Codex-ACP through the correlated permission channel. `paperclip_finish` returned accepted, one run succeeded in 17s, issue committed done, and no recovery turn followed. The model omitted the requested literal two-line prose, so the designed semantic-summary fallback became the primary reply | Captured | PROTOCOL PASS / MODEL COMPLIANCE MISS |
| 15:58Z | ACPX Codex / `gpt-5.6-sol` | Explicit pre-tool final response plus schema self-correction | [DOT-177](http://localhost:3100/DOT/issues/DOT-177) | `2f5de1d6…` | Stream exact final first; surface invalid completion calls; self-correct; deduplicate settled reply; done | Exact `LINE ONE\nLINE TWO` streamed before tools. Four malformed completion attempts received specific schema errors; the fifth was accepted without a new provider turn. Browser shows collapsed `Worked for 53s` and one primary exact reply, issue done | Captured | PASS |
| 16:03Z | ACPX Codex / `gpt-5.6-sol` | Raw ACP debugger controls and topology validation | [DOT-178](http://localhost:3100/DOT/issues/DOT-178) → [DOT-180](http://localhost:3100/DOT/issues/DOT-180) | bounded controls | Capture ACP traffic at the native object boundary and classify completeness by provider topology | First clean reload captured 32 frames, proving the adapter hook; the trace store then correctly exposed its old Codex-specific `rust_native` completeness assumption. DOT-180 captured 28 frames after the native callback survived runtime-owner rebinding | Sidecar controls | FIXED |
| 16:09Z | ACPX Codex / `gpt-5.6-sol` | Complete raw ACP trace | [DOT-181](http://localhost:3100/DOT/issues/DOT-181) | `4b87f7a2…` | Complete status, ordered bidirectional ACP frames, inspector usability | 28 frames / 32,765 bytes, 4 client→provider and 24 provider→client, digest verified. Native methods include `initialize`, `session/new`, `session/prompt`, `session/request_permission`, and `session/update`; inspector rendered all frames and stages | Complete | PASS |
| 16:12Z | ACPX Codex / `gpt-5.6-sol` | Raw ACP frame → PRP lineage | [DOT-182](http://localhost:3100/DOT/issues/DOT-182) | `bbbabdd2…` | Correlate provider frames to canonical PRP and visible/folded presentation | 28 complete frames; 10 runtime-normalization stages link 10 provider frames to 10 PRP event IDs. Browser inspector selected frame 8 and showed raw JSON, transport + normalization stages, `item.completed`, and its visible/folded production decision | Complete | PASS |
| 16:15Z | ACPX Claude / `claude-sonnet-5` | Cross-provider ACP trace, immediate commentary, and exact final | [DOT-183](http://localhost:3100/DOT/issues/DOT-183) | `66f0c428…` | Prove the provider-neutral ACP hook on Claude, including commentary order, exact durable final, redaction, and semantic completion | One early commentary comment precedes one exact final. Trace is complete with 49 frames / 52,351 bytes, 7 client→provider and 42 provider→client, 32 correlated runtime events, and three redacted frames. Browser shows collapsed activity plus one primary final | Complete | PASS |
| 16:17Z | ACPX Claude / `claude-sonnet-5` | Semantic tool frame → one-to-many PRP lineage | [DOT-184](http://localhost:3100/DOT/issues/DOT-184) | `eb90adf3…` | Link the originating ACP approval/tool frame to tool start, semantic result, and tool completion | Inspector groups frames 27–28 with exactly three PRP events: `item.started`, `run.result.proposed`, and `item.completed`. Trace is complete with 38 frames. Claude omitted the requested literal final, so the accepted semantic summary was used correctly | Complete | PROTOCOL PASS / MODEL COMPLIANCE MISS |
## Defect ledger
| ID | Evidence | Layer | Fix | Regression | Live retest |
| --- | --- | --- | --- | --- | --- |
| STRESS-001 | DOT-102 / DOT-104 rejected `ask` although PRP actions and UI support ask mode | Runtime admission + native execution contract | Admit `ask` for `paperclip_runner`, round-trip it in v2/v3 model envelopes, retain the old Codex-local boundary, and expose ask-safe semantic tools | Runner contract 2 tests; server runtime mode 1 test; focused suites 50/50 pass; runner/server typechecks pass | DOT-106 admitted; terminal result pending |
| STRESS-002 | DOT-103: `ACPX native backend requires its own instance runtime directory` | runnerd-to-provider backend wiring | Pass the isolated instance ACPX directory at the native backend factory seam | Server backend wiring test; focused server suite 35/35 pass | DOT-108 launched and executed its file command; terminal result pending |
| STRESS-003 | DOT-107 live ACPX Claude turn: provider notice `Authentication required` after static probe passed | Environment qualification / provider auth | Bound the existing encrypted company Anthropic secret to the ACPX Claude agent; no plaintext configuration or logging | Live environment qualification passed | DOT-110 authenticated, streamed, emitted a semantic result, and completed `done` |
| STRESS-004 | DOT-108 implemented and verified the file, then its semantic-finalization retry treated a transient run workspace binding as a missing persisted workspace | Native recovery workspace resolution | Distinguish transient run-id workspace bindings from persisted `execution_workspaces`; reuse only an existing persisted row while retaining fail-closed behavior for archived rows | Workspace-session + runtime mode + executor suites 180/180 pass; server typecheck passes | Fresh projectless retries in DOT-113 resumed the same provider session instead of failing setup |
| STRESS-005 | DOT-109 ACPX Pi runtime approval requests were cancelled by the provider before an immediate administrator resolution reached the live session | Runtime request lifetime / response routing | Investigation pending; preserve cancelled request and failed-tool lineage and ensure no run hang | Live request/cancellation records | DOT-109 still progressing after cancellation |
| STRESS-006 | DOT-114 rendered only “I’ll create the file…” although a later accepted semantic summary existed | OpenCode text-channel normalization / presentation selection | Record text completion source sequence and require a settled final candidate to occur after the last non-terminal work tool; when none exists, emit no final `agentMessage` so the semantic summary wins | OpenCode fixture `commentary-only-before-work`; focused protocol/driver suites pass | DOT-116 shows the exact post-work response and verification caveat as the durable final reply |
| STRESS-007 | DOT-113 exact frame 3322: `paperclip_paperclip_finish`, `completed`, criterion `passed` rejected by the canonical schema | Provider-neutral semantic result normalization | Normalize known completion schema, disposition, and criterion-status aliases before canonical validation; retain strict rejection for unknown values | Exact GPT-OSS-shaped regression in `result-normalization.test.ts`; focused suites pass | DOT-113 recovered and completed `done` with canonical accepted result |
| STRESS-008 | DOT-113 recovery checkpoint contained the same turn in both `activeTurnId` and `terminalTurns`; the resumed runtime waited forever for an already-consumed idle event | Driver checkpoint normalization + native recovery orchestration | All drivers clear a terminal turn from active state before emitting/persisting terminal facts; recovered driver state is authoritative and the runtime no longer resurrects the older persisted active ID | OpenCode driver checkpoint regression plus native runtime continuation regression; 74/74 focused tests and complete runner typecheck pass | DOT-113 resumed and completed on the original provider session |
| STRESS-009 | DOT-113 aliases were accepted by post-dispatch normalization but rejected earlier by the provider-facing MCP bridge | Shared completion contract / MCP admission | Make the provider schema accept only the documented compatibility aliases, then normalize into the strict canonical output schema; add verification aliases `pass/success/succeeded/fail` | Completion-contract, normalization, MCP bridge, and driver focused suites 35/35 pass | DOT-113 accepted live after rebuild |
| STRESS-010 | DOT-117 accumulated watchdog `attemptCount: 9` across different run fingerprints and terminal exhaustion left the failed low-capability model as its own inactive recovery owner | Native session recovery and issue disposition | Supersede watchdogs when run/failure identity changes; on attempt-3 result-less exhaustion preserve named failure authority, move issue to `in_review`, and create a board-owned recovery action with the source agent as return owner | Native recovery projection regression; native session/resumption/recovery suites 78/78 pass; server and runner typechecks pass | DOT-121 is `in_review`, board-owned, attempt 1/3, named `native_session_retry_exhausted`, with no extra provider run |
| STRESS-011 | DOT-123 raw OpenCode frame 128 contained `callID=functions.todowrite:0`, `state.error=Tool execution aborted`, while PRP/UI showed `Tool failed · unknown` with empty output | OpenCode provider-event normalization | Recover a display label only from structured `functions.<name>:<index>` call IDs when the provider reports `unknown`; preserve bounded terminal `state.error` as output; opaque call IDs remain opaque | Provider-event regressions for structured and opaque call IDs; OpenCode/provider suites 25/25 pass | Rebuild and live tool-failure retest pending |
| STRESS-012 | DOT-123 OpenCode replayed accepted finish as `tool=unknown`, `callID=functions.paperclip_paperclip_finish:24`; the driver treated it as later work and selected a short trailing acknowledgement instead of the substantive response | OpenCode final-response boundary | Canonicalize structured call-ID tool identity before deciding whether a part is terminal semantic work; terminal replays no longer move the work boundary | Fake OpenCode now injects the exact unknown terminal replay before a trailing acknowledgement; substantive final selection suite passes | Fresh Kimi/GLM live retest pending |
| STRESS-013 | DOT-125 emitted a substantive final before same-session semantic-result recovery, then a terse acknowledgement after recovery; latest-message precedence displaced the actual answer | Response resolution + native recovery envelope | Mark semantic-result recovery structurally, select the best completed final before that boundary, and send a disposition-only recovery prompt that forbids redoing implementation or adding prose | Heartbeat summary and native-session-runtime regressions; full runner TS 684/684 and Rust 92/92 pass | DOT-126 normal path passes; deterministic live recovery replay still pending |
| STRESS-014 | Historical DOT-119 recovery actions exceeded their configured maximum (`attemptCount` reached 23) and continued scheduling wakes | Central recovery action policy | Enforce the effective max in the shared service; atomically escalate to board ownership, clear wake/monitor policies, cap attempts, preserve return owner, and make replays no-ops while allowing distinct failure identities to supersede | Recovery-action regression plus native executor/process recovery suites 187/187 pass | Historical row now projects exhausted; fresh live exhaustion already bounded at runtime in DOT-121 |
| STRESS-015 | DOT-127 stored a steering comment at submission time and rendered it above the entire settled run, despite a later durable steering acknowledgement | Issue conversation projection | Derive the causal slot from the first non-duplicate `issue.queued_comment_steered` event and split Paperclip settled activity at each acknowledgement so both collapsed and expanded timelines preserve pre-steer → human → post-steer → final order | TaskChatThread causal regression; 24/24 component tests, UI typecheck and production build pass | Browser verified positions: Worked 757, steering 957, post-steer Worked 1161/commentary 1194, final 3498 |
| STRESS-016 | DOT-130 review card did not visibly identify its human owner | Review presentation | Render durable addressee and resolver policy on the review card | UI component suite and build pass | DOT-130 shows `Assigned to You · board only` |
| STRESS-017 | DOT-132 cancelled turns left unfinished tool/provider rows visually Running | Terminal presentation normalization | Project unfinished provider/tool activity as Interrupted whenever the authoritative terminal is cancelled/interrupted | Transcript and task-thread regressions pass | DOT-132 browser shows `Stopped` with no stuck-running terminal activity |
| STRESS-018 | Native semantic-result retries depended on the periodic scheduler and could miss their exact 30-second deadline | Same-run recovery timer | Arm an in-process timer for the persisted retry deadline while retaining durable restart recovery | Native runtime timing regressions pass | Live retries fire at the intended 30-second boundary |
| STRESS-019 | ACPX response-wake checkpoints lost `providerIdentity` in the harness/native backend conversion | Backend checkpoint identity | Round-trip the provider identity through `HarnessDriverBackend` | Focused backend tests and full runner typecheck pass | ACPX continuation reached the provider instead of failing the identity assertion |
| STRESS-020 | Generic stranded-issue recovery could launch another run after native retry exhaustion had already created a board-owned action | Cross-run recovery ownership | Skip generic recovery when an active board-owned recovery action already owns the issue | Process-recovery regression passes | DOT-131 stayed at attempt 3 with no fourth provider run |
| STRESS-021 | Cheap ACPX models continued polling after creating a durable human question | Governed-wait runtime control | Detect the committed wait after each event, cancel the provider turn, and synthesize authoritative yielded terminal facts | Native runtime governed-wait regression passes | DOT-134–136 parked in 9–22s with only legitimate pre-wait tools |
| STRESS-022 | ACPX Pi's persisted child session cannot resume after a governed wait, even with intact identity | ACPX response-wake recovery policy | Permit a fresh provider record only after a yielded `response_wake`, while preserving normalized Paperclip session lineage and strict policy elsewhere | ACPX host/driver and resume tests pass | DOT-136 consumed the answered Maple choice on the replacement provider record |
| STRESS-023 | Browser runtime approval routing existed only for runnerd transports; in-process ACPX/OpenCode sessions had no control-plane authority registration | Native runtime request routing | Add direct active-session resolution with turn binding, idempotency, conflict detection, and runnerd fallback | Backend and server typechecks/tests pass | DOT-138 API and browser UI resolutions both returned accepted and resumed Pi |
| STRESS-024 | Pi treated every slash inside a relative shell path as an absolute path, rejected creation when intermediate directories did not exist, and did not persist `accept_for_session` | ACPX Pi workspace/permission extension | Match absolute paths only at shell-token boundaries, reject parent traversal segments, canonicalize physical roots, walk to the nearest existing ancestor safely, and cache session approval for the extension lifetime | Pi extension regression, runner typecheck, Rust check, and full runner build pass | DOT-139 created a nested file after exactly one approval and completed with exact-byte evidence |
| STRESS-025 | DOT-140's native completion review existed durably but the open issue page did not refresh | Realtime UI invalidation | Invalidate the active issue conversation when native status-decision effects materialize an interaction | Live-updates regression; 35 UI tests, typecheck, and production build pass | DOT-140 rendered the review card without reload and approval completed |
| STRESS-026 | Rejecting DOT-142's `wake_assignee_on_accept` plan interaction did not wake the planner for revision | Interaction continuation policy | Treat rejection of the current issue plan as a revision wake even when acceptance has an accept-only continuation policy | Issue-thread route suite 73/73 pass | DOT-142 revision 2 woke immediately, used the exact rejection note, and did no implementation |
| STRESS-027 | Child completion enqueued both generic dependency and parent wakes; dispatcher coalescing let the poorer reason win, and the comment race produced a null summary | Native status effect materialization | Coalesce the parent/dependency edge at the committer; include child lineage once; source the just-finished child's summary from the accepted native result before presentation comments exist | Native control-plane regression; 10/10 pass | DOT-146 wake reason is `issue_children_completed` and carries the full DOT-147 accepted summary |
| STRESS-028 | DOT-146's run snapshot contained the child summary, but the closed native execution input sent only the task brief to the provider; GLM rediscovered history and files | Native execution-input projection | Render the sanitized provider-neutral Paperclip wake delta into every native task prompt, using resume-delta form for a continued provider session | Native session/input and control-plane suites 17/17 pass; server typecheck passes | DOT-148 persisted the child handoff in its provider prompt and completed in 8s with no discovery tools |
| STRESS-029 | DOT-142 was done from a newer child-return run, then an older accepted-plan finalizer exhausted and reopened it to board review | Native finalization ordering | Detect a newer run owning the issue's latest authoritative decision; retire the older coordinator as `native_finalization_superseded`, resolve its recovery as a false positive, and never mutate issue state | Native finalization recovery integration 2/2 pass | DOT-142 recovery was resolved and the issue restored to the later authoritative `done` decision |
| STRESS-030 | DOT-142 evidence `interaction:<uuid> (request_confirmation, status accepted)` was passed wholesale to a Postgres UUID comparison and crashed finalization | Durable evidence classification | Extract a canonical UUID prefix from annotated model evidence; malformed values remain unverifiable rather than reaching a UUID query | Evidence-classifier plus native finalization/input/control-plane suites; 23/23 pass | Original failure identified exactly from coordinator evidence; fresh annotated-reference integration fixture passes |
| STRESS-031 | DOT-154 accepted `suggest_tasks` through the polished board UI, then native response materialization rejected the interaction kind, fell back to a 75ms legacy failure, and launched `issue_review_path_lost` recovery | Native interaction-response contract + continuation routing | Extend the provider-neutral native response envelope and authorized bridge to all five PRP interaction kinds, including partial item verdicts; keep governed tool approvals excluded; do not attach fallback review-loss instructions to a normal accepted-response wake | Native contract 15/15; bridge 5/5; server typecheck passes | DOT-156 resumed the original Codex session through exactly one native interaction wake, created only DOT-157, streamed normally, and committed done without recovery |
| STRESS-032 | DOT-159 delivered an authoritative partial item verdict, but Codex had to recreate a wait, the duplicate request failed, and semantic-result recovery began | Native governed-wait ownership | Treat an authority-checked pending `request_item_verdicts` response as the continuation run's existing wait target; park immediately after the next committed event or completed turn | Native executor + interaction bridge suites 31/31 pass | DOT-160 partial continuation settled in 942ms on the same interaction, then the completed response resumed and finished done |
| STRESS-033 | DOT-160 materialized identical `Waiting for Proposal verdicts` assistant comments from top-level adapter summaries even though yielded semantic results are control-plane liveness facts | Response resolver / settled presentation | Suppress legacy top-level summaries whenever the accepted semantic result is `yielded`; preserve explicit comments, final agent messages, and marked finals; show the durable interaction/system wait instead | Response-resolver, executor, and bridge suites 58/58 pass; server typecheck passes | DOT-161 pending state has zero issue comments and `commentAction:none`; browser shows the interaction card/activity, then one real final after selection |
| STRESS-034 | OpenCode's provider-native `question` tool created an invisible provider-side wait, while ask-mode also omitted Paperclip's canonical `request_human_input` from the dynamic tool catalog | Provider tool policy + semantic action modes | Disable OpenCode `question` in both runtime config and every prompt; add `ask` to the canonical, live, and scenario task modes for `request_human_input` so every provider receives one durable governance path | OpenCode driver 14/14, catalog 8/8, runner authority 7/7 pass; runner rebuilt | DOT-166 exposed only the canonical Paperclip operation, created one card, consumed one answer, and completed without a provider-native wait |
| STRESS-035 | A healthy OpenCode governed wait displayed `Received a provider notice` because OpenCode represents normal `/abort` control as `session.error` / `MessageAbortedError` | Provider terminal normalization | Normalize that exact OpenCode abort shape to `turn.cancelled` without a false error notice; retain all other `session.error` events as failures | Exact fake-provider abort regression; OpenCode driver 14/14 passes | DOT-167 has `turn.cancelled`, accepted yielded result, no provider notice/failed turn, and browser shows only the question activity before successful continuation |
| STRESS-036 | ACPX streamed the genuine user-facing answer only as `item.delta`; with no completed final `agentMessage`, the response resolver showed the model-authored status summary instead | ACPX final-answer normalization | Track assistant prose after the last non-terminal tool, snapshot it when `paperclip_finish`/`paperclip_block` is accepted, and emit one canonical completed final message on successful settlement; exclude post-result acknowledgements and preserve result-less finals | ACPX final-boundary + runtime-host suite 13/13 passes; runner typecheck/build pass | DOT-169 chose `final_agent_message` and rendered only the upstream bullets; the later `Completed` acknowledgement remained folded in activity |
| STRESS-037 | ACPX Codex cancelled every Paperclip semantic MCP call even though the same runner-owned operations worked through Claude and Pi | ACPX capability advertisement + pinned Codex-ACP approval correlation | Do not advertise generic ACP form elicitations to Codex because Paperclip owns durable user interaction through PRP; patch the pinned Codex-ACP permission request to retain `serverName`; auto-approve only the closed set of runner-created MCP servers, retaining interactive policy for filesystem/process and future external tools | Codex-ACP patch dry-run valid; lockfile patch hash refreshed; ACPX driver/host 14/14, runner typecheck and full build pass | DOT-176 accepted `paperclip_finish`, emitted a native semantic result, committed `done`, and launched no recovery retry |
| STRESS-038 | ACPX agents had tracing enabled but produced zero provider frames; once captured, the store incorrectly demanded Codex's Rust + rehydration channels and marked the TypeScript-native ACP trace incomplete | ACPX transport trace + provider-neutral trace integrity | Forward ACP JSON-RPC objects from the pinned ACPX runtime into the restricted trace sink, preserve direction/method/digest, retain the callback across runtime owner rebinding, require `typescript_acpx_native` for ACPX completeness, and link runtime-normalized PRP event IDs back to inbound frames | ACPX patch dry-run valid; patch hash refreshed; host/driver 14/14; trace-store 5/5; runner build/typecheck and server typecheck pass | DOT-181 is complete with 28 frames; DOT-182's inspector shows two stages and a real PRP/presentation outcome for correlated frames |
| STRESS-039 | In DOT-183 the provider transport and ordinary runtime events were correlated, but the model-authored `paperclip_finish` PRP rows remained detached as “no raw correlation” | ACPX semantic-tool bridge lineage | Capture the current inbound ACP frame when the dynamic tool dispatch begins and attach all emitted PRP IDs, including validation failures and tool results, as one mapped normalization record | ACPX host regression asserts a three-event one-to-many mapping; ACPX host/driver 14/14 and full runner typecheck/build pass | DOT-184 groups the `paperclip_finish` frame pair with `item.started`, `run.result.proposed`, and `item.completed`; no detached semantic-result row remains |
| STRESS-040 | The superseded pre-fix DOT-154 control retained an old agent-owned recovery action and woke again during the final liveness audit | Campaign control cleanup / recovery ownership | Cancel the obsolete control run and retire the pre-fix issue; keep DOT-156 as the authoritative post-fix evidence | Final live-run query returns zero; every remaining nonterminal campaign issue has an active named recovery path or pending board interaction | DOT-154 is cancelled and its recovery action is cleared; no background run remains |
| STRESS-041 | The final server sweep showed that `chosenSource:none` also disabled the one bounded disposition/comment recovery for a genuine legacy adapter with no semantic result | Legacy completion recovery / response presentation boundary | Suppress artificial-comment recovery only when an accepted semantic result exists or a legacy adapter summary was deliberately classified ambiguous; preserve the disposition-only retry for truly result-less legacy runs. Refresh the stale accepted-plan prompt assertion to the current single-child directive | `heartbeat-comment-wake-batching` + `heartbeat-accepted-plan-workspace-refresh`: 17/17 pass; response resolver: 27/27 pass; server typecheck pass | Deterministic gateway fixture now queues exactly one bounded recovery, while native no-response/yielded results remain suppressed |
| STRESS-042 | The broad route-inventory gate found the provider-trace and queued-steering endpoints mounted but absent from the public OpenAPI document | API discoverability / contract inventory | Document provider-trace list, redacted inspect, exact reveal, NDJSON download, deletion, queued-comment operations, and the two existing agent-profile route groups; register their route files in the inventory | OpenAPI route suite 5/5 pass; server typecheck pass | All mounted routes are represented and the debugger API is discoverable in the local OpenAPI document |
| STRESS-043 | Server startup tests predated the Runner PRP WebSocket upgrade listener and supplied an HTTP server mock without `.on()` | Startup integration test contract | Extend the test server double with the standard event-listener method used by the Runner WebSocket boundary | Server startup suite 16/16 pass | Startup wiring, feedback export, scheduler suppression, and public URL cases all remain green with the Runner listener installed |
| STRESS-044 | Project skill discovery canonicalized macOS `/var` paths to `/private/var`, then rejected them against the persisted noncanonical workspace root; case-insensitive filesystems also classified lowercase `skill.md` as canonical | Project skill portability / path safety | Validate containment on canonical pairs while retaining lexical symlink traversal when possible; traverse the verified canonical pair only for equivalent path aliases; require an exact `SKILL.md` directory entry; make path assertions realpath-aware | Full company-skills service suite 56/56 pass; server typecheck pass | Valid project skills import on macOS, out-of-workspace and symlink candidates still fail closed, and lowercase lookalikes are not advertised as skills |
## Coverage checklist
- [x] Immediate commentary and ongoing status (OpenCode Ultra; ACPX Codex streamed pre-tool text)
- [x] Streaming assistant output; empty and non-empty reasoning (OpenCode Ultra; ACPX Codex thinking marker)
- [x] Native tool calls, search, file and command activity (DOT-128, DOT-129, DOT-139)
- [x] Pure Q&A and long final response (DOT-110)
- [x] Exact file creation and exact-byte verification (DOT-139 exact 9-byte file and semantic completion)
- [x] Small code/test task and multi-file implementation (DOT-105)
- [x] Failed command and unavailable verifier result (DOT-129 recovered failed tool; DOT-116 accepted and displayed `not_run/tool_unavailable` caveat)
- [x] Structured user question and exact continuation (DOT-122, browser and exact-byte evidence)
- [x] Governed suggested-task selection and same-session continuation (DOT-156 → DOT-157)
- [x] Checkbox selection and incremental item-verdict continuation (DOT-158, DOT-160)
- [x] Planning result and owned confirmation handoff; rejection/revision, acceptance, delegated implementation, and parent return (DOT-142 → DOT-143)
- [x] Queued follow-up ordering and narrow continuation (DOT-123); true in-turn steering and idempotent replay (DOT-127)
- [x] No-response terminal state without synthetic reply (DOT-132: zero comments, compact `Stopped`; resolver/system-notice regressions)
- [x] Subtask creation, parent/child dependency blocking, delegated implementation, child completion, summary handoff, and parent return (DOT-142–149)
- [x] Runtime permission approval and session-scoped continuation (DOT-138/139); human completion-review rejection, narrow continuation, and approval (DOT-130)
- [x] Forced server restart/process loss, bounded retry, session reuse, and successful recovered completion (DOT-115); cancellation/interruption presentation (DOT-132)
- [x] Duplicate steering delivery and idempotency (DOT-127); general PRP duplicate and terminal-authority fixtures (113-test edge corpus)
- [x] Trace capture on/off and raw-to-presentation lineage (DOT-122 inspector; DOT-124 capture-off)
- [x] Unknown/ignored/rejected/generic/one-to-many/many-to-one mappings (provider/result/generic-capability fixtures; Codex result call 1→2; DOT-128 correlated 4 frames→6 PRP events)
- [x] Terminal status, disposition, response resolution, collapsed activity, steering-boundary split, and no-response system notice (DOT-116, DOT-121, DOT-125–127, DOT-132)
## Automated conformance ledger
| Suite | Scope | Result |
| --- | --- | --- |
| `native-execution.test.ts` | v1/v2/v3 contract, including ask-mode round-trip and plan rejection | 15/15 pass |
| `runtime-mode.test.ts` + `native-session-executor.test.ts` | admission, recovery/steering/session seams, ACPX runnerd directory | 35/35 pass |
| `@paperclipai/paperclip-runner typecheck` | TypeScript, Rust workspace, browser bundle | PASS |
| Full Runner TypeScript suite | All driver, contract, catalog, eval, replay, trace, and presentation tests; generated operation-group and seeded-catalog drift gates included | 691/691 pass after refreshing both canonical generated artifacts |
| Full Runner Rust workspace | Provider transports, durable state/outbox, exact trace bytes, replay, steering, process cleanup, and contract integration | 100/100 pass |
| Protocol inventory and cross-language parity | 41-operation coverage/inventory plus TypeScript↔Rust conformance and 16 replay fixtures | PASS |
| `@paperclipai/server typecheck` | Server integration and vendored runner boundary | PASS |
| OpenCode + Codex driver, native recovery, result normalization focused suites | Text selection, alias compatibility, stale terminal checkpoint recovery | 74/74 pass |
| Provider/trace edge corpus | Exact Rust/TypeScript correlation, unknown provider methods, generic fallback safety, ignored/rejected result records, and one-to-many result lineage | 113/113 pass |
| Response resolver | Explicit/final/semantic precedence, yielded waits, and no-response/no-placeholder behavior | 27/27 pass |
| Runner Inspector + system notices | Correlated grouping/pipeline plus compact no-response terminal presentation | 23/23 pass |
| Final server + UI sweep | Trace storage/finalization/interaction plus inspector/thread/presentation regressions | 22/22 server and 114/114 UI pass |
| Final legacy-recovery isolation | Result-less gateway retry, mentioned-agent deferral, accepted-plan continuation, and response resolution after STRESS-041 | 44/44 pass; server typecheck PASS |
| Final repaired server aggregate | Legacy recovery, accepted-plan continuation, response resolution, OpenAPI parity, startup wiring, and project-skill portability | 121/121 pass across 6 files; server typecheck PASS |
| Broad server stress sweep | Whole-suite concurrency plus isolated reruns outside the Runner scope | Runner-related failures, OpenAPI drift, startup mock drift, and project-skill path portability isolated and fixed. Remaining failures reproduce as repository test-harness drift: macOS workspace-path assertions, synthetic port exhaustion, and unrelated workspace registration fixtures |
## Known gaps and remaining risks
- ACPX Pi is runnable and DOT-139 proved session approval plus an exact nested write;
it remains more likely than stronger models to exhaust the strict completion schema.
- Very small/cheap models can discover and call `paperclip_finish` yet repeatedly
fail its strict schema. Bounded recovery works, but schema ergonomics and error
visibility need investigation before this lane can be considered reliable.
- Historical local issues include deliberately broken and legacy runs. Campaign
conclusions use newly created issues plus isolated fixtures, not old status alone.
- The final ownership audit found no live runs. Remaining nonterminal stress controls
have an active board/agent recovery action or a pending board interaction; the one
superseded control that woke unexpectedly (DOT-154) was cancelled and retired.
- The full server suite is not globally green in this checkout. Isolated failures are
outside the Runner protocol path and are recorded above: macOS realpath-sensitive
workspace assertions, synthetic test-port exhaustion after a long campaign,
and workspace source-registration fixtures. Dedicated Runner, protocol,
response, trace, completion, server-integration, and UI suites are green.
package.json
+4 -1
View File
@@ -65,6 +65,8 @@
"test:e2e:headed": "npx playwright test --config tests/e2e/playwright.config.ts --headed",
"test:e2e:multiuser-authenticated": "npx playwright test --config tests/e2e/playwright-multiuser-authenticated.config.ts",
"evals:smoke": "cd evals/promptfoo && npx promptfoo@0.103.3 eval",
"test:runner-workflow-evals": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals",
"report:runner-workflow-evals": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals",
"test:release-smoke": "npx playwright test --config tests/release-smoke/playwright.config.ts",
"test:release-smoke:headed": "npx playwright test --config tests/release-smoke/playwright.config.ts --headed",
"metrics:paperclip-commits": "tsx scripts/paperclip-commit-metrics.ts",
@@ -87,7 +89,8 @@
"embedded-postgres@18.1.0-beta.16": "patches/embedded-postgres@18.1.0-beta.16.patch",
"acpx@0.13.1": "patches/acpx@0.13.1.patch",
"pi-acp@0.0.33": "patches/pi-acp@0.0.33.patch",
"@agentclientprotocol/claude-agent-acp@0.70.0": "patches/@agentclientprotocol__claude-agent-acp@0.70.0.patch"
"@agentclientprotocol/claude-agent-acp@0.70.0": "patches/@agentclientprotocol__claude-agent-acp@0.70.0.patch",
"@agentclientprotocol/codex-acp@1.6.2": "patches/@agentclientprotocol__codex-acp@1.6.2.patch"
},
"overrides": {
"rollup": ">=4.59.0",
packages/adapter-utils/src/index.ts
+4
View File
@@ -36,6 +36,10 @@ export type {
QuotaWindow,
ProviderQuotaResult,
TranscriptEntry,
PaperclipQuestion,
PaperclipQuestionOption,
PaperclipQuestionResponse,
PaperclipQuestionSet,
StdoutLineParser,
CLIAdapterModule,
CreateConfigValues,
packages/adapter-utils/src/types.ts
+39 -1
View File
@@ -564,6 +564,44 @@ export interface TranscriptRunArtifact {
title?: string;
}
export interface PaperclipQuestionOption {
id: string;
label: string;
description?: string;
}
export interface PaperclipQuestion {
id: string;
header?: string;
prompt: string;
helpText?: string;
required: boolean;
answerMode: "single_select" | "multi_select" | "text";
options?: PaperclipQuestionOption[];
customAnswer?: { enabled: true; label?: string; placeholder?: string };
textValidation?: {
minLength?: number;
maxLength?: number;
pattern?: string;
inputType?: "text" | "number" | "integer";
minimum?: number;
maximum?: number;
};
}
export interface PaperclipQuestionSet {
schema: "paperclip.question_set.v1";
title?: string;
description?: string;
submitLabel?: string;
questions: PaperclipQuestion[];
}
export interface PaperclipQuestionResponse {
schema: "paperclip.question_response.v1";
answers: Record<string, { selectedOptionIds?: string[]; text?: string; customText?: string }>;
}
export type TranscriptEntry =
| { kind: "assistant"; ts: string; text: string; delta?: boolean; channel?: "progress" | "final" | "unknown" }
| { kind: "thinking"; ts: string; text: string; delta?: boolean; lifecycle?: "started" | "completed"; channel?: "summary" | "detail" | "unknown" }
@@ -579,7 +617,7 @@ export type TranscriptEntry =
| { kind: "provider_activity"; ts: string; family: ProviderActivityFamily; eventType: string; status: ProviderActivityStatus; title: string; summary: string; payload: Record<string, unknown> }
| { kind: "workspace_change"; ts: string; changeSetId: string; revision: number; source: "harness_reported" | "runner_verified"; complete: boolean; files: TranscriptWorkspaceChangeFile[]; totals: { files: number; additions: number | null; deletions: number | null }; patchArtifactRef: string | null }
| { kind: "workspace_file_reference"; ts: string; referenceId: string; source: "harness_reported" | "runner_verified"; path: string; displayName: string; mediaType: string | null; presentation: "document" | "code" | "image" | "generic"; line: number | null; preview: string | null; previewTruncated: boolean; contentDigest: string | null }
| { kind: "runtime_request"; ts: string; requestId: string; requestKind: "command_approval" | "file_approval" | "permission_approval" | "user_input" | "elicitation" | null; turnId: string | null; requestType: "permission" | "input"; status: "pending" | "resolved" | "expired" | "cancelled"; prompt: string; choices: Array<{ key: string; label: string }>; fields: Array<{ name: string; label: string; placeholder: string | null }> }
| { kind: "runtime_request"; ts: string; requestId: string; requestKind: "runtime" | "command_approval" | "file_approval" | "permission_approval" | "user_input" | "elicitation" | null; turnId: string | null; requestType: "permission" | "input"; status: "pending" | "resolved" | "expired" | "cancelled"; prompt: string; choices: Array<{ key: string; label: string }>; fields: Array<{ name: string; label: string; placeholder: string | null }>; questionSet?: PaperclipQuestionSet | null }
| { kind: "run_result"; ts: string; disposition: "done" | "blocked" | "needs_review" | "yielded"; summary: string; objectiveSatisfied: boolean | null; verification: TranscriptRunVerification[]; remainingWork: Array<{ description: string; blocksCompletion: boolean }>; blocker: { reasonCode: string; unblockAction: string; scope: "current_track" | "task_wide" } | null; artifacts: TranscriptRunArtifact[] }
| { kind: "run_terminal"; ts: string; turnState: "completed" | "failed" | "interrupted" | "cancelled"; runState: "succeeded" | "failed" | "cancelled"; disposition: "done" | "blocked" | "needs_review" | "yielded"; stopReason?: string };
packages/db/src/migrations/0231_pretty_meteorite.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE "issue_thread_interactions" ADD COLUMN "addressee_user_id" text;--> statement-breakpoint
CREATE INDEX "issue_thread_interactions_addressee_user_idx" ON "issue_thread_interactions" USING btree ("addressee_user_id");
packages/db/src/migrations/meta/0231_snapshot.json
Generated
+41656
View File
File diff suppressed because it is too large. Load diff
packages/db/src/migrations/meta/_journal.json
Generated
+8 -1
View File
@@ -1604,6 +1604,13 @@
"when": 1787416987133,
"tag": "0230_pink_longshot",
"breakpoints": true
},
{
"idx": 231,
"version": "7",
"when": 1787433588265,
"tag": "0231_pretty_meteorite",
"breakpoints": true
}
]
}
}
packages/db/src/schema/issue_thread_interactions.ts
+2
View File
@@ -45,6 +45,7 @@ export const issueThreadInteractions = pgTable(
summary: text("summary"),
createdByAgentId: uuid("created_by_agent_id").references(() => agents.id),
addresseeAgentId: uuid("addressee_agent_id").references(() => agents.id, { onDelete: "set null" }),
addresseeUserId: text("addressee_user_id"),
createdByUserId: text("created_by_user_id"),
resolvedByAgentId: uuid("resolved_by_agent_id").references(() => agents.id),
resolvedByRunId: uuid("resolved_by_run_id").references(() => heartbeatRuns.id, { onDelete: "set null" }),
@@ -72,5 +73,6 @@ export const issueThreadInteractions = pgTable(
.where(sql`${table.idempotencyKey} IS NOT NULL`),
sourceCommentIdx: index("issue_thread_interactions_source_comment_idx").on(table.sourceCommentId),
addresseeAgentIdx: index("issue_thread_interactions_addressee_agent_idx").on(table.addresseeAgentId),
addresseeUserIdx: index("issue_thread_interactions_addressee_user_idx").on(table.addresseeUserId),
}),
);
packages/paperclip-runner/docs/runner-workflow-evals.md
+24
View File
@@ -0,0 +1,24 @@
# Stress-derived Runner workflow evals
The Runner workflow eval system turns the `STRESS-001`–`STRESS-044` campaign into three complementary lanes. It is additive to the 41-operation inventory, the 106 capability cases, and the v1 scoring/report readers.
## Lanes
- `pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals` runs the credential-free deterministic PR gate. It exercises all twelve workflow families through sanitized Codex, OpenCode, and ACPX normalization fixtures and validates the versioned contracts, lifecycle gates, schedule, traceability, reports, and alert policy.
- `pnpm --filter @paperclipai/paperclip-runner report:runner-workflow-evals` writes JSON, Markdown, JUnit, and GitHub-safe deterministic reports under `.paperclip-local/evals/workflows/`.
- `pnpm --filter @paperclipai/paperclip-runner report:runner-live-evals` runs the balanced forty-execution nightly matrix against real provider sessions. Live candidate failures are trend-only. Missing credentials, qualification failures, and provider outages are unscored; one retry is allowed only for retryable infrastructure failures.
- `pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals` writes the eight-scenario fault schedule. The weekly and pre-release workflow executes the corresponding Runner and server restart, replay, trace, finalization, interaction, and wake-race regression suites.
The checked-in candidate manifest stores only adapter, model, reasoning configuration, qualification variable names, and budgets. Credentials remain in the environment. `PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD` defaults to 12 USD for scheduled live runs.
## Trace and reasoning safety
Every live execution opts into provider tracing in a run-local mode-`0600` sidecar. The evaluator verifies raw frame byte lengths, SHA-256 digests, order, dispositions, and lineage, retains only redacted observations plus a trace digest, and destroys the exact temporary trace after the execution. Exact prompts, credentials, tool arguments, and reasoning text never enter eval reports or GitHub artifacts. Production Runner traces continue to use the restricted 24-hour trace store and run inspector.
The evals score time to first visible progress and visible activity families. They do not inspect or grade hidden chain-of-thought content or require exact natural-language phrasing.
## Compatibility and trends
The live bundle ID binds the Runner version, prompt policy, schedule seed, adapters, resolved models, and reasoning settings. Comparisons and rolling seven-day alerts use only matching bundle IDs. Baseline alerts remain off until seven compatible reports exist. Safe reports are retained for 30 days; exact raw provider traces are never uploaded.
The traceability manifest is `spec/evals/stress-workflow-traceability.json`. CI fails if a stress finding is missing, a workflow ID is unknown, or a referenced regression test no longer exists.
packages/paperclip-runner/package.json
+8 -2
View File
@@ -54,7 +54,7 @@
],
"scripts": {
"build": "pnpm run build:typescript && pnpm run build:rust && pnpm run build:browser && pnpm run build:sdk && pnpm run build:standalone && pnpm run build:scenarios && pnpm run build:issue-thread",
"build:typescript": "pnpm run generate:protocol-types && tsc -p tsconfig.json",
"build:typescript": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm run generate:protocol-types && tsc -p tsconfig.json",
"build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace",
"build:browser": "vite build --config vite.config.ts",
"build:sdk": "vite build --config vite.sdk.config.ts",
@@ -66,7 +66,7 @@
"generate:semantic-contracts": "pnpm run build:typescript && node scripts/generate-semantic-contracts.mjs",
"check:semantic-contracts": "pnpm run build:typescript && node scripts/generate-semantic-contracts.mjs --check",
"typecheck": "pnpm run typecheck:typescript && pnpm run typecheck:rust && pnpm run typecheck:browser",
"typecheck:typescript": "pnpm run check:protocol-types && tsc -p tsconfig.json --noEmit",
"typecheck:typescript": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm run check:protocol-types && tsc -p tsconfig.json --noEmit",
"typecheck:rust": "cargo check --manifest-path runner/Cargo.toml --locked --workspace",
"typecheck:browser": "tsc -p tsconfig.browser.json --noEmit",
"test": "pnpm run build:runner-binaries && pnpm run test:typescript && pnpm run test:rust && node --test scripts/aws-agentcore-provisioning.test.mjs scripts/check-forbidden-imports.test.mjs scripts/check-numbered-milestones.test.mjs scripts/check-tracked-imports.test.mjs scripts/record-codex-evidence.test.mjs scripts/validate-doc-links.test.mjs test/capability-contract.test.mjs",
@@ -97,8 +97,13 @@
"test:capability-inventory": "node scripts/check-capability-inventory.test.mjs",
"test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts",
"test:eval-slice": "vitest run src/eval",
"test:runner-workflow-evals": "vitest run src/eval/workflow-evals.test.ts",
"check:runner-workflow-traceability": "pnpm run build:typescript && node scripts/check-runner-workflow-traceability.mjs",
"report:capability-evals": "pnpm run build:typescript && node scripts/run-capability-eval-suite.mjs",
"report:capability-live-evals": "pnpm run build:typescript && cargo build --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/run-capability-live-eval-matrix.mjs",
"report:runner-workflow-evals": "pnpm run build:typescript && node scripts/run-runner-workflow-evals.mjs",
"report:runner-live-evals": "pnpm run build:typescript && pnpm run build:runner-binaries && node scripts/run-runner-live-eval-schedule.mjs --mode nightly --execute",
"report:runner-chaos-evals": "pnpm run build:typescript && node scripts/run-runner-live-eval-schedule.mjs --mode chaos",
"check:conformance-parity": "vitest run src/tracer/conformance-runner.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core runs_the_mock_core_path_with_stable_output && node -e \"process.stdout.write('Rust and TypeScript Conformance tracer output matches the shared golden fixture.\\n')\"",
"check:replay-parity": "vitest run src/reducer/session-reducer.test.ts && cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core replay_fixture_parity && node -e \"process.stdout.write('Rust and TypeScript Replay reducer summaries match the shared golden fixtures.\\n')\"",
"docs:validate": "node scripts/validate-doc-links.mjs && node scripts/validate-okf.mjs",
@@ -142,6 +147,7 @@
"verify:rootless": "bash scripts/verify-rootless-linux.sh"
},
"dependencies": {
"@paperclipai/paperclip-eval-kernel": "workspace:*",
"@agentclientprotocol/claude-agent-acp": "0.70.0",
"@agentclientprotocol/codex-acp": "1.6.2",
"@earendil-works/pi-coding-agent": "0.84.2",
packages/paperclip-runner/protocol/fixtures/evals/native-execution-seeded.json
+1 -1
View File
@@ -24,7 +24,7 @@
"prpVersion": 1,
"nativeExecutionVersion": 1,
"catalogVersion": 1,
"catalogSha256": "sha256:b6c2723c0b2a041f7b23c00a6043bd81d69ef3d299f67b6dbd689afb7c36dc01",
"catalogSha256": "sha256:cb5ece735f3c9668a549977bb9939aefd9b16028dfce9ea7248c019d1f25892d",
"driverContractVersion": 1,
"driverKind": "paperclip-deterministic",
"driverVersion": "1.0.0"
packages/paperclip-runner/protocol/fixtures/questions/acpx.json
+48
View File
@@ -0,0 +1,48 @@
{
"schema": "paperclip.question_adapter_fixture.v1",
"adapter": "acpx",
"nativeRequest": {
"method": "elicitation/create",
"params": {
"mode": "form",
"message": "Deployment input",
"requestedSchema": {
"type": "object",
"required": ["environment"],
"properties": {
"environment": {
"type": "string",
"title": "Environment",
"description": "Where should we deploy?",
"oneOf": [
{ "const": "staging", "title": "Staging", "description": "Deploy to staging first." },
{ "const": "production", "title": "Production", "description": "Deploy directly to production." }
]
}
}
}
}
},
"canonicalQuestionSet": {
"schema": "paperclip.question_set.v1",
"title": "Deployment input",
"submitLabel": "Submit answers",
"questions": [{
"id": "environment",
"header": "Environment",
"prompt": "Where should we deploy?",
"required": true,
"answerMode": "single_select",
"options": [
{ "id": "option-1", "label": "Staging", "description": "Deploy to staging first." },
{ "id": "option-2", "label": "Production", "description": "Deploy directly to production." }
],
"customAnswer": { "enabled": true, "label": "Other" }
}]
},
"canonicalResponse": {
"schema": "paperclip.question_response.v1",
"answers": { "environment": { "selectedOptionIds": ["option-1"] } }
},
"nativeResponse": { "action": "accept", "content": { "environment": "staging" } }
}
packages/paperclip-runner/protocol/fixtures/questions/codex.json
+41
View File
@@ -0,0 +1,41 @@
{
"schema": "paperclip.question_adapter_fixture.v1",
"adapter": "codex",
"nativeRequest": {
"method": "item/tool/requestUserInput",
"params": {
"questions": [{
"id": "environment",
"header": "Environment",
"question": "Where should we deploy?",
"options": [
{ "label": "Staging", "description": "Deploy to staging first." },
{ "label": "Production", "description": "Deploy directly to production." }
],
"isOther": true
}]
}
},
"canonicalQuestionSet": {
"schema": "paperclip.question_set.v1",
"title": "Deployment input",
"submitLabel": "Submit answers",
"questions": [{
"id": "environment",
"header": "Environment",
"prompt": "Where should we deploy?",
"required": true,
"answerMode": "single_select",
"options": [
{ "id": "option-1", "label": "Staging", "description": "Deploy to staging first." },
{ "id": "option-2", "label": "Production", "description": "Deploy directly to production." }
],
"customAnswer": { "enabled": true, "label": "Other" }
}]
},
"canonicalResponse": {
"schema": "paperclip.question_response.v1",
"answers": { "environment": { "selectedOptionIds": ["option-1"] } }
},
"nativeResponse": { "answers": { "environment": { "answers": ["Staging"] } } }
}
packages/paperclip-runner/protocol/fixtures/questions/opencode.json
+43
View File
@@ -0,0 +1,43 @@
{
"schema": "paperclip.question_adapter_fixture.v1",
"adapter": "opencode",
"nativeRequest": {
"type": "question.asked",
"properties": {
"id": "request-1",
"sessionID": "session-1",
"questions": [{
"question": "Where should we deploy?",
"header": "Environment",
"options": [
{ "label": "Staging", "description": "Deploy to staging first." },
{ "label": "Production", "description": "Deploy directly to production." }
],
"multiple": false,
"custom": true
}]
}
},
"canonicalQuestionSet": {
"schema": "paperclip.question_set.v1",
"title": "Deployment input",
"submitLabel": "Submit answers",
"questions": [{
"id": "environment",
"header": "Environment",
"prompt": "Where should we deploy?",
"required": true,
"answerMode": "single_select",
"options": [
{ "id": "option-1", "label": "Staging", "description": "Deploy to staging first." },
{ "id": "option-2", "label": "Production", "description": "Deploy directly to production." }
],
"customAnswer": { "enabled": true, "label": "Other" }
}]
},
"canonicalResponse": {
"schema": "paperclip.question_response.v1",
"answers": { "environment": { "selectedOptionIds": ["option-1"] } }
},
"nativeResponse": { "answers": [["Staging"]] }
}
packages/paperclip-runner/protocol/schemas/question-adapter-fixture.schema.json
+16
View File
@@ -0,0 +1,16 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/question-adapter-fixture.schema.json",
"title": "Harness question adapter conformance fixture",
"type": "object",
"required": ["schema", "adapter", "nativeRequest", "canonicalQuestionSet", "canonicalResponse", "nativeResponse"],
"properties": {
"schema": { "const": "paperclip.question_adapter_fixture.v1" },
"adapter": { "type": "string", "minLength": 1, "maxLength": 160 },
"nativeRequest": {},
"canonicalQuestionSet": { "$ref": "https://paperclip.dev/schemas/prp/v1/question-set.schema.json" },
"canonicalResponse": { "$ref": "https://paperclip.dev/schemas/prp/v1/question-response.schema.json" },
"nativeResponse": {}
},
"additionalProperties": false
}
packages/paperclip-runner/protocol/schemas/question-response.schema.json
+28
View File
@@ -0,0 +1,28 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/question-response.schema.json",
"title": "Paperclip question response",
"type": "object",
"required": ["schema", "answers"],
"properties": {
"schema": { "const": "paperclip.question_response.v1" },
"answers": {
"type": "object",
"propertyNames": { "minLength": 1, "maxLength": 160 },
"additionalProperties": {
"type": "object",
"properties": {
"selectedOptionIds": {
"type": "array",
"uniqueItems": true,
"items": { "type": "string", "minLength": 1, "maxLength": 160 }
},
"text": { "type": "string", "maxLength": 100000 },
"customText": { "type": "string", "maxLength": 100000 }
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}
packages/paperclip-runner/protocol/schemas/question-set.schema.json
+74
View File
@@ -0,0 +1,74 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/prp/v1/question-set.schema.json",
"title": "Paperclip question set",
"type": "object",
"required": ["schema", "questions"],
"properties": {
"schema": { "const": "paperclip.question_set.v1" },
"title": { "type": "string", "maxLength": 1000 },
"description": { "type": "string", "maxLength": 4000 },
"submitLabel": { "type": "string", "maxLength": 200 },
"questions": {
"type": "array",
"minItems": 1,
"maxItems": 64,
"items": { "$ref": "#/$defs/question" }
}
},
"$defs": {
"option": {
"type": "object",
"required": ["id", "label"],
"properties": {
"id": { "type": "string", "minLength": 1, "maxLength": 160 },
"label": { "type": "string", "minLength": 1, "maxLength": 1000 },
"description": { "type": "string", "maxLength": 4000 }
},
"additionalProperties": false
},
"customAnswer": {
"type": "object",
"required": ["enabled"],
"properties": {
"enabled": { "const": true },
"label": { "type": "string", "maxLength": 1000 },
"placeholder": { "type": "string", "maxLength": 1000 }
},
"additionalProperties": false
},
"textValidation": {
"type": "object",
"properties": {
"minLength": { "type": "integer", "minimum": 0, "maximum": 100000 },
"maxLength": { "type": "integer", "minimum": 0, "maximum": 100000 },
"pattern": { "type": "string", "maxLength": 1000 },
"inputType": { "enum": ["text", "number", "integer"] },
"minimum": { "type": "number" },
"maximum": { "type": "number" }
},
"additionalProperties": false
},
"question": {
"type": "object",
"required": ["id", "prompt", "required", "answerMode"],
"properties": {
"id": { "type": "string", "minLength": 1, "maxLength": 160 },
"header": { "type": "string", "maxLength": 1000 },
"prompt": { "type": "string", "minLength": 1, "maxLength": 4000 },
"helpText": { "type": "string", "maxLength": 4000 },
"required": { "type": "boolean" },
"answerMode": { "enum": ["single_select", "multi_select", "text"] },
"options": {
"type": "array",
"maxItems": 128,
"items": { "$ref": "#/$defs/option" }
},
"customAnswer": { "$ref": "#/$defs/customAnswer" },
"textValidation": { "$ref": "#/$defs/textValidation" }
},
"additionalProperties": false
}
},
"additionalProperties": false
}
packages/paperclip-runner/protocol/schemas/request.schema.json
+26
View File
@@ -3,6 +3,32 @@
"$id": "https://paperclip.dev/schemas/prp/v1/request.schema.json",
"title": "PRP request",
"oneOf": [
{
"type": "object",
"required": ["schema", "requestKind", "requestId", "type", "status", "prompt", "input"],
"properties": {
"schema": { "const": "paperclip.runtime_request.v2" },
"requestKind": { "const": "runtime" },
"requestId": { "type": "string", "minLength": 1, "maxLength": 160 },
"type": { "const": "input" },
"status": { "enum": ["pending", "resolved", "expired", "cancelled"] },
"prompt": { "type": "string", "minLength": 1, "maxLength": 4000 },
"input": { "$ref": "https://paperclip.dev/schemas/prp/v1/question-set.schema.json" },
"origin": {
"type": "object",
"required": ["adapter"],
"properties": {
"adapter": { "type": "string", "minLength": 1, "maxLength": 160 },
"provider": { "type": "string", "minLength": 1, "maxLength": 160 },
"method": { "type": "string", "minLength": 1, "maxLength": 500 }
},
"additionalProperties": false
},
"turnId": { "type": "string", "minLength": 1, "maxLength": 160 },
"itemId": { "type": "string", "minLength": 1, "maxLength": 160 }
},
"additionalProperties": false
},
{
"type": "object",
"required": ["schema", "requestKind", "requestId", "type", "status", "prompt"],
packages/paperclip-runner/protocol/schemas/result.schema.json
+32 -2
View File
@@ -57,12 +57,30 @@
"evidence": { "type": "array", "items": { "type": "object" } },
"verification": {
"type": "array",
"description": "Checks used to verify the work. Use failed only when the check actually ran and found a defect in the work. If a check could not run or complete because the environment, tool, dependency, permission, credential, policy, external service, or budget was unavailable, use not_run with reasonCode even when the attempted command exited non-zero.",
"items": {
"type": "object",
"required": ["commandOrCheck", "status"],
"properties": {
"commandOrCheck": { "type": "string", "minLength": 1 },
"status": { "enum": ["passed", "failed", "not_run"] },
"status": {
"enum": ["passed", "failed", "not_run"],
"description": "passed means the check ran and succeeded; failed means it ran to a meaningful verdict and found the work incorrect; not_run means no meaningful verdict was possible because the check was not attempted or could not complete."
},
"reasonCode": {
"description": "Required for not_run; identify why the check had no meaningful verdict.",
"enum": [
"environment_unavailable",
"tool_unavailable",
"dependency_missing",
"permission_denied",
"credential_missing",
"policy_restricted",
"external_service_unavailable",
"budget_exhausted",
"other"
]
},
"detail": { "type": "string" },
"artifactRef": { "type": "string" }
},
@@ -80,7 +98,19 @@
},
"additionalProperties": true
},
"attentionRequests": { "type": "array", "items": { "type": "object" } },
"attentionRequests": {
"type": "array",
"items": {
"type": "object",
"properties": {
"kind": { "type": "string", "minLength": 1 },
"summary": { "type": "string", "minLength": 1 },
"ownerClass": { "enum": ["human", "agent", "external_system"] },
"targetAgentId": { "type": "string", "minLength": 1 }
},
"additionalProperties": true
}
},
"artifacts": {
"type": "array",
"items": {
packages/paperclip-runner/runner/Cargo.lock
Generated
+121 -3
View File
@@ -110,7 +110,7 @@ dependencies = [
"fastrand",
"hex",
"http 1.5.0",
"sha1",
"sha1 0.10.7",
"time",
"tokio",
"tracing",
@@ -171,7 +171,9 @@ dependencies = [
"bytes",
"bytes-utils",
"fastrand",
"http 0.2.12",
"http 1.5.0",
"http-body 0.4.6",
"http-body 1.1.0",
"percent-encoding",
"pin-project-lite",
@@ -206,6 +208,43 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws-sdk-s3"
version = "1.143.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0ade5433c9561daac7c0c6bc910f1240b4f8ec0d6148b0b463aac0691d747c9"
dependencies = [
"arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-sigv4",
"aws-smithy-async",
"aws-smithy-checksums",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes",
"fastrand",
"hex",
"hmac 0.13.0",
"http 0.2.12",
"http 1.5.0",
"http-body 1.1.0",
"lru",
"percent-encoding",
"regex-lite",
"sha2 0.11.0",
"tracing",
"url",
]
[[package]]
name = "aws-sdk-sso"
version = "1.107.0"
@@ -323,6 +362,27 @@ dependencies = [
"tokio",
]
[[package]]
name = "aws-smithy-checksums"
version = "0.65.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
"bytes",
"crc-fast",
"hex",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"md-5",
"pin-project-lite",
"sha1 0.11.0",
"sha2 0.11.0",
"tracing",
]
[[package]]
name = "aws-smithy-eventstream"
version = "0.61.2"
@@ -739,6 +799,16 @@ dependencies = [
"libc",
]
[[package]]
name = "crc-fast"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
dependencies = [
"digest 0.10.7",
"spin",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -1149,6 +1219,17 @@ dependencies = [
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
@@ -1432,7 +1513,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown",
"hashbrown 0.17.1",
]
[[package]]
@@ -1556,12 +1637,31 @@ version = "0.4.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]]
name = "lru"
version = "0.16.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39"
dependencies = [
"hashbrown 0.16.1",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "md-5"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98"
dependencies = [
"cfg-if",
"digest 0.11.3",
]
[[package]]
name = "memchr"
version = "2.8.3"
@@ -1712,6 +1812,7 @@ dependencies = [
"aes-gcm",
"aws-config",
"aws-sdk-bedrockagentcore",
"aws-sdk-s3",
"aws-sdk-sts",
"aws-smithy-types",
"aws-types",
@@ -1987,7 +2088,7 @@ dependencies = [
"ahash",
"fluent-uri",
"getrandom 0.3.4",
"hashbrown",
"hashbrown 0.17.1",
"itoa",
"micromap",
"parking_lot",
@@ -2294,6 +2395,17 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "sha1"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"digest 0.11.3",
]
[[package]]
name = "sha2"
version = "0.10.9"
@@ -2364,6 +2476,12 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "spin"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3"
[[package]]
name = "spki"
version = "0.7.3"
packages/paperclip-runner/runner/Cargo.toml
+1
View File
@@ -20,6 +20,7 @@ jsonschema = { version = "0.50", default-features = false }
aws-config = { version = "1.11", default-features = false, features = ["behavior-version-latest", "credentials-process", "default-https-client", "rt-tokio", "sso"] }
aws-sdk-bedrockagentcore = { version = "1.63", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-sdk-sts = { version = "1.112", default-features = false, features = ["default-https-client", "rt-tokio", "sigv4a"] }
aws-sdk-s3 = { version = "1.112", default-features = false, features = ["default-https-client", "rt-tokio"] }
aws-smithy-types = "1.6"
aws-types = "1.3"
tokio = { version = "1.50", features = ["rt-multi-thread", "sync", "time"] }
packages/paperclip-runner/runner/crates/runner-core/Cargo.toml
+1
View File
@@ -17,6 +17,7 @@ jsonschema.workspace = true
aws-config.workspace = true
aws-sdk-bedrockagentcore.workspace = true
aws-sdk-sts.workspace = true
aws-sdk-s3.workspace = true
aws-smithy-types.workspace = true
aws-types.workspace = true
tokio.workspace = true
packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider.rs
+19 -8
View File
@@ -86,6 +86,7 @@ impl AcpxProvider {
"workingDirectory": config.cwd,
"agent": config.agent,
"model": config.model,
"systemInstructions": config.instructions,
"tools": provider.tools,
"expectedIdentity": expected_identity,
}),
@@ -262,15 +263,24 @@ impl AcpxProvider {
}))
}
GeneratedAcpxSidecarEventType::RuntimePermissionRequested => {
let request_id = required_text(&payload, "requestId", "permission request")?;
let request_kind = permission_kind(payload.get("kind").and_then(Value::as_str));
Ok(Some(ProviderEvent::RuntimeRequest {
request_id: required_text(&payload, "requestId", "permission request")?,
request_kind: permission_kind(payload.get("kind").and_then(Value::as_str)),
title: payload
.get("title")
.and_then(Value::as_str)
.unwrap_or("ACP permission request")
.to_owned(),
details: payload,
request: json!({
"schema": "paperclip.runtime_request.v1",
"requestKind": "runtime",
"requestId": request_id,
"type": "permission",
"status": "pending",
"prompt": payload.get("title").and_then(Value::as_str).unwrap_or("ACP permission request"),
"origin": {
"adapter": "acpx-runtime-sidecar",
"provider": "acpx",
"method": "runtime.permission_requested",
"kind": request_kind,
},
"details": payload,
}),
}))
}
GeneratedAcpxSidecarEventType::RuntimeTurnTerminal => {
@@ -734,6 +744,7 @@ mod tests {
cwd: "/tmp/workspace".to_owned(),
instructions: "safe".to_owned(),
permission_policy: "interactive".to_owned(),
runtime_context: None,
};
assert!(validate_config(&config).is_err());
}
packages/paperclip-runner/runner/crates/runner-core/src/aws_agentcore_provider.rs
+369 -6
View File
@@ -5,6 +5,8 @@
//! bridge. No Paperclip credential or callback address enters AgentCore.
use std::collections::{BTreeMap, HashMap, VecDeque};
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::mpsc::{self, Receiver, SyncSender};
use std::thread::{self, JoinHandle};
use std::time::Duration;
@@ -13,11 +15,13 @@ use aws_config::sts::AssumeRoleProvider;
use aws_sdk_bedrockagentcore::error::SdkError;
use aws_sdk_bedrockagentcore::types::{
HarnessContentBlock, HarnessContentBlockDelta, HarnessContentBlockStart,
HarnessConversationRole, HarnessInlineFunctionConfig, HarnessMessage, HarnessTool,
HarnessConversationRole, HarnessInlineFunctionConfig, HarnessMessage, HarnessRemoteMcpConfig,
HarnessSkill, HarnessSkillS3Source, HarnessSystemContentBlock, HarnessTool,
HarnessToolConfiguration, HarnessToolResultBlock, HarnessToolResultContentBlock,
HarnessToolType, HarnessToolUseBlock, HarnessToolUseStatus, HarnessToolUseType,
InvokeHarnessStreamOutput,
};
use aws_smithy_types::byte_stream::ByteStream;
use aws_smithy_types::error::metadata::ProvideErrorMetadata;
use aws_smithy_types::{Document, Number};
use aws_types::region::Region;
@@ -38,6 +42,7 @@ const MAX_AGENTCORE_TOOLS: usize = 64;
// bounded cold start, but leave enough of the eval's outer deadline for the
// durable runner to classify the failure and service runner.shutdown cleanly.
const AGENTCORE_INVOCATION_DELIVERY_TIMEOUT: Duration = Duration::from_secs(120);
const MAX_CONTEXT_UPLOAD_BYTES: usize = 64 * 1024 * 1024;
#[cfg(test)]
const AGENTCORE_INLINE_TOOL_ALLOWLIST: &str = "@*/pc_*";
#[derive(Clone, Debug)]
@@ -192,6 +197,247 @@ impl Drop for NetworkWorker {
}
}
fn take_agentcore_mcp_tool() -> Result<Option<(HarnessTool, String)>, LocalRunnerError> {
let name = std::env::var("PAPERCLIP_NATIVE_MCP_NAME").ok();
let url = std::env::var("PAPERCLIP_NATIVE_MCP_URL").ok();
let token = std::env::var("PAPERCLIP_NATIVE_MCP_TOKEN").ok();
for key in [
"PAPERCLIP_NATIVE_MCP_NAME",
"PAPERCLIP_NATIVE_MCP_URL",
"PAPERCLIP_NATIVE_MCP_TOKEN",
] {
std::env::remove_var(key);
}
match (name, url, token) {
(None, None, None) => Ok(None),
(Some(name), Some(url), Some(mut token))
if !name.trim().is_empty() && url.starts_with("https://") && !token.is_empty() =>
{
let remote = HarnessRemoteMcpConfig::builder()
.url(url)
.headers("Authorization", format!("Bearer {token}"))
.build()
.map_err(|_| {
LocalRunnerError::invalid("failed to build AgentCore remote MCP configuration")
})?;
token.clear();
let tool = HarnessTool::builder()
.r#type(HarnessToolType::RemoteMcp)
.name(name.clone())
.config(HarnessToolConfiguration::RemoteMcp(remote))
.build()
.map_err(|_| {
LocalRunnerError::invalid("failed to build AgentCore remote MCP tool")
})?;
Ok(Some((tool, format!("@{name}/*"))))
}
(_, _, Some(mut token)) => {
token.clear();
Err(LocalRunnerError::invalid(
"AgentCore MCP binding is incomplete or unsafe",
))
}
_ => Err(LocalRunnerError::invalid(
"AgentCore MCP binding is incomplete",
)),
}
}
fn context_text<'a>(value: &'a Value, pointer: &str) -> Result<&'a str, String> {
value
.pointer(pointer)
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| format!("runtimeContext {pointer} is missing"))
}
fn collect_context_files(root: &Path) -> Result<Vec<(PathBuf, Vec<u8>)>, String> {
fn visit(
root: &Path,
current: &Path,
files: &mut Vec<(PathBuf, Vec<u8>)>,
total: &mut usize,
) -> Result<(), String> {
let mut entries = fs::read_dir(current)
.map_err(|error| format!("failed to read runtime context asset: {error}"))?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| format!("failed to enumerate runtime context asset: {error}"))?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries {
let path = entry.path();
let metadata = fs::symlink_metadata(&path)
.map_err(|error| format!("failed to inspect runtime context asset: {error}"))?;
if metadata.file_type().is_symlink() {
return Err("AgentCore context bundles may not contain symlinks".to_owned());
}
if metadata.is_dir() {
visit(root, &path, files, total)?;
} else if metadata.is_file() {
let relative = path
.strip_prefix(root)
.map_err(|_| "runtime context asset escaped its root".to_owned())?
.to_path_buf();
let bytes = fs::read(&path)
.map_err(|error| format!("failed to read runtime context file: {error}"))?;
*total = total.saturating_add(bytes.len());
if *total > MAX_CONTEXT_UPLOAD_BYTES {
return Err("AgentCore context upload exceeded its size limit".to_owned());
}
files.push((relative, bytes));
}
}
Ok(())
}
let metadata = fs::symlink_metadata(root)
.map_err(|error| format!("runtime context asset is unavailable: {error}"))?;
if metadata.file_type().is_symlink() || !metadata.is_dir() {
return Err("runtime context asset root must be a real directory".to_owned());
}
let mut files = Vec::new();
let mut total = 0;
visit(root, root, &mut files, &mut total)?;
Ok(files)
}
async fn upload_context_directory(
client: &aws_sdk_s3::Client,
config: &AwsAgentCoreProviderConfig,
digest: &str,
files: Vec<(PathBuf, Vec<u8>)>,
generated_skill: Option<String>,
) -> Result<HarnessSkill, String> {
let prefix = config.context_prefix.trim_matches('/');
let asset_prefix = format!("{prefix}/assets/{digest}");
let mut files = files;
if let Some(skill) = generated_skill {
files.push((PathBuf::from("SKILL.md"), skill.into_bytes()));
}
for (relative, bytes) in files {
let relative = relative
.components()
.map(|component| component.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/");
if relative.is_empty()
|| relative
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
{
return Err("runtime context upload path is unsafe".to_owned());
}
let key = format!("{asset_prefix}/{relative}");
let content_digest = format!("{:x}", Sha256::digest(&bytes));
if let Ok(existing) = client
.head_object()
.bucket(&config.context_bucket)
.key(&key)
.send()
.await
{
let metadata_digest = existing
.metadata()
.and_then(|metadata| metadata.get("paperclip-sha256"))
.map(String::as_str);
if existing.content_length() != Some(bytes.len() as i64)
|| metadata_digest != Some(content_digest.as_str())
|| existing.server_side_encryption()
!= Some(&aws_sdk_s3::types::ServerSideEncryption::AwsKms)
|| existing.ssekms_key_id() != Some(config.context_kms_key_arn.as_str())
{
return Err(format!(
"AgentCore context S3 asset verification failed: {}",
sha_hex(&key, 16)
));
}
continue;
}
client
.put_object()
.bucket(&config.context_bucket)
.key(&key)
.server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::AwsKms)
.ssekms_key_id(&config.context_kms_key_arn)
.metadata("paperclip-sha256", content_digest)
.body(ByteStream::from(bytes))
.send()
.await
.map_err(|error| {
format!(
"AgentCore context S3 upload failed: {}",
redact_aws_error(&error.to_string())
)
})?;
}
let source = HarnessSkillS3Source::builder()
.uri(format!("s3://{}/{asset_prefix}/", config.context_bucket))
.build()
.map_err(|_| "failed to build AgentCore S3 skill source".to_owned())?;
Ok(HarnessSkill::S3(source))
}
async fn upload_agentcore_runtime_context(
client: &aws_sdk_s3::Client,
config: &AwsAgentCoreProviderConfig,
) -> Result<Vec<HarnessSkill>, String> {
let context = config.runtime_context.as_ref().ok_or_else(|| {
"AgentCore requires paperclip.native-execution-input.v3 runtimeContext".to_owned()
})?;
let instruction_digest = context_text(context, "/instructions/bundle/digest")?;
let instruction_root = Path::new(context_text(context, "/instructions/bundle/rootPath")?);
let entry_path = context_text(context, "/instructions/entryPath")?;
let instruction_files = collect_context_files(instruction_root)?
.into_iter()
.map(|(path, bytes)| (PathBuf::from("instructions").join(path), bytes))
.collect();
let instruction_companion = format!(
"---\nname: paperclip-instructions-{}\ndescription: Paperclip agent instruction sibling bundle\n---\nRead `instructions/{entry_path}` and its sibling files as read-only context.\n",
&instruction_digest[..12]
);
let instruction_asset_digest = sha_hex(
&format!("{instruction_digest}\0{entry_path}\0{instruction_companion}"),
64,
);
let mut skills = vec![
upload_context_directory(
client,
config,
&instruction_asset_digest,
instruction_files,
Some(instruction_companion),
)
.await?,
];
let assigned = context
.get("skills")
.and_then(Value::as_array)
.ok_or_else(|| "runtimeContext.skills must be an array".to_owned())?;
for skill in assigned {
let digest = context_text(skill, "/bundle/digest")?;
let root = Path::new(context_text(skill, "/bundle/rootPath")?);
let files = collect_context_files(root)?;
if !files.iter().any(|(path, _)| path == Path::new("SKILL.md")) {
return Err("AgentCore custom skill bundle is missing SKILL.md".to_owned());
}
skills.push(upload_context_directory(client, config, digest, files, None).await?);
}
Ok(skills)
}
fn agentcore_system_instructions(config: &AwsAgentCoreProviderConfig) -> Result<String, String> {
let Some(context) = config.runtime_context.as_ref() else {
return Ok(config.instructions.clone());
};
let instruction_root = context_text(context, "/instructions/bundle/rootPath")?;
let local_directive = format!("Read-only instruction sibling root: {instruction_root}");
config
.instructions
.strip_suffix(&local_directive)
.map(|prefix| format!(
"{prefix}Read-only instruction siblings are in the attached Paperclip HarnessSkill under `instructions/`."
))
.ok_or_else(|| "AgentCore instruction-root directive is missing or inconsistent".to_owned())
}
fn network_loop(
config: AwsAgentCoreProviderConfig,
session_id: String,
@@ -231,10 +477,24 @@ fn network_loop(
.credentials_provider(assumed)
.load()
.await;
(aws_sdk_bedrockagentcore::Client::new(&shared), config)
let s3 = aws_sdk_s3::Client::new(&shared);
let skills = upload_agentcore_runtime_context(&s3, &config).await?;
let system_instructions = agentcore_system_instructions(&config)?;
Ok::<_, String>((
aws_sdk_bedrockagentcore::Client::new(&shared),
config,
skills,
system_instructions,
))
});
let (client, config, skills, system_instructions) = match clients {
Ok(clients) => clients,
Err(error) => {
let _ = ready.send(Err(error));
return;
}
};
let _ = ready.send(Ok(()));
let (client, config) = clients;
while let Ok(command) = commands.recv() {
match command {
NetworkCommand::Invoke {
@@ -249,6 +509,8 @@ fn network_loop(
let session_id = session_id.clone();
let actor_id = actor_id.clone();
let events = events.clone();
let skills = skills.clone();
let system_instructions = system_instructions.clone();
runtime.spawn(async move {
// The qualified Harness version is the immutable model
// authority. Supplying a redundant invocation override
@@ -269,7 +531,10 @@ fn network_loop(
// silently withheld, so admit each authorized tool as
// `@*/<collision-resistant pc_ name>` and nothing else.
.set_allowed_tools(Some(allowed_tools))
.set_skills(Some(Vec::new()))
.set_system_prompt(Some(vec![HarnessSystemContentBlock::Text(
system_instructions,
)]))
.set_skills(Some(skills))
.max_iterations(config.max_iterations as i32)
.max_tokens(config.max_output_tokens as i32)
.timeout_seconds(config.timeout_seconds as i32)
@@ -524,6 +789,8 @@ pub struct AwsAgentCoreHarnessProvider {
usage: Value,
interrupted: bool,
max_estimated_cost_usd: f64,
assigned_mcp_tool: Option<HarnessTool>,
assigned_mcp_allowed: Option<String>,
}
impl AwsAgentCoreHarnessProvider {
@@ -538,8 +805,13 @@ impl AwsAgentCoreHarnessProvider {
.map(str::to_owned)
.unwrap_or_else(new_runtime_session_id);
let actor_id = format!("paperclip-{}", sha_hex(&session_id, 32));
let assigned_mcp = take_agentcore_mcp_tool()?;
let worker = NetworkWorker::start(config.clone(), session_id.clone(), actor_id.clone())?;
let (encoded, allowed, reverse, schemas) = encode_tools(&tools)?;
let (mut encoded, mut allowed, reverse, schemas) = encode_tools(&tools)?;
if let Some((tool, allow)) = assigned_mcp.as_ref() {
encoded.push(tool.clone());
allowed.push(allow.clone());
}
Ok(Self {
config: config.clone(),
session_id,
@@ -559,6 +831,8 @@ impl AwsAgentCoreHarnessProvider {
usage: json!({ "inputTokens": 0, "outputTokens": 0, "cacheReadInputTokens": 0, "cacheWriteInputTokens": 0, "requestCount": 0, "estimatedCostUsd": 0.0, "costSource": "paperclip_estimate" }),
interrupted: false,
max_estimated_cost_usd: config.max_estimated_session_cost_usd,
assigned_mcp_tool: assigned_mcp.as_ref().map(|value| value.0.clone()),
assigned_mcp_allowed: assigned_mcp.as_ref().map(|value| value.1.clone()),
})
}
@@ -614,7 +888,13 @@ impl Provider for AwsAgentCoreHarnessProvider {
"cannot replace AgentCore tools while a turn is active",
));
}
let (encoded, allowed, reverse, schemas) = encode_tools(&tools)?;
let (mut encoded, mut allowed, reverse, schemas) = encode_tools(&tools)?;
if let Some(tool) = self.assigned_mcp_tool.as_ref() {
encoded.push(tool.clone());
}
if let Some(allow) = self.assigned_mcp_allowed.as_ref() {
allowed.push(allow.clone());
}
self.tools = encoded;
self.allowed_tools = allowed;
self.remote_to_canonical = reverse;
@@ -992,7 +1272,11 @@ fn validate_config(config: &AwsAgentCoreProviderConfig) -> Result<(), LocalRunne
config.memory_arn.as_str(),
config.memory_id.as_str(),
config.invocation_role_arn.as_str(),
config.context_bucket.as_str(),
config.context_prefix.as_str(),
config.context_kms_key_arn.as_str(),
config.qualification_revision.as_str(),
config.instructions.as_str(),
]
.iter()
.any(|value| value.trim().is_empty())
@@ -1001,6 +1285,17 @@ fn validate_config(config: &AwsAgentCoreProviderConfig) -> Result<(), LocalRunne
"AgentCore profile fields must be non-empty",
));
}
if config.context_prefix.starts_with('/')
|| config
.context_prefix
.split('/')
.any(|part| part.is_empty() || part == "." || part == "..")
|| !config.context_kms_key_arn.starts_with("arn:aws:kms:")
{
return Err(LocalRunnerError::invalid(
"AgentCore context S3 qualification is unsafe",
));
}
if config.event_expiry_days != 90 {
return Err(LocalRunnerError::invalid(
"AgentCore profile must use the qualified 90-day Memory expiry",
@@ -1267,6 +1562,36 @@ mod tests {
HarnessToolUseBlockDelta, HarnessToolUseBlockStart,
};
fn config() -> AwsAgentCoreProviderConfig {
AwsAgentCoreProviderConfig {
model: "global.anthropic.claude-sonnet-4-6".to_owned(),
profile_id: "profile-test".to_owned(),
region: "us-east-1".to_owned(),
account_id: "123456789012".to_owned(),
harness_arn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:harness/test".to_owned(),
harness_version: "1".to_owned(),
endpoint_arn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:endpoint/test"
.to_owned(),
endpoint_qualifier: "1".to_owned(),
agent_runtime_arn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/test"
.to_owned(),
memory_arn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/test".to_owned(),
memory_id: "memory-test".to_owned(),
invocation_role_arn: "arn:aws:iam::123456789012:role/paperclip-agentcore".to_owned(),
context_bucket: "paperclip-context-test".to_owned(),
context_prefix: "companies/company-test/profiles/profile-test".to_owned(),
context_kms_key_arn: "arn:aws:kms:us-east-1:123456789012:key/test".to_owned(),
qualification_revision: "aws-agentcore-harness-v1".to_owned(),
event_expiry_days: 90,
max_estimated_session_cost_usd: 1.0,
max_iterations: 8,
max_output_tokens: 4096,
timeout_seconds: 300,
instructions: "Paperclip test instructions".to_owned(),
runtime_context: None,
}
}
#[test]
fn runtime_session_ids_are_valid_and_unique() {
let first = new_runtime_session_id();
@@ -1275,6 +1600,44 @@ mod tests {
assert_ne!(first, second);
}
#[test]
fn agentcore_system_replaces_the_local_instruction_path_with_the_harness_skill() {
let mut config = config();
let local_root = "/paperclip/runtime/instructions";
config.instructions = format!(
"paperclip prompt\n\nAGENTS entry\n\nRead-only instruction sibling root: {local_root}"
);
config.runtime_context = Some(json!({
"instructions": {
"entryPath": "AGENTS.md",
"bundle": { "digest": "abc123", "rootPath": local_root }
},
"skills": []
}));
let instructions = agentcore_system_instructions(&config).unwrap();
assert!(instructions.starts_with("paperclip prompt\n\nAGENTS entry\n\n"));
assert!(instructions.ends_with("attached Paperclip HarnessSkill under `instructions/`."));
assert!(!instructions.contains(local_root));
}
#[test]
fn agentcore_rejects_unsafe_context_prefixes_before_cloud_access() {
for unsafe_prefix in [
"/absolute",
"company//profile",
"company/../profile",
"company/./profile",
] {
let mut config = config();
config.context_prefix = unsafe_prefix.to_owned();
assert!(
validate_config(&config).is_err(),
"prefix should fail: {unsafe_prefix}"
);
}
}
#[test]
fn tool_names_are_safe_stable_and_collision_resistant() {
let first = remote_tool_name("issues.comment:create");
packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs
+75
View File
@@ -9,6 +9,7 @@ struct Mode {
large_event: bool,
oversized_event: bool,
linger_after_turn_start: bool,
runtime_question: bool,
}
fn send(value: Value) -> io::Result<()> {
@@ -24,11 +25,13 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
large_event: args.iter().any(|arg| arg == "--large-event"),
oversized_event: args.iter().any(|arg| arg == "--oversized-event"),
linger_after_turn_start: args.iter().any(|arg| arg == "--linger-after-turn-start"),
runtime_question: args.iter().any(|arg| arg == "--runtime-question"),
};
let mut turn_count = 0_u64;
let mut active_turn_id: Option<String> = None;
let mut planning_thread = false;
let mut pending_tool_turns = BTreeMap::<String, String>::new();
let mut pending_question_turns = BTreeMap::<String, String>::new();
let stdin = io::stdin();
for line in stdin.lock().lines() {
let request: Value = serde_json::from_str(&line?)?;
@@ -149,6 +152,45 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
if mode.linger_after_turn_start {
continue;
}
if mode.runtime_question {
let request_id = format!("rpc-question-{turn_count}");
pending_question_turns.insert(request_id.clone(), turn_id.clone());
send(json!({
"id": request_id,
"method": "item/tool/requestUserInput",
"params": {
"threadId": "fake-thread",
"turnId": turn_id,
"itemId": "question-item",
"questions": [
{
"id": "environment",
"header": "Environment",
"question": "Where should we deploy?",
"options": [
{"label": "Staging", "description": "Deploy to staging first."},
{"label": "Production", "description": "Deploy directly to production."}
],
"isOther": true
},
{
"id": "regions",
"header": "Regions",
"question": "Which regions?",
"options": [{"label": "US"}, {"label": "EU"}],
"multiSelect": true
},
{
"id": "notes",
"header": "Notes",
"question": "Anything else?",
"required": false
}
]
}
}))?;
continue;
}
pending_tool_turns.insert(request_id.clone(), turn_id.clone());
send(json!({
"id": request_id, "method": "item/tool/call",
@@ -193,6 +235,39 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
json!({"method": "turn/completed", "params": {"threadId": "fake-thread", "turn": {"id": turn_id, "status": "completed"}}}),
)?;
}
let completed_question = request
.get("id")
.and_then(Value::as_str)
.and_then(|request_id| pending_question_turns.remove(request_id));
if let Some(turn_id) = completed_question.filter(|_| request.get("result").is_some()) {
if request.pointer("/result/answers")
!= Some(&json!({
"environment": {"answers": ["Staging"]},
"regions": {"answers": ["US", "EU"]},
"notes": {"answers": ["Ship during the maintenance window."]}
}))
{
return Err(
format!("runtime question received wrong response: {}", request).into(),
);
}
send(json!({
"method": "paperclip/runResult",
"params": {
"threadId": "fake-thread",
"turnId": turn_id,
"itemId": "semantic-result",
"result": {
"schema": "paperclip.prp.result.v1",
"reportedWorkDisposition": "done",
"summary": "Fake provider completed after runtime input."
}
}
}))?;
send(
json!({"method": "turn/completed", "params": {"threadId": "fake-thread", "turn": {"id": turn_id, "status": "completed"}}}),
)?;
}
}
Ok(())
}
packages/paperclip-runner/runner/crates/runner-core/src/claude_managed_provider.rs
+477 -6
View File
@@ -1,5 +1,7 @@
use std::collections::{BTreeMap, VecDeque};
use std::fs;
use std::io::{BufRead, BufReader, Read};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{self, Receiver, SyncSender};
use std::sync::Arc;
@@ -21,11 +23,19 @@ use crate::provider_bridge::{AuthorizedTool, ToolResult};
const ANTHROPIC_ORIGIN: &str = "https://api.anthropic.com";
const ANTHROPIC_VERSION: &str = "2023-06-01";
const QUALIFIED_BETA: &str = "managed-agents-2026-04-01";
const SKILLS_BETA: &str = "skills-2025-10-02";
const MAX_REMOTE_EVENT_BYTES: usize = 4 * 1024 * 1024;
const MAX_REMOTE_RESPONSE_BYTES: usize = 8 * 1024 * 1024;
const MAX_REMOTE_TOOLS: usize = 128;
const MAX_HISTORY_EVENTS: usize = 10_000;
const MAX_HISTORY_PAGES: usize = 100;
const MAX_SKILL_UPLOAD_BYTES: usize = 32 * 1024 * 1024;
#[derive(Clone)]
struct ManagedMcpBinding {
name: String,
capability_url: String,
}
struct SensitiveApiKey(String);
@@ -184,6 +194,376 @@ impl Drop for NetworkWorker {
}
}
fn take_managed_mcp_binding() -> Result<Option<ManagedMcpBinding>, LocalRunnerError> {
let name = std::env::var("PAPERCLIP_NATIVE_MCP_NAME").ok();
let url = std::env::var("PAPERCLIP_NATIVE_MCP_URL").ok();
let token = std::env::var("PAPERCLIP_NATIVE_MCP_TOKEN").ok();
for key in [
"PAPERCLIP_NATIVE_MCP_NAME",
"PAPERCLIP_NATIVE_MCP_URL",
"PAPERCLIP_NATIVE_MCP_TOKEN",
] {
std::env::remove_var(key);
}
match (name, url, token) {
(None, None, None) => Ok(None),
(Some(name), Some(url), Some(mut token))
if !name.trim().is_empty() && url.starts_with("https://") && !token.is_empty() =>
{
let separator = if url.contains('?') { '&' } else { '?' };
let capability_url = format!(
"{url}{separator}paperclip_capability={}",
percent_encode_query(&token)
);
token.clear();
Ok(Some(ManagedMcpBinding {
name,
capability_url,
}))
}
(_, _, Some(mut token)) => {
token.clear();
Err(LocalRunnerError::invalid(
"Claude Managed MCP binding is incomplete or unsafe",
))
}
_ => Err(LocalRunnerError::invalid(
"Claude Managed MCP binding is incomplete",
)),
}
}
fn runtime_array<'a>(context: &'a Value, field: &str) -> Result<&'a Vec<Value>, LocalRunnerError> {
context.get(field).and_then(Value::as_array).ok_or_else(|| {
LocalRunnerError::invalid(format!("runtimeContext.{field} must be an array"))
})
}
fn runtime_text<'a>(value: &'a Value, pointer: &str) -> Result<&'a str, LocalRunnerError> {
value
.pointer(pointer)
.and_then(Value::as_str)
.filter(|value| !value.trim().is_empty())
.ok_or_else(|| LocalRunnerError::invalid(format!("runtimeContext {pointer} is missing")))
}
fn collect_bundle_files(root: &Path) -> Result<Vec<(PathBuf, Vec<u8>)>, LocalRunnerError> {
fn visit(
root: &Path,
current: &Path,
files: &mut Vec<(PathBuf, Vec<u8>)>,
total: &mut usize,
) -> Result<(), LocalRunnerError> {
let mut entries = fs::read_dir(current)
.map_err(|error| {
LocalRunnerError::invalid(format!("failed to read runtime asset: {error}"))
})?
.collect::<Result<Vec<_>, _>>()
.map_err(|error| {
LocalRunnerError::invalid(format!("failed to enumerate runtime asset: {error}"))
})?;
entries.sort_by_key(|entry| entry.file_name());
for entry in entries {
let path = entry.path();
let metadata = fs::symlink_metadata(&path).map_err(|error| {
LocalRunnerError::invalid(format!("failed to inspect runtime asset: {error}"))
})?;
if metadata.file_type().is_symlink() {
return Err(LocalRunnerError::invalid(
"managed skill bundles may not contain symlinks",
));
}
if metadata.is_dir() {
visit(root, &path, files, total)?;
} else if metadata.is_file() {
let relative = path
.strip_prefix(root)
.map_err(|_| LocalRunnerError::invalid("runtime asset escaped its root"))?
.to_path_buf();
let bytes = fs::read(&path).map_err(|error| {
LocalRunnerError::invalid(format!("failed to read runtime asset file: {error}"))
})?;
*total = total.saturating_add(bytes.len());
if *total > MAX_SKILL_UPLOAD_BYTES {
return Err(LocalRunnerError::invalid(
"managed skill upload exceeded its size limit",
));
}
files.push((relative, bytes));
}
}
Ok(())
}
let metadata = fs::symlink_metadata(root).map_err(|error| {
LocalRunnerError::invalid(format!("runtime asset is unavailable: {error}"))
})?;
if !metadata.is_dir() || metadata.file_type().is_symlink() {
return Err(LocalRunnerError::invalid(
"runtime asset root must be a real directory",
));
}
let mut files = Vec::new();
let mut total = 0;
visit(root, root, &mut files, &mut total)?;
Ok(files)
}
fn multipart_field(body: &mut Vec<u8>, boundary: &str, name: &str, value: &str) {
body.extend_from_slice(
format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
)
.as_bytes(),
);
}
fn multipart_file(body: &mut Vec<u8>, boundary: &str, filename: &str, bytes: &[u8]) {
body.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: form-data; name=\"files[]\"; filename=\"{filename}\"\r\nContent-Type: application/octet-stream\r\n\r\n").as_bytes());
body.extend_from_slice(bytes);
body.extend_from_slice(b"\r\n");
}
fn upload_managed_skill(
client: &Client,
title: &str,
top_level: &str,
mut files: Vec<(PathBuf, Vec<u8>)>,
generated_skill: Option<String>,
) -> Result<Value, LocalRunnerError> {
let listed: Value = client
.get(format!(
"{ANTHROPIC_ORIGIN}/v1/skills?limit=1000&source=custom"
))
.send()
.map_err(|_| LocalRunnerError::invalid("Anthropic skill mapping lookup failed"))?
.json()
.map_err(|_| {
LocalRunnerError::invalid("Anthropic skill mapping lookup returned invalid JSON")
})?;
if let Some(existing) = listed
.get("data")
.and_then(Value::as_array)
.into_iter()
.flatten()
.find(|skill| {
skill.get("display_title").and_then(Value::as_str) == Some(title)
|| skill.get("title").and_then(Value::as_str) == Some(title)
})
{
let skill_id = existing
.get("id")
.and_then(Value::as_str)
.ok_or_else(|| LocalRunnerError::invalid("Anthropic skill mapping omitted id"))?;
let version = existing.get("latest_version").cloned().ok_or_else(|| {
LocalRunnerError::invalid("Anthropic skill mapping omitted latest_version")
})?;
return Ok(json!({ "type": "custom", "skill_id": skill_id, "version": version }));
}
if let Some(skill) = generated_skill {
files.push((PathBuf::from("SKILL.md"), skill.into_bytes()));
}
if !files.iter().any(|(path, _)| path == Path::new("SKILL.md")) {
return Err(LocalRunnerError::invalid(
"managed custom skill bundle is missing SKILL.md",
));
}
let boundary = format!(
"paperclip-{}",
&format!("{:x}", Sha256::digest(title.as_bytes()))[..24]
);
let mut body = Vec::new();
multipart_field(&mut body, &boundary, "display_title", title);
multipart_field(
&mut body,
&boundary,
"description",
"Paperclip-owned immutable runtime context asset",
);
files.sort_by(|left, right| left.0.cmp(&right.0));
for (path, bytes) in files {
let relative = path
.components()
.map(|part| part.as_os_str().to_string_lossy())
.collect::<Vec<_>>()
.join("/");
multipart_file(
&mut body,
&boundary,
&format!("{top_level}/{relative}"),
&bytes,
);
}
body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
let response = client
.post(format!("{ANTHROPIC_ORIGIN}/v1/skills"))
.header(
CONTENT_TYPE,
format!("multipart/form-data; boundary={boundary}"),
)
.body(body)
.send()
.map_err(|_| LocalRunnerError::invalid("Anthropic skill upload transport failed"))?;
if !response.status().is_success() {
return Err(LocalRunnerError::invalid(format!(
"Anthropic skill upload failed with HTTP {}",
response.status().as_u16()
)));
}
let value: Value = response
.json()
.map_err(|_| LocalRunnerError::invalid("Anthropic skill upload returned invalid JSON"))?;
let skill_id = value
.get("id")
.and_then(Value::as_str)
.ok_or_else(|| LocalRunnerError::invalid("Anthropic skill upload omitted id"))?;
let version = value.get("latest_version").cloned().ok_or_else(|| {
LocalRunnerError::invalid("Anthropic skill upload omitted latest_version")
})?;
Ok(json!({ "type": "custom", "skill_id": skill_id, "version": version }))
}
fn upload_managed_runtime_skills(
api_key: &str,
config: &ClaudeManagedProviderConfig,
) -> Result<Vec<Value>, LocalRunnerError> {
let Some(context) = config.runtime_context.as_ref() else {
return Ok(Vec::new());
};
let mut headers = HeaderMap::new();
headers.insert(
"x-api-key",
HeaderValue::from_str(api_key).map_err(|_| {
LocalRunnerError::invalid("Anthropic API key contains invalid header bytes")
})?,
);
headers.insert(
"anthropic-version",
HeaderValue::from_static(ANTHROPIC_VERSION),
);
headers.insert("anthropic-beta", HeaderValue::from_static(SKILLS_BETA));
let client = Client::builder()
.default_headers(headers)
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(60))
.https_only(true)
.build()
.map_err(|_| LocalRunnerError::invalid("failed to construct Anthropic skills client"))?;
let instruction_root = Path::new(runtime_text(context, "/instructions/bundle/rootPath")?);
let instruction_digest = runtime_text(context, "/instructions/bundle/digest")?;
let entry_path = runtime_text(context, "/instructions/entryPath")?;
let instruction_files = collect_bundle_files(instruction_root)?
.into_iter()
.map(|(path, bytes)| (PathBuf::from("instructions").join(path), bytes))
.collect();
let instruction_identity = format!(
"{:x}",
Sha256::digest(format!("{instruction_digest}\0{entry_path}").as_bytes())
);
let instruction_name = format!("paperclip-instructions-{}", &instruction_identity[..12]);
let companion = format!(
"---\nname: {instruction_name}\ndescription: Paperclip agent instruction sibling bundle\n---\nRead `instructions/{entry_path}` and its sibling files when the system instructions require them. Treat all files as read-only.\n"
);
let mut attachments = vec![upload_managed_skill(
&client,
&format!(
"pc:{}:{}:instructions:{}",
config.profile_id, config.agent_version, instruction_identity
),
&instruction_name,
instruction_files,
Some(companion),
)?];
for skill in runtime_array(context, "skills")? {
let key = skill.get("key").and_then(Value::as_str).unwrap_or("skill");
let runtime_name = runtime_text(skill, "/runtimeName")?;
let digest = runtime_text(skill, "/bundle/digest")?;
let root = Path::new(runtime_text(skill, "/bundle/rootPath")?);
attachments.push(upload_managed_skill(
&client,
&format!(
"pc:{}:{}:{}:{}",
config.profile_id,
config.agent_version,
&format!("{:x}", Sha256::digest(key.as_bytes()))[..12],
digest
),
runtime_name,
collect_bundle_files(root)?,
None,
)?);
}
Ok(attachments)
}
fn managed_system_instructions(
config: &ClaudeManagedProviderConfig,
) -> Result<String, LocalRunnerError> {
let Some(context) = config.runtime_context.as_ref() else {
return Ok(config.instructions.clone());
};
let instruction_root = runtime_text(context, "/instructions/bundle/rootPath")?;
let instruction_digest = runtime_text(context, "/instructions/bundle/digest")?;
let entry_path = runtime_text(context, "/instructions/entryPath")?;
let instruction_identity = format!(
"{:x}",
Sha256::digest(format!("{instruction_digest}\0{entry_path}").as_bytes())
);
let instruction_name = format!("paperclip-instructions-{}", &instruction_identity[..12]);
let local_directive = format!("Read-only instruction sibling root: {instruction_root}");
let replacement = format!(
"Read-only instruction siblings are in the attached `{instruction_name}` skill under `instructions/`."
);
config
.instructions
.strip_suffix(&local_directive)
.map(|prefix| format!("{prefix}{replacement}"))
.ok_or_else(|| {
LocalRunnerError::invalid(
"Claude Managed instruction-root directive is missing or inconsistent",
)
})
}
fn managed_agent_overrides(
config: &ClaudeManagedProviderConfig,
system_instructions: &str,
mut custom_tools: Vec<Value>,
skills: &[Value],
mcp: Option<&ManagedMcpBinding>,
) -> Value {
let mut tools = vec![json!({
"type": "agent_toolset_20260401",
"default_config": { "enabled": false },
"configs": [{ "name": "read", "enabled": true }]
})];
tools.append(&mut custom_tools);
let mcp_servers = mcp
.map(|binding| {
vec![json!({
"type": "url",
"name": binding.name,
"url": binding.capability_url,
})]
})
.unwrap_or_default();
if let Some(binding) = mcp {
tools.push(json!({
"type": "mcp_toolset",
"mcp_server_name": binding.name,
"default_config": { "enabled": true }
}));
}
json!({
"model": { "id": config.model },
"system": system_instructions,
"tools": tools,
"mcp_servers": mcp_servers,
"skills": skills,
})
}
fn network_loop(
api_key: SensitiveApiKey,
beta: String,
@@ -445,6 +825,10 @@ pub struct ClaudeManagedProvider {
model_request_count: u64,
current_budget_cents: u64,
reconnect_backoff: Duration,
config: ClaudeManagedProviderConfig,
system_instructions: String,
managed_skills: Vec<Value>,
mcp_binding: Option<ManagedMcpBinding>,
}
impl ClaudeManagedProvider {
@@ -459,12 +843,16 @@ impl ClaudeManagedProvider {
LocalRunnerError::invalid("ANTHROPIC_API_KEY is required for Claude Agent")
})?;
std::env::remove_var("ANTHROPIC_API_KEY");
let mcp_binding = take_managed_mcp_binding()?;
let managed_skills = upload_managed_runtime_skills(&api_key, config)?;
Self::start_with_worker(
config,
tools,
resume_session_id,
resume_event_cursor,
resume_model_request_count,
managed_skills,
mcp_binding,
SensitiveApiKey(api_key),
NetworkWorker::start,
)
@@ -476,6 +864,8 @@ impl ClaudeManagedProvider {
resume_session_id: Option<&str>,
resume_event_cursor: Option<&str>,
resume_model_request_count: u64,
managed_skills: Vec<Value>,
mcp_binding: Option<ManagedMcpBinding>,
api_key: SensitiveApiKey,
start_worker: F,
) -> Result<Self, LocalRunnerError>
@@ -483,8 +873,16 @@ impl ClaudeManagedProvider {
F: FnOnce(SensitiveApiKey, &str) -> Result<NetworkWorker, LocalRunnerError>,
{
validate_config(config)?;
let system_instructions = managed_system_instructions(config)?;
let worker = start_worker(api_key, &config.beta_version)?;
let (tool_payload, reverse, operation_input_schemas) = encode_tools(&tools)?;
let agent_overrides = managed_agent_overrides(
config,
&system_instructions,
tool_payload,
&managed_skills,
mcp_binding.as_ref(),
);
let mut replay_queue = VecDeque::new();
let session_id = if let Some(session_id) = resume_session_id {
let session = worker.request(
@@ -496,7 +894,7 @@ impl ClaudeManagedProvider {
worker.request(
Method::POST,
format!("/v1/sessions/{session_id}?beta=true"),
Some(json!({ "agent": { "tools": tool_payload, "mcp_servers": [] } })),
Some(json!({ "agent": agent_overrides })),
)?;
session_id.to_owned()
} else {
@@ -516,10 +914,11 @@ impl ClaudeManagedProvider {
"type": "agent_with_overrides",
"id": config.anthropic_agent_id,
"version": version,
"model": { "id": config.model },
"tools": tool_payload,
"mcp_servers": [],
"skills": []
"model": agent_overrides["model"].clone(),
"system": agent_overrides["system"].clone(),
"tools": agent_overrides["tools"].clone(),
"mcp_servers": agent_overrides["mcp_servers"].clone(),
"skills": agent_overrides["skills"].clone()
},
"environment_id": config.environment_id,
"budget": {
@@ -558,6 +957,10 @@ impl ClaudeManagedProvider {
model_request_count: resume_model_request_count,
current_budget_cents: spend_cap_cents(config.max_session_list_cost_usd)?,
reconnect_backoff: Duration::from_millis(250),
config: config.clone(),
system_instructions,
managed_skills,
mcp_binding,
})
}
@@ -806,10 +1209,17 @@ impl Provider for ClaudeManagedProvider {
fn configure_tools(&mut self, tools: Vec<AuthorizedTool>) -> Result<(), LocalRunnerError> {
let (payload, reverse, operation_input_schemas) = encode_tools(&tools)?;
let overrides = managed_agent_overrides(
&self.config,
&self.system_instructions,
payload,
&self.managed_skills,
self.mcp_binding.as_ref(),
);
self.worker.request(
Method::POST,
format!("/v1/sessions/{}?beta=true", self.session_id),
Some(json!({ "agent": { "tools": payload, "mcp_servers": [] } })),
Some(json!({ "agent": overrides })),
)?;
self.remote_to_canonical = reverse;
self.operation_input_schemas = operation_input_schemas;
@@ -999,6 +1409,7 @@ fn validate_config(config: &ClaudeManagedProviderConfig) -> Result<(), LocalRunn
config.anthropic_agent_id.as_str(),
config.agent_version.as_str(),
config.environment_id.as_str(),
config.instructions.as_str(),
]
.iter()
.any(|value| value.trim().is_empty())
@@ -1432,6 +1843,8 @@ mod tests {
environment_id: "env_test".to_owned(),
beta_version: QUALIFIED_BETA.to_owned(),
max_session_list_cost_usd: 1.0,
instructions: "Paperclip test system instructions".to_owned(),
runtime_context: None,
}
}
@@ -1463,6 +1876,8 @@ mod tests {
resume_session_id,
cursor,
0,
Vec::new(),
None,
SensitiveApiKey("anthropic-test-secret".to_owned()),
move |key, beta| NetworkWorker::start_at(key, beta, &origin, false),
)
@@ -1480,6 +1895,62 @@ mod tests {
assert_eq!(first, remote_tool_name("issues.comment:create"));
}
#[test]
fn managed_overrides_replace_saved_context_and_pin_skills_and_mcp() {
let binding = ManagedMcpBinding {
name: "paperclip-assigned".to_owned(),
capability_url: "https://paperclip.example/mcp?paperclip_capability=secret".to_owned(),
};
let overrides = managed_agent_overrides(
&config(),
"Paperclip test system instructions",
vec![json!({ "type": "custom", "name": "pc_finish" })],
&[json!({ "type": "custom", "skill_id": "skill_1", "version": 7 })],
Some(&binding),
);
assert_eq!(
overrides.get("system"),
Some(&json!("Paperclip test system instructions"))
);
assert_eq!(overrides.pointer("/skills/0/version"), Some(&json!(7)));
assert_eq!(
overrides.pointer("/mcp_servers/0/name"),
Some(&json!("paperclip-assigned"))
);
assert!(overrides
.get("tools")
.and_then(Value::as_array)
.is_some_and(|tools| tools
.iter()
.any(|tool| tool.get("type") == Some(&json!("agent_toolset_20260401")))));
assert!(serde_json::to_string(&config())
.unwrap()
.find("paperclip_capability")
.is_none());
}
#[test]
fn managed_system_replaces_the_local_instruction_path_with_the_companion_skill() {
let mut config = config();
let local_root = "/paperclip/runtime/instructions";
config.instructions = format!(
"paperclip prompt\n\nAGENTS entry\n\nRead-only instruction sibling root: {local_root}"
);
config.runtime_context = Some(json!({
"instructions": {
"entryPath": "AGENTS.md",
"bundle": { "digest": "abc123", "rootPath": local_root }
},
"skills": []
}));
let instructions = managed_system_instructions(&config).unwrap();
assert!(instructions.starts_with("paperclip prompt\n\nAGENTS entry\n\n"));
assert!(instructions.contains("attached `paperclip-instructions-"));
assert!(instructions.ends_with("skill under `instructions/`."));
assert!(!instructions.contains(local_root));
}
#[test]
fn rejects_oversized_remote_catalogs() {
let tools = (0..129)
packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs
+596 -12
View File
@@ -222,9 +222,12 @@ fn default_collaboration_mode_instructions() -> bool {
true
}
fn skillless_thread_config(include_collaboration_mode_instructions: bool) -> Value {
fn isolated_thread_config(
include_collaboration_mode_instructions: bool,
include_skill_instructions: bool,
) -> Value {
json!({
"skills.include_instructions": false,
"skills.include_instructions": include_skill_instructions,
"include_apps_instructions": false,
"include_collaboration_mode_instructions": include_collaboration_mode_instructions,
"features.apps": false,
@@ -261,6 +264,10 @@ pub struct LocalProviderConfig {
pub collaboration_mode: String,
#[serde(default = "default_collaboration_mode_instructions")]
pub include_collaboration_mode_instructions: bool,
#[serde(default)]
pub include_skill_instructions: bool,
#[serde(default)]
pub runtime_context: Option<Value>,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
@@ -273,6 +280,9 @@ pub struct ClaudeManagedProviderConfig {
pub environment_id: String,
pub beta_version: String,
pub max_session_list_cost_usd: f64,
pub instructions: String,
#[serde(default)]
pub runtime_context: Option<Value>,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
@@ -290,12 +300,18 @@ pub struct AwsAgentCoreProviderConfig {
pub memory_arn: String,
pub memory_id: String,
pub invocation_role_arn: String,
pub context_bucket: String,
pub context_prefix: String,
pub context_kms_key_arn: String,
pub qualification_revision: String,
pub event_expiry_days: u16,
pub max_estimated_session_cost_usd: f64,
pub max_iterations: u32,
pub max_output_tokens: u32,
pub timeout_seconds: u32,
pub instructions: String,
#[serde(default)]
pub runtime_context: Option<Value>,
}
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
@@ -318,6 +334,8 @@ pub struct AcpxProviderConfig {
pub cwd: String,
pub instructions: String,
pub permission_policy: String,
#[serde(default)]
pub runtime_context: Option<Value>,
}
#[derive(Clone, Debug, PartialEq)]
@@ -552,10 +570,7 @@ pub enum ProviderEvent {
item_id: Option<String>,
},
RuntimeRequest {
request_id: String,
request_kind: String,
title: String,
details: Value,
request: Value,
},
Exited,
}
@@ -569,12 +584,459 @@ pub struct CodexProvider {
active_provider_turn_id: Option<String>,
pending_messages: VecDeque<(Value, Option<u64>)>,
pending_tool_requests: BTreeMap<String, Value>,
pending_runtime_requests: BTreeMap<String, PendingCodexRuntimeRequest>,
collaboration_mode: String,
collaboration_mode_payload: Option<Value>,
trace: Option<ProviderTraceSink>,
last_trace_frame_id: Option<u64>,
}
#[derive(Clone, Debug)]
struct PendingCodexRuntimeRequest {
rpc_id: Value,
method: String,
provider_params: Value,
question_set: Value,
}
fn latest_active_provider_turn_id(thread_read: &Value) -> Option<String> {
thread_read
.pointer("/thread/turns")
.and_then(Value::as_array)
.and_then(|turns| {
turns.iter().rev().find_map(|turn| {
let status = turn.get("status").and_then(Value::as_str)?;
if matches!(status, "inProgress" | "in_progress" | "running" | "active") {
turn.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
} else {
None
}
})
})
}
fn runtime_request_kind(method: &str) -> Option<&'static str> {
match method {
"item/commandExecution/requestApproval" | "execCommandApproval" => Some("command_approval"),
"item/fileChange/requestApproval" | "applyPatchApproval" => Some("file_approval"),
"item/permissions/requestApproval" => Some("permission_approval"),
"item/tool/requestUserInput" | "tool/requestUserInput" => Some("user_input"),
"mcpServer/elicitation/request" => Some("elicitation"),
"paperclip/runtimeRequest" => Some("runtime"),
_ => None,
}
}
fn bounded_string(value: &str, max: usize) -> String {
value.chars().take(max).collect()
}
fn codex_question_set(method: &str, params: &Value) -> Option<Value> {
if matches!(
method,
"item/tool/requestUserInput" | "tool/requestUserInput"
) {
let raw_questions = params.get("questions")?.as_array()?;
if raw_questions.is_empty() {
return None;
}
let questions = raw_questions
.iter()
.take(64)
.enumerate()
.map(|(question_index, question)| {
let question_id = question
.get("id")
.and_then(Value::as_str)
.filter(|value| !value.is_empty())
.map(|value| bounded_string(value, 160))
.unwrap_or_else(|| format!("question-{}", question_index + 1));
let options = question
.get("options")
.and_then(Value::as_array)
.map(|values| {
values
.iter()
.take(128)
.enumerate()
.map(|(option_index, option)| {
let label = option
.get("label")
.or_else(|| option.get("value"))
.and_then(Value::as_str)
.or_else(|| option.as_str())
.map(|value| bounded_string(value, 1_000))
.unwrap_or_else(|| format!("Option {}", option_index + 1));
let mut normalized = json!({
"id": format!("option-{}", option_index + 1),
"label": label,
});
if let Some(description) =
option.get("description").and_then(Value::as_str)
{
normalized["description"] =
json!(bounded_string(description, 4_000));
}
normalized
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
let mut normalized = json!({
"id": question_id,
"prompt": bounded_string(
question.get("question")
.or_else(|| question.get("prompt"))
.and_then(Value::as_str)
.unwrap_or("Question"),
4_000,
),
"required": question.get("required").and_then(Value::as_bool).unwrap_or(true),
"answerMode": if options.is_empty() {
"text"
} else if question.get("multiSelect").and_then(Value::as_bool).unwrap_or(false)
|| question.get("multiple").and_then(Value::as_bool).unwrap_or(false) {
"multi_select"
} else {
"single_select"
},
});
if let Some(header) = question.get("header").and_then(Value::as_str) {
normalized["header"] = json!(bounded_string(header, 1_000));
}
if let Some(description) = question.get("description").and_then(Value::as_str) {
normalized["helpText"] = json!(bounded_string(description, 4_000));
}
if !options.is_empty() {
normalized["options"] = json!(options);
}
if question
.get("isOther")
.and_then(Value::as_bool)
.unwrap_or(false)
|| question
.get("allowOther")
.and_then(Value::as_bool)
.unwrap_or(false)
{
normalized["customAnswer"] = json!({
"enabled": true,
"label": "Other",
"placeholder": "Enter another answer",
});
}
normalized
})
.collect::<Vec<_>>();
return Some(json!({
"schema": "paperclip.question_set.v1",
"title": params.get("title").and_then(Value::as_str).unwrap_or("Codex needs your input"),
"submitLabel": params.get("submitLabel").and_then(Value::as_str).unwrap_or("Submit answers"),
"questions": questions,
}));
}
if method == "mcpServer/elicitation/request" {
let requested_schema = params
.get("requestedSchema")
.or_else(|| params.get("schema"))?;
let properties = requested_schema.get("properties")?.as_object()?;
if properties.is_empty() {
return None;
}
let required = requested_schema
.get("required")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let questions = properties
.iter()
.take(64)
.map(|(name, property)| {
let property_type = property.get("type").and_then(Value::as_str).unwrap_or("string");
let select_schema = if property_type == "array" {
property.get("items").unwrap_or(&Value::Null)
} else {
property
};
let option_values = select_schema.get("enum").and_then(Value::as_array).cloned()
.or_else(|| select_schema.get("oneOf").and_then(Value::as_array).map(|entries| entries.iter().map(|entry| entry.get("const").cloned().unwrap_or(Value::Null)).collect()))
.unwrap_or_default();
let options = if property_type == "boolean" {
vec![json!({"id":"true","label":"Yes"}), json!({"id":"false","label":"No"})]
} else {
option_values.iter().enumerate().map(|(index, value)| {
let one_of = select_schema.get("oneOf").and_then(Value::as_array).and_then(|items| items.get(index));
json!({
"id": format!("option-{}", index + 1),
"label": one_of.and_then(|item| item.get("title")).and_then(Value::as_str)
.map(str::to_owned).unwrap_or_else(|| value.as_str().map(str::to_owned).unwrap_or_else(|| value.to_string())),
"description": one_of.and_then(|item| item.get("description")).and_then(Value::as_str),
})
}).collect::<Vec<_>>()
};
let answer_mode = if property_type == "array" && !options.is_empty() {
"multi_select"
} else if !options.is_empty() {
"single_select"
} else {
"text"
};
let mut question = json!({
"id": name,
"header": property.get("title").and_then(Value::as_str),
"prompt": property.get("title").and_then(Value::as_str).unwrap_or(name),
"helpText": property.get("description").and_then(Value::as_str),
"required": required.iter().any(|entry| entry.as_str() == Some(name)),
"answerMode": answer_mode,
});
if !options.is_empty() {
question["options"] = json!(options);
} else {
question["textValidation"] = json!({
"inputType": match property_type { "number" => "number", "integer" => "integer", _ => "text" },
"minLength": property.get("minLength"),
"maxLength": property.get("maxLength"),
"minimum": property.get("minimum"),
"maximum": property.get("maximum"),
"pattern": property.get("pattern"),
});
}
question
})
.collect::<Vec<_>>();
return Some(json!({
"schema": "paperclip.question_set.v1",
"title": "A tool needs your input",
"description": params.get("message").and_then(Value::as_str),
"submitLabel": "Submit",
"questions": questions,
}));
}
None
}
fn canonical_runtime_request(
kind: ProviderKind,
request_id: &str,
method: &str,
params: &Value,
question_set: &Value,
) -> Value {
json!({
"schema": "paperclip.runtime_request.v2",
"requestKind": "runtime",
"requestId": request_id,
"type": "input",
"status": "pending",
"prompt": if method == "mcpServer/elicitation/request" { "A tool requests structured user input." } else { "Codex requests user input." },
"input": question_set,
"origin": {
"adapter": if kind == ProviderKind::Opencode { "opencode-app-server-proxy" } else { "codex-app-server" },
"provider": if kind == ProviderKind::Opencode { "opencode" } else { "codex" },
"method": method,
},
"itemId": params.get("itemId").and_then(Value::as_str).unwrap_or(request_id),
})
}
fn canonical_codex_answers(question_set: &Value, response: &Value) -> Value {
let answers = response.get("answers").and_then(Value::as_object);
let mut native = serde_json::Map::new();
for question in question_set
.get("questions")
.and_then(Value::as_array)
.into_iter()
.flatten()
{
let Some(question_id) = question.get("id").and_then(Value::as_str) else {
continue;
};
let Some(answer) = answers.and_then(|values| values.get(question_id)) else {
continue;
};
let mut values = Vec::new();
for selected in answer
.get("selectedOptionIds")
.and_then(Value::as_array)
.into_iter()
.flatten()
{
if let Some(option_id) = selected.as_str() {
if let Some(label) = question
.get("options")
.and_then(Value::as_array)
.into_iter()
.flatten()
.find(|option| option.get("id").and_then(Value::as_str) == Some(option_id))
.and_then(|option| option.get("label"))
.and_then(Value::as_str)
{
values.push(json!(label));
}
}
}
if let Some(value) = answer.get("text").and_then(Value::as_str) {
values.push(json!(value));
}
if let Some(value) = answer.get("customText").and_then(Value::as_str) {
values.push(json!(value));
}
native.insert(question_id.to_owned(), json!({"answers": values}));
}
Value::Object(native)
}
fn elicitation_option_value(schema: &Value, option_id: &str) -> Value {
if option_id == "true" {
return json!(true);
}
if option_id == "false" {
return json!(false);
}
let index = option_id
.strip_prefix("option-")
.and_then(|value| value.parse::<usize>().ok())
.and_then(|value| value.checked_sub(1));
if let Some(index) = index {
if let Some(value) = schema
.get("enum")
.and_then(Value::as_array)
.and_then(|values| values.get(index))
{
return value.clone();
}
if let Some(value) = schema
.get("oneOf")
.and_then(Value::as_array)
.and_then(|values| values.get(index))
.and_then(|entry| entry.get("const"))
{
return value.clone();
}
}
json!(option_id)
}
fn canonical_elicitation_content(pending: &PendingCodexRuntimeRequest, response: &Value) -> Value {
let schema = pending
.provider_params
.get("requestedSchema")
.or_else(|| pending.provider_params.get("schema"))
.unwrap_or(&Value::Null);
let properties = schema.get("properties").and_then(Value::as_object);
let answers = response.get("answers").and_then(Value::as_object);
let mut content = serde_json::Map::new();
for question in pending
.question_set
.get("questions")
.and_then(Value::as_array)
.into_iter()
.flatten()
{
let Some(question_id) = question.get("id").and_then(Value::as_str) else {
continue;
};
let Some(answer) = answers.and_then(|values| values.get(question_id)) else {
continue;
};
let property = properties
.and_then(|values| values.get(question_id))
.unwrap_or(&Value::Null);
let mode = question
.get("answerMode")
.and_then(Value::as_str)
.unwrap_or("text");
let value = if mode == "text" {
let raw = answer
.get("text")
.and_then(Value::as_str)
.unwrap_or_default();
match property.get("type").and_then(Value::as_str) {
Some("integer") => raw.parse::<i64>().map(Value::from).unwrap_or(Value::Null),
Some("number") => raw
.parse::<f64>()
.ok()
.and_then(serde_json::Number::from_f64)
.map(Value::Number)
.unwrap_or(Value::Null),
_ => json!(raw),
}
} else if mode == "multi_select" {
let item_schema = property.get("items").unwrap_or(&Value::Null);
Value::Array(
answer
.get("selectedOptionIds")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(|option_id| elicitation_option_value(item_schema, option_id))
.collect(),
)
} else if let Some(option_id) = answer
.get("selectedOptionIds")
.and_then(Value::as_array)
.and_then(|values| values.first())
.and_then(Value::as_str)
{
elicitation_option_value(property, option_id)
} else {
answer.get("customText").cloned().unwrap_or(Value::Null)
};
content.insert(question_id.to_owned(), value);
}
Value::Object(content)
}
fn codex_runtime_response(pending: &PendingCodexRuntimeRequest, resolution: &Value) -> Value {
let action = resolution
.get("action")
.and_then(Value::as_str)
.unwrap_or("cancel");
match pending.method.as_str() {
"item/tool/requestUserInput" | "tool/requestUserInput" => {
if action == "submit" {
if let Some(response) = resolution.get("response") {
json!({"answers": canonical_codex_answers(&pending.question_set, response)})
} else {
json!({"answers": resolution.get("answers").cloned().unwrap_or_else(|| json!({}))})
}
} else {
json!({"answers": {}})
}
}
"mcpServer/elicitation/request" => {
if action == "submit" {
json!({
"action": "accept",
"content": resolution.get("content").cloned().unwrap_or_else(|| {
resolution.get("response").map(|response| canonical_elicitation_content(pending, response)).unwrap_or_else(|| json!({}))
}),
"_meta": Value::Null,
})
} else {
json!({"action": action, "content": Value::Null, "_meta": Value::Null})
}
}
"paperclip/runtimeRequest" => json!({"resolution": resolution}),
"item/permissions/requestApproval" => json!({
"permissions": {},
"scope": if action == "accept_for_session" { "session" } else { "turn" },
}),
_ => json!({
"decision": match action {
"accept_for_session" => "acceptForSession",
"accept" => "accept",
"decline" => "decline",
_ => "cancel",
}
}),
}
}
impl CodexProvider {
pub fn process_id(&self) -> u32 {
self.process.id()
@@ -599,6 +1061,7 @@ impl CodexProvider {
active_provider_turn_id: None,
pending_messages: VecDeque::new(),
pending_tool_requests: BTreeMap::new(),
pending_runtime_requests: BTreeMap::new(),
collaboration_mode: config.collaboration_mode.clone(),
collaboration_mode_payload: None,
trace: ProviderTraceSink::from_environment(config.kind),
@@ -638,7 +1101,10 @@ impl CodexProvider {
"approvalPolicy": "never",
"permissions": permission_profile,
"runtimeWorkspaceRoots": [config.cwd],
"config": skillless_thread_config(config.include_collaboration_mode_instructions),
"config": isolated_thread_config(
config.include_collaboration_mode_instructions,
config.include_skill_instructions,
),
"baseInstructions": config.instructions,
"dynamicTools": dynamic_tools,
"experimentalRawEvents": true,
@@ -654,7 +1120,10 @@ impl CodexProvider {
"approvalPolicy": "never",
"permissions": permission_profile,
"runtimeWorkspaceRoots": [config.cwd],
"config": skillless_thread_config(config.include_collaboration_mode_instructions),
"config": isolated_thread_config(
config.include_collaboration_mode_instructions,
config.include_skill_instructions,
),
"baseInstructions": config.instructions,
"dynamicTools": dynamic_tools,
"experimentalRawEvents": true,
@@ -715,6 +1184,10 @@ impl CodexProvider {
})
.filter(|value| !value.is_empty())
.map(str::to_owned);
if resume_thread_id.is_some() {
let recovered = provider.read_thread()?;
provider.active_provider_turn_id = latest_active_provider_turn_id(&recovered);
}
Ok(provider)
}
@@ -839,6 +1312,8 @@ impl CodexProvider {
.map(|value| {
if value == "item/tool/call" {
"codex.tool_call"
} else if runtime_request_kind(value).is_some() {
"provider.runtime_request"
} else {
"codex.notification"
}
@@ -881,6 +1356,72 @@ impl CodexProvider {
input: params.get("arguments").cloned().unwrap_or(Value::Null),
}));
}
if let (Some(rpc_id), Some(method)) = (
message.get("id").cloned(),
message.get("method").and_then(Value::as_str),
) {
if let Some(request_kind) = runtime_request_kind(method) {
let params = message.get("params").cloned().unwrap_or(Value::Null);
if method != "paperclip/runtimeRequest"
&& params.get("threadId").and_then(Value::as_str)
!= Some(self.thread_id.as_str())
{
return Err(LocalRunnerError::invalid(
"provider runtime request named another thread",
));
}
let request_id = params
.pointer("/request/requestId")
.or_else(|| params.get("requestId"))
.and_then(Value::as_str)
.map(str::to_owned)
.unwrap_or_else(|| {
rpc_id
.as_str()
.map(str::to_owned)
.unwrap_or_else(|| rpc_id.to_string())
});
if self.pending_runtime_requests.contains_key(&request_id) {
return Err(LocalRunnerError::invalid(
"provider reused a pending runtime request id",
));
}
let question_set = codex_question_set(method, &params)
.or_else(|| params.pointer("/request/input").cloned())
.unwrap_or(Value::Null);
let request = params.get("request").cloned().unwrap_or_else(|| {
if !question_set.is_null() {
canonical_runtime_request(self.kind, &request_id, method, &params, &question_set)
} else {
json!({
"schema": "paperclip.runtime_request.v1",
"requestKind": "runtime",
"requestId": request_id,
"type": "permission",
"status": "pending",
"prompt": params.get("reason").or_else(|| params.get("message")).and_then(Value::as_str)
.unwrap_or("The provider requests runtime approval."),
"origin": {
"adapter": if self.kind == ProviderKind::Opencode { "opencode-app-server-proxy" } else { "codex-app-server" },
"provider": if self.kind == ProviderKind::Opencode { "opencode" } else { "codex" },
"method": method,
"kind": request_kind,
}
})
}
});
self.pending_runtime_requests.insert(
request_id,
PendingCodexRuntimeRequest {
rpc_id,
method: method.to_owned(),
provider_params: params,
question_set,
},
);
return Ok(Some(ProviderEvent::RuntimeRequest { request }));
}
}
if let Some(method) = message.get("method").and_then(Value::as_str) {
if method == "paperclip/runResult" {
let params = message.get("params").cloned().unwrap_or(Value::Null);
@@ -1111,6 +1652,24 @@ impl Provider for CodexProvider {
fn deliver_tool_result(&mut self, result: &ToolResult) -> Result<(), LocalRunnerError> {
CodexProvider::deliver_tool_result(self, result)
}
fn resolve_runtime_request(
&mut self,
request_id: &str,
_turn_id: &str,
resolution: &Value,
) -> Result<(), LocalRunnerError> {
let pending = self
.pending_runtime_requests
.get(request_id)
.cloned()
.ok_or_else(|| {
LocalRunnerError::invalid("runtime request has no pending JSON-RPC request")
})?;
let result = codex_runtime_response(&pending, resolution);
self.send_frame(&json!({"id": pending.rpc_id, "result": result}))?;
self.pending_runtime_requests.remove(request_id);
Ok(())
}
fn shutdown(&mut self) -> Result<(), LocalRunnerError> {
CodexProvider::shutdown(self)
}
@@ -1120,6 +1679,27 @@ impl Provider for CodexProvider {
mod provider_trace_tests {
use super::*;
#[test]
fn recovers_the_latest_active_codex_turn_from_thread_read() {
assert_eq!(
latest_active_provider_turn_id(&json!({
"thread": {
"turns": [
{"id": "turn-complete", "status": "completed"},
{"id": "turn-active", "status": "inProgress"}
]
}
})),
Some("turn-active".to_owned())
);
assert_eq!(
latest_active_provider_turn_id(&json!({
"thread": {"turns": [{"id": "turn-complete", "status": "completed"}]}
})),
None
);
}
#[test]
fn collaboration_instructions_default_on_and_allow_explicit_opt_out() {
let default_config: LocalProviderConfig = serde_json::from_value(json!({
@@ -1134,8 +1714,10 @@ mod provider_trace_tests {
.unwrap();
assert!(default_config.include_collaboration_mode_instructions);
assert_eq!(
skillless_thread_config(default_config.include_collaboration_mode_instructions)
["include_collaboration_mode_instructions"],
isolated_thread_config(
default_config.include_collaboration_mode_instructions,
default_config.include_skill_instructions,
)["include_collaboration_mode_instructions"],
json!(true)
);
@@ -1152,8 +1734,10 @@ mod provider_trace_tests {
.unwrap();
assert!(!opt_out_config.include_collaboration_mode_instructions);
assert_eq!(
skillless_thread_config(opt_out_config.include_collaboration_mode_instructions)
["include_collaboration_mode_instructions"],
isolated_thread_config(
opt_out_config.include_collaboration_mode_instructions,
opt_out_config.include_skill_instructions,
)["include_collaboration_mode_instructions"],
json!(false)
);
}
packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs
+250 -57
View File
@@ -804,23 +804,21 @@ fn process_command_and_provider(
let resolution = command
.pointer("/payload/resolution")
.unwrap_or(&command["payload"]);
if matches!(
state.provider_config,
Some(crate::codex_provider::ProviderConfig::Acpx(_))
) {
{
let pending = state
.pending_provider_runtime_requests
.get(request_id)
.ok_or_else(|| {
DurableRunnerError::invalid(
"request.resolve named a stale or expired ACPX permission request",
"request.resolve named a stale or expired runtime request",
)
})?;
if pending.turn_id != turn_id {
return Err(DurableRunnerError::invalid(
"request.resolve named a stale ACPX permission turn",
"request.resolve named a stale runtime request turn",
));
}
validate_runtime_request_resolution(&pending.request, resolution)?;
}
provider
.as_mut()
@@ -1039,7 +1037,7 @@ fn poll_provider(
Ok(None) => break,
Err(error) => {
let detail = error.to_string();
expire_acpx_permission_requests_after_provider_loss(state, config)?;
expire_provider_runtime_requests_after_provider_loss(state, config)?;
persist_provider_failure(state, store, &detail)?;
return Err(DurableRunnerError::invalid(format!(
"provider failed: {detail}"
@@ -1048,7 +1046,7 @@ fn poll_provider(
};
let trace_frame_id = runtime.take_provider_trace_frame_id();
let trace_first_source_seq = state.next_source_seq;
let (trace_rule_id, trace_disposition, trace_dropped_fields, trace_reason) = match &event {
let (mut trace_rule_id, mut trace_disposition, trace_dropped_fields, mut trace_reason) = match &event {
crate::codex_provider::ProviderEvent::ToolCall { .. } => (
"provider.tool_call",
"mapped",
@@ -1100,12 +1098,71 @@ fn poll_provider(
.provider_tool_bridge
.pending_calls()
.any(|pending| pending.call_id == call_id);
let call = state
.provider_tool_bridge
.begin_call(call_id, operation_id, input)
.map_err(|error| {
DurableRunnerError::invalid(format!("provider tool call rejected: {error}"))
})?;
let call = match state.provider_tool_bridge.begin_call(
call_id.clone(),
operation_id.clone(),
input,
) {
Ok(call) => call,
Err(error) => {
let reason = error.to_string();
runtime
.deliver_tool_result(&crate::provider_bridge::ToolResult {
call_id: call_id.clone(),
operation_id: operation_id.clone(),
result: json!({
"error": {
"code": "invalid_tool_call",
"message": reason,
"retryable": true,
}
}),
is_error: true,
})
.map_err(|delivery_error| {
DurableRunnerError::invalid(format!(
"provider tool rejection could not be delivered: {delivery_error}"
))
})?;
enqueue_event(
state,
config,
"harness.diagnostic",
1,
json!({
"operationId": operation_id,
"callId": call_id,
"code": "semantic_tool_denied",
"status": "failed",
"paperclipExecuted": false,
"reasons": ["schema_invalid_or_unauthorized_input"],
"summary": "Provider tool call was rejected before Paperclip execution.",
}),
Some(&state.item_id.clone()),
)?;
trace_rule_id = "provider.tool_call.rejected";
trace_disposition = "rejected";
trace_reason = "Provider tool call failed authorization or input validation and was returned to the provider for correction";
store.save(state)?;
if let Some(frame_id) = trace_frame_id {
let emitted_event_ids = (trace_first_source_seq..state.next_source_seq)
.map(|source_seq| {
format!("event_{}_{source_seq:06}", state.runner_instance_id)
})
.collect::<Vec<_>>();
runtime.record_provider_trace_interpretation(
frame_id,
"rust_durable_normalization",
trace_rule_id,
trace_disposition,
emitted_event_ids,
trace_dropped_fields,
trace_reason,
);
}
continue;
}
};
if pending_before_resume {
enqueue_event(
state,
@@ -1247,13 +1304,14 @@ fn poll_provider(
)?;
store.save(state)?;
}
crate::codex_provider::ProviderEvent::RuntimeRequest {
request_id,
request_kind,
title,
details,
} => {
crate::codex_provider::ProviderEvent::RuntimeRequest { request } => {
state.last_activity_at_unix_ms = current_unix_ms()?;
let request_id = request.get("requestId").and_then(Value::as_str)
.filter(|value| !value.is_empty())
.ok_or_else(|| DurableRunnerError::invalid("provider runtime request omitted requestId"))?
.to_owned();
let request_kind = request.pointer("/origin/kind").and_then(Value::as_str)
.unwrap_or("runtime").to_owned();
if state.pending_provider_runtime_requests.contains_key(&request_id) {
return Err(DurableRunnerError::invalid(
"provider reused a pending runtime request id",
@@ -1266,6 +1324,7 @@ fn poll_provider(
turn_id: state.turn_id.clone(),
request_kind: request_kind.clone(),
created_at_unix_ms: state.last_activity_at_unix_ms,
request: request.clone(),
},
);
state.lifecycle = "waiting_input".to_owned();
@@ -1274,24 +1333,13 @@ fn poll_provider(
config,
"runtime_request.created",
0,
json!({
"request": {
"schema": "paperclip.runtime_request.v1",
"requestKind": request_kind,
"requestId": request_id,
"type": "permission",
"status": "pending",
"prompt": title,
"turnId": state.turn_id,
"details": details,
}
}),
json!({ "request": request }),
Some(&state.item_id.clone()),
)?;
store.save(state)?;
}
crate::codex_provider::ProviderEvent::Exited => {
expire_acpx_permission_requests_after_provider_loss(state, config)?;
expire_provider_runtime_requests_after_provider_loss(state, config)?;
persist_provider_failure(state, store, "local provider exited unexpectedly")?;
return Err(DurableRunnerError::invalid(
"native_runner_process_exited: local provider exited unexpectedly",
@@ -1422,17 +1470,156 @@ fn should_suspend_for_idle_at(
Ok(now_unix_ms.saturating_sub(state.last_activity_at_unix_ms) >= timeout_ms)
}
fn expire_acpx_permission_requests_after_provider_loss(
fn validate_runtime_request_resolution(
request: &Value,
resolution: &Value,
) -> Result<(), DurableRunnerError> {
let action = resolution.get("action").and_then(Value::as_str).ok_or_else(|| {
DurableRunnerError::invalid("runtime request resolution.action is required")
})?;
if !matches!(action, "accept" | "accept_for_session" | "decline" | "cancel" | "submit") {
return Err(DurableRunnerError::invalid("unsupported runtime request resolution action"));
}
if request.get("schema").and_then(Value::as_str) != Some("paperclip.runtime_request.v2") {
return Ok(());
}
if action != "submit" {
if matches!(action, "accept" | "accept_for_session") {
return Err(DurableRunnerError::invalid(
"structured input requires submit, decline, or cancel",
));
}
return Ok(());
}
let response = resolution.get("response").ok_or_else(|| {
DurableRunnerError::invalid("structured input submission requires response")
})?;
if response.get("schema").and_then(Value::as_str) != Some("paperclip.question_response.v1") {
return Err(DurableRunnerError::invalid(
"structured input response must use paperclip.question_response.v1",
));
}
let answers = response.get("answers").and_then(Value::as_object).ok_or_else(|| {
DurableRunnerError::invalid("structured input response.answers must be an object")
})?;
let questions = request.pointer("/input/questions").and_then(Value::as_array).ok_or_else(|| {
DurableRunnerError::invalid("persisted runtime request has no question set")
})?;
for answer_id in answers.keys() {
if !questions.iter().any(|question| question.get("id").and_then(Value::as_str) == Some(answer_id)) {
return Err(DurableRunnerError::invalid(format!(
"structured input response named unknown question {answer_id}"
)));
}
}
for question in questions {
let question_id = question.get("id").and_then(Value::as_str).ok_or_else(|| {
DurableRunnerError::invalid("persisted question omitted id")
})?;
let answer = answers.get(question_id);
if answer.is_none() {
if question.get("required").and_then(Value::as_bool).unwrap_or(false) {
return Err(DurableRunnerError::invalid(format!(
"structured input response omitted required question {question_id}"
)));
}
continue;
}
let answer = answer.and_then(Value::as_object).ok_or_else(|| {
DurableRunnerError::invalid(format!("answer {question_id} must be an object"))
})?;
let selected = answer.get("selectedOptionIds").and_then(Value::as_array).cloned().unwrap_or_default();
if selected.iter().any(|value| value.as_str().is_none()) {
return Err(DurableRunnerError::invalid(format!(
"answer {question_id} selectedOptionIds must contain strings"
)));
}
let mode = question.get("answerMode").and_then(Value::as_str).unwrap_or("");
let text = answer.get("text").and_then(Value::as_str);
let custom = answer.get("customText").and_then(Value::as_str);
if mode == "text" {
if !selected.is_empty() || custom.is_some() {
return Err(DurableRunnerError::invalid(format!(
"text answer {question_id} carries select fields"
)));
}
} else {
if text.is_some() {
return Err(DurableRunnerError::invalid(format!(
"select answer {question_id} carries text"
)));
}
if mode == "single_select" && selected.len() > 1 {
return Err(DurableRunnerError::invalid(format!(
"single-select answer {question_id} selected multiple options"
)));
}
let options = question.get("options").and_then(Value::as_array).cloned().unwrap_or_default();
for option_id in selected.iter().filter_map(Value::as_str) {
if !options.iter().any(|option| option.get("id").and_then(Value::as_str) == Some(option_id)) {
return Err(DurableRunnerError::invalid(format!(
"answer {question_id} selected unknown option {option_id}"
)));
}
}
if custom.is_some() && question.pointer("/customAnswer/enabled").and_then(Value::as_bool) != Some(true) {
return Err(DurableRunnerError::invalid(format!(
"answer {question_id} used a disabled custom answer"
)));
}
}
let has_value = !selected.is_empty()
|| text.is_some_and(|value| !value.trim().is_empty())
|| custom.is_some_and(|value| !value.trim().is_empty());
if question.get("required").and_then(Value::as_bool).unwrap_or(false) && !has_value {
return Err(DurableRunnerError::invalid(format!(
"answer {question_id} is required"
)));
}
let bounded = if mode == "text" { text } else { custom };
if let Some(value) = bounded {
let validation = question.get("textValidation").unwrap_or(&Value::Null);
if let Some(minimum) = validation.get("minLength").and_then(Value::as_u64) {
if value.chars().count() < minimum as usize {
return Err(DurableRunnerError::invalid(format!("answer {question_id} is too short")));
}
}
if let Some(maximum) = validation.get("maxLength").and_then(Value::as_u64) {
if value.chars().count() > maximum as usize {
return Err(DurableRunnerError::invalid(format!("answer {question_id} is too long")));
}
}
if let Some(input_type) = validation.get("inputType").and_then(Value::as_str) {
if matches!(input_type, "number" | "integer") {
let numeric = value.parse::<f64>().map_err(|_| DurableRunnerError::invalid(format!(
"answer {question_id} must be a valid {input_type}"
)))?;
if input_type == "integer" && numeric.fract() != 0.0 {
return Err(DurableRunnerError::invalid(format!("answer {question_id} must be an integer")));
}
if validation.get("minimum").and_then(Value::as_f64).is_some_and(|minimum| numeric < minimum)
|| validation.get("maximum").and_then(Value::as_f64).is_some_and(|maximum| numeric > maximum)
{
return Err(DurableRunnerError::invalid(format!("answer {question_id} is outside its numeric bounds")));
}
}
}
}
}
Ok(())
}
fn expire_provider_runtime_requests_after_provider_loss(
state: &mut DurableRunnerState,
config: &DurableRunnerConfig,
) -> Result<(), DurableRunnerError> {
if !matches!(
state.provider_config,
Some(crate::codex_provider::ProviderConfig::Acpx(_))
) || state.pending_provider_runtime_requests.is_empty()
{
if state.pending_provider_runtime_requests.is_empty() {
return Ok(());
}
let is_acpx = matches!(
state.provider_config,
Some(crate::codex_provider::ProviderConfig::Acpx(_))
);
let pending = std::mem::take(&mut state.pending_provider_runtime_requests);
for request in pending.into_values() {
enqueue_event(
@@ -1444,28 +1631,31 @@ fn expire_acpx_permission_requests_after_provider_loss(
"requestId": request.request_id,
"turnId": request.turn_id,
"requestKind": request.request_kind,
"reason": "acpx_provider_process_lost",
"reason": "provider_process_lost",
"replayAllowed": false,
"request": request.request,
}),
Some(&state.item_id.clone()),
)?;
}
if is_acpx {
state.active_turn = false;
enqueue_event(
state,
config,
"turn.failed",
0,
json!({
"turn": { "id": state.turn_id, "status": "failed" },
"error": {
"code": "acpx_permission_transport_lost",
"message": "The ACP process exited while runtime input was pending; the request will not be replayed against the dead provider.",
"recoverable": true,
}
}),
Some(&state.item_id.clone()),
)?;
}
state.active_turn = false;
enqueue_event(
state,
config,
"turn.failed",
0,
json!({
"turn": { "id": state.turn_id, "status": "failed" },
"error": {
"code": "acpx_permission_transport_lost",
"message": "The ACP process exited while permission was pending; the prompt and approval will not be replayed.",
"recoverable": true,
}
}),
Some(&state.item_id.clone()),
)?;
Ok(())
}
@@ -1769,6 +1959,7 @@ mod tests {
cwd: "/tmp/workspace".to_owned(),
instructions: "safe".to_owned(),
permission_policy: "interactive".to_owned(),
runtime_context: None,
},
));
state.lifecycle = "waiting_input".to_owned();
@@ -1808,6 +1999,7 @@ mod tests {
cwd: "/tmp/workspace".to_owned(),
instructions: "safe".to_owned(),
permission_policy: "interactive".to_owned(),
runtime_context: None,
},
));
state.active_turn = true;
@@ -1819,10 +2011,11 @@ mod tests {
turn_id: state.turn_id.clone(),
request_kind: "file_approval".to_owned(),
created_at_unix_ms: 1,
request: Value::Null,
},
);
expire_acpx_permission_requests_after_provider_loss(&mut state, &runner_config).unwrap();
expire_provider_runtime_requests_after_provider_loss(&mut state, &runner_config).unwrap();
assert!(state.pending_provider_runtime_requests.is_empty());
assert!(!state.active_turn);
packages/paperclip-runner/runner/crates/runner-core/src/durable/state.rs
+2
View File
@@ -163,6 +163,8 @@ pub struct PendingProviderRuntimeRequest {
pub turn_id: String,
pub request_kind: String,
pub created_at_unix_ms: u64,
#[serde(default)]
pub request: Value,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
packages/paperclip-runner/runner/crates/runner-core/src/process_supervisor.rs
+4
View File
@@ -237,6 +237,10 @@ impl SupervisedProcess {
"PAPERCLIP_RUNNER_INSTANCE_ID",
"PAPERCLIP_RUN_ID",
"PAPERCLIP_NORMALIZED_SESSION_ID",
"PAPERCLIP_NATIVE_MCP_NAME",
"PAPERCLIP_NATIVE_MCP_URL",
"PAPERCLIP_NATIVE_MCP_TOKEN",
"PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH",
] {
if let Some(value) = std::env::var_os(key) {
command.env(key, value);
packages/paperclip-runner/runner/crates/runner-core/src/provider_events.rs
+21 -2
View File
@@ -70,11 +70,16 @@ pub fn canonical_provider_events(method: &str, params: &Value) -> Vec<(String, V
result.push((event.to_owned(), payload, item_id.to_owned()))
};
if (method == "item/started" || complete) && kind == "plan" || method == "turn/plan/updated" {
let steps = params.get("plan").and_then(Value::as_array).map(|values| values.iter().take(256).enumerate().map(|(index, value)| json!({"stepId": format!("step-{}", index + 1), "body": text(value.get("step")).chars().take(4000).collect::<String>(), "status": match text(value.get("status")) { "inProgress" | "in_progress" => "in_progress", "completed" => "completed", "blocked" => "blocked", _ => "pending" }})).collect()).unwrap_or_else(Vec::new);
let steps: Vec<Value> = params.get("plan").and_then(Value::as_array).map(|values| values.iter().take(256).enumerate().map(|(index, value)| json!({"stepId": format!("step-{}", index + 1), "body": text(value.get("step")).chars().take(4000).collect::<String>(), "status": match text(value.get("status")) { "inProgress" | "in_progress" => "in_progress", "completed" => "completed", "blocked" => "blocked", _ => "pending" }})).collect()).unwrap_or_else(Vec::new);
let plan_complete = complete
|| (!steps.is_empty()
&& steps
.iter()
.all(|step| text(step.get("status")) == "completed"));
push(
&mut result,
"plan.updated",
json!({"schema":"paperclip.plan.updated.v1","planId":item_id,"revision":params.get("revision").and_then(Value::as_u64).unwrap_or(1),"explanation":params.get("explanation").and_then(Value::as_str),"steps":steps,"complete":complete,"syncStatus":if complete {"pending"} else {"streaming"},"documentRevision":Value::Null}),
json!({"schema":"paperclip.plan.updated.v1","planId":item_id,"revision":params.get("revision").and_then(Value::as_u64).unwrap_or(1),"explanation":params.get("explanation").and_then(Value::as_str),"steps":steps,"complete":plan_complete,"syncStatus":if plan_complete {"pending"} else {"streaming"},"documentRevision":Value::Null}),
&item_id,
);
} else if (method == "item/started" || complete)
@@ -388,4 +393,18 @@ mod tests {
assert_eq!(events[0].1["sources"][0]["sourceId"], "source-1");
assert_eq!(events[0].1["sources"][0]["url"], "https://example.com/prp");
}
#[test]
fn completed_turn_plan_is_terminal_without_an_item_completion() {
let events = canonical_provider_events(
"turn/plan/updated",
&json!({"turnId":"turn-1","plan":[
{"step":"Inspect","status":"completed"},
{"step":"Implement","status":"completed"}
]}),
);
assert_eq!(events[0].0, "plan.updated");
assert_eq!(events[0].1["complete"], true);
assert_eq!(events[0].1["syncStatus"], "pending");
}
}
packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs
+78
View File
@@ -29,6 +29,8 @@ fn codex_plan_mode_is_qualified_and_selected_on_turn_start() {
instructions: "Author a plan without editing files.".to_owned(),
collaboration_mode: "plan".to_owned(),
include_collaboration_mode_instructions: true,
include_skill_instructions: false,
runtime_context: None,
},
vec![test_tool()].into_iter(),
None,
@@ -50,6 +52,8 @@ fn codex_dynamic_tool_round_trips_through_the_rust_provider_boundary() {
instructions: "Use the authorized Paperclip tools.".to_owned(),
collaboration_mode: "default".to_owned(),
include_collaboration_mode_instructions: true,
include_skill_instructions: false,
runtime_context: None,
},
vec![AuthorizedTool {
operation_id: "get_task_context".to_owned(),
@@ -115,6 +119,74 @@ fn codex_dynamic_tool_round_trips_through_the_rust_provider_boundary() {
}
}
#[test]
fn dot_185_runtime_question_round_trips_through_the_rust_provider_boundary() {
let mut provider = CodexProvider::start(
&LocalProviderConfig {
kind: ProviderKind::Codex,
command: PathBuf::from(env!("CARGO_BIN_EXE_fake-codex-app-server")),
args: vec!["--runtime-question".to_owned()],
cwd: "/tmp".to_owned(),
model: None,
instructions: "Ask for deployment input.".to_owned(),
collaboration_mode: "default".to_owned(),
include_collaboration_mode_instructions: true,
include_skill_instructions: false,
runtime_context: None,
},
vec![test_tool()].into_iter(),
None,
)
.unwrap();
provider
.start_turn("Ask the deployment questions.", "/tmp")
.unwrap();
let deadline = Instant::now() + Duration::from_secs(2);
loop {
assert!(
Instant::now() < deadline,
"fake Codex did not request runtime input"
);
if let Some(ProviderEvent::RuntimeRequest { request }) = provider.poll().unwrap() {
assert_eq!(request["schema"], "paperclip.runtime_request.v2");
assert_eq!(request["requestKind"], "runtime");
assert_eq!(request["type"], "input");
assert_eq!(request["input"]["questions"].as_array().unwrap().len(), 3);
Provider::resolve_runtime_request(
&mut provider,
"rpc-question-1",
"paperclip-turn",
&json!({
"action": "submit",
"response": {
"schema": "paperclip.question_response.v1",
"answers": {
"environment": {"selectedOptionIds": ["option-1"]},
"regions": {"selectedOptionIds": ["option-1", "option-2"]},
"notes": {"text": "Ship during the maintenance window."}
}
}
}),
)
.unwrap();
break;
}
}
let deadline = Instant::now() + Duration::from_secs(2);
loop {
assert!(
Instant::now() < deadline,
"fake Codex did not finish after input"
);
if let Some(ProviderEvent::Notification { method, .. }) = provider.poll().unwrap() {
if method == "turn/completed" {
break;
}
}
}
provider.shutdown().unwrap();
}
#[test]
fn provider_contract_preserves_the_opencode_tag() {
let mut provider = CodexProvider::start(
@@ -127,6 +199,8 @@ fn provider_contract_preserves_the_opencode_tag() {
instructions: "Use the authorized Paperclip tools.".to_owned(),
collaboration_mode: "default".to_owned(),
include_collaboration_mode_instructions: false,
include_skill_instructions: false,
runtime_context: None,
},
vec![AuthorizedTool {
operation_id: "get_task_context".to_owned(),
@@ -155,6 +229,8 @@ fn image_like_prompt_stays_skillless_and_accepts_a_valid_one_point_two_megabyte_
instructions: "Use only the authorized Paperclip tools.".to_owned(),
collaboration_mode: "default".to_owned(),
include_collaboration_mode_instructions: true,
include_skill_instructions: false,
runtime_context: None,
},
vec![AuthorizedTool {
operation_id: "get_task_context".to_owned(),
@@ -215,6 +291,8 @@ fn codex_provider_rejects_an_event_above_the_four_megabyte_hard_limit() {
instructions: "Use only the authorized Paperclip tools.".to_owned(),
collaboration_mode: "default".to_owned(),
include_collaboration_mode_instructions: true,
include_skill_instructions: false,
runtime_context: None,
},
vec![AuthorizedTool {
operation_id: "get_task_context".to_owned(),
packages/paperclip-runner/scripts/check-runner-workflow-traceability.mjs
+14
View File
@@ -0,0 +1,14 @@
import { access, readFile } from "node:fs/promises";
import { resolve } from "node:path";
const packageRoot = resolve(import.meta.dirname, "..");
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
const manifest = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8"));
const summary = evals.validateStressTraceabilityManifest(manifest);
for (const finding of manifest.findings) {
for (const path of finding.regressionTests) await access(resolve(packageRoot, path));
}
if (summary.coveredWorkflows !== evals.RUNNER_WORKFLOW_IDS.length) {
throw new Error(`stress traceability covers ${summary.coveredWorkflows}/${evals.RUNNER_WORKFLOW_IDS.length} workflows`);
}
process.stdout.write(`Runner stress traceability passed: ${summary.findings} findings, ${summary.coveredWorkflows} workflows, ${summary.exclusions} explicit exclusion.\n`);
packages/paperclip-runner/scripts/generate-protocol-schema-module.mjs
+3
View File
@@ -24,6 +24,9 @@ const schemaNames = [
"usage",
"stop-reason",
"terminal",
"question-set",
"question-response",
"question-adapter-fixture",
"request",
"result",
"event",
packages/paperclip-runner/scripts/run-runner-live-eval-schedule.mjs
+167
View File
@@ -0,0 +1,167 @@
import { createHash } from "node:crypto";
import { mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
const packageRoot = resolve(import.meta.dirname, "..");
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
const modeIndex = process.argv.indexOf("--mode");
const mode = modeIndex < 0 ? "nightly" : process.argv[modeIndex + 1];
const execute = process.argv.includes("--execute");
if (mode !== "nightly" && mode !== "chaos") throw new Error(`unsupported workflow eval schedule mode: ${mode}`);
const now = process.env.PAPERCLIP_EVAL_GENERATED_AT ?? new Date().toISOString();
const rotationDay = Number(process.env.PAPERCLIP_EVAL_ROTATION_DAY ?? Math.floor(Date.parse(now) / 86_400_000) % 7);
const seed = process.env.PAPERCLIP_EVAL_SCHEDULE_SEED ?? "runner-live-seven-day-v1";
const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows");
const historyDirectory = resolve(process.env.PAPERCLIP_EVAL_HISTORY_DIR ?? resolve(outputDirectory, "history"));
await mkdir(outputDirectory, { recursive: true });
function safeBundleId(schedule) {
const runnerBuild = process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version;
const identity = JSON.stringify({
runnerVersion: packageManifest.version,
runnerBuild,
promptPolicyId: "runner-live-workflow-v1",
seed: schedule.seed,
candidates: schedule.candidates.map(({ id, adapter, model, reasoningEffort }) => ({ id, adapter, model, reasoningEffort })),
});
return `runner-live-v2-${createHash("sha256").update(identity).digest("hex").slice(0, 16)}`;
}
function qualificationFailure(entry, candidate, evalCase) {
const missing = candidate.qualification.requiredEnvironment.filter((name) => !process.env[name]);
if (missing.length === 0) return null;
return evals.unavailableLiveRunnerWorkflowObservation({
entry,
candidate,
evalCase,
classification: "skipped",
code: "qualification_environment_missing",
category: "qualification",
retryable: false,
message: `Required credential reference unavailable: ${missing.join(", ")}`,
});
}
async function readCompatibleHistory(bundleId) {
const names = await readdir(historyDirectory).catch(() => []);
const reports = [];
for (const name of names.filter((value) => value.endsWith(".json")).sort()) {
try {
const report = JSON.parse(await readFile(resolve(historyDirectory, name), "utf8"));
if (report?.schema === evals.RUNNER_WORKFLOW_REPORT_SCHEMA && report?.bundle?.id === bundleId) reports.push(report);
} catch {
// A malformed historical artifact is ignored; it cannot affect the current candidate score.
}
}
return reports.slice(-7);
}
async function retainHistory(report) {
await mkdir(historyDirectory, { recursive: true });
const stamp = report.generatedAt.replaceAll(/[^0-9A-Za-z.-]/g, "-");
await writeFile(resolve(historyDirectory, `${stamp}-${report.bundle.id}.json`), `${JSON.stringify(report, null, 2)}\n`);
const expiry = Date.now() - 30 * 24 * 60 * 60 * 1_000;
for (const name of await readdir(historyDirectory)) {
if (!name.endsWith(".json")) continue;
const metadata = await stat(resolve(historyDirectory, name));
if (metadata.mtimeMs < expiry) await rm(resolve(historyDirectory, name));
}
}
if (mode === "nightly") {
const schedule = evals.buildRunnerLiveEvalSchedule({ seed, rotationDay, generatedAt: now });
const coverage = evals.runnerLiveScheduleCoverage(seed);
if (!Object.values(coverage).every(Boolean)) throw new Error(`live schedule coverage failed: ${JSON.stringify(coverage)}`);
await writeFile(resolve(outputDirectory, "nightly-schedule.json"), `${JSON.stringify({ schedule, coverage }, null, 2)}\n`);
if (!execute) {
process.stdout.write(`Runner live eval schedule ready: ${schedule.expectedExecutions} executions, rotation day ${schedule.rotationDay}. Use --execute to run providers.\n`);
process.exit(0);
}
const campaignCostLimit = Number(process.env.PAPERCLIP_EVAL_MAX_CAMPAIGN_COST_USD ?? "12");
let observedCampaignCost = 0;
const observations = await evals.executeRunnerLiveSchedule(
schedule,
async (entry, candidate) => {
const evalCase = evals.runnerWorkflowCase(entry.caseId);
const unavailable = qualificationFailure(entry, candidate, evalCase);
if (unavailable) return unavailable;
if (observedCampaignCost >= campaignCostLimit) {
return evals.unavailableLiveRunnerWorkflowObservation({
entry,
candidate,
evalCase,
classification: "skipped",
code: "campaign_cost_ceiling_reached",
category: "orchestration",
retryable: false,
message: `Campaign cost ceiling reached before this execution (${campaignCostLimit} USD)`,
});
}
const observation = await evals.executeLiveRunnerWorkflow({ entry, candidate, evalCase });
observedCampaignCost += observation.metrics.costUsd ?? 0;
return observation;
},
(entry, candidate, error) => evals.unavailableLiveRunnerWorkflowObservation({
entry,
candidate,
evalCase: evals.runnerWorkflowCase(entry.caseId),
classification: "infrastructure_failure",
code: error.code,
category: "provider",
retryable: error.retryable,
message: error.message,
}),
);
const bundleId = safeBundleId(schedule);
const results = observations.map((observation) => ({
scenarioId: observation.caseId,
candidateId: observation.candidateId,
observation,
scorecard: evals.scoreRunnerWorkflow(observation, { bundleId }),
}));
const providerVersions = Object.fromEntries(schedule.candidates.map((candidate) => [candidate.id, `${candidate.adapter}:${candidate.model}`]));
const report = evals.buildRunnerWorkflowEvalReport({
source: "live",
bundle: {
id: bundleId,
runnerVersion: packageManifest.version,
runnerBuild: process.env.PAPERCLIP_EVAL_RUNNER_BUILD ?? packageManifest.version,
promptPolicyId: "runner-live-workflow-v1",
providerVersions,
scheduleSeed: schedule.seed,
},
results,
generatedAt: now,
});
const history = await readCompatibleHistory(bundleId);
const comparison = history.length === 0
? null
: evals.compareRunnerWorkflowReports(report, history.at(-1));
const alerts = evals.runnerWorkflowAlerts({
current: report,
history,
baselineReady: process.env.PAPERCLIP_EVAL_BASELINE_READY === "true" && history.length >= 7,
});
await Promise.all([
writeFile(resolve(outputDirectory, "live-report.json"), `${JSON.stringify({ report, alerts, comparison }, null, 2)}\n`),
writeFile(resolve(outputDirectory, "live-report.md"), evals.renderRunnerWorkflowMarkdown(report)),
writeFile(resolve(outputDirectory, "live-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)),
writeFile(
resolve(outputDirectory, "github-live-summary.md"),
`${evals.renderRunnerWorkflowGitHubSummary(report, alerts)}\n## Previous compatible bundle\n\n${comparison === null ? "No compatible prior bundle is available." : `Pass-rate delta: ${comparison.passRateDelta}; overall delta: ${comparison.overallDelta}.`}\n`,
),
]);
await retainHistory(report);
process.stdout.write(`Runner live evals complete: ${report.aggregate.passed}/${report.aggregate.scoreable} scoreable passed; ${report.aggregate.infrastructureFailures} infrastructure, ${report.aggregate.skipped} skipped, ${alerts.length} alert(s).\n`);
} else {
const payload = {
schema: "paperclip.runner.chaos-eval-schedule.v1",
generatedAt: now,
seed,
scenarios: evals.RUNNER_CHAOS_SCENARIOS,
};
await writeFile(resolve(outputDirectory, "chaos-schedule.json"), `${JSON.stringify(payload, null, 2)}\n`);
process.stdout.write(`Runner chaos eval schedule ready: ${payload.scenarios.length} scenarios.\n`);
}
packages/paperclip-runner/scripts/run-runner-workflow-evals.mjs
+32
View File
@@ -0,0 +1,32 @@
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { resolve } from "node:path";
const packageRoot = resolve(import.meta.dirname, "..");
const evals = await import(resolve(packageRoot, "dist/eval/index.js"));
const packageManifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
const traceability = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8"));
const traceabilitySummary = evals.validateStressTraceabilityManifest(traceability);
const results = await evals.runDeterministicRunnerWorkflowMatrix({ bundleId: "runner-workflows-deterministic-v1" });
const report = evals.buildRunnerWorkflowEvalReport({
source: "deterministic",
bundle: {
id: "runner-workflows-deterministic-v1",
runnerVersion: packageManifest.version,
promptPolicyId: "stress-sanitized-v1",
providerVersions: { codex: "fixture-v1", opencode: "fixture-v1", acpx: "fixture-v1" },
},
results,
traceability: traceabilitySummary,
});
if (report.aggregate.passed !== report.aggregate.scoreable) {
throw new Error(`Runner workflow evals passed ${report.aggregate.passed}/${report.aggregate.scoreable}`);
}
const outputDirectory = resolve(packageRoot, ".paperclip-local/evals/workflows");
await mkdir(outputDirectory, { recursive: true });
await Promise.all([
writeFile(resolve(outputDirectory, "deterministic-report.json"), `${JSON.stringify(report, null, 2)}\n`),
writeFile(resolve(outputDirectory, "deterministic-report.md"), evals.renderRunnerWorkflowMarkdown(report)),
writeFile(resolve(outputDirectory, "deterministic-report.junit.xml"), evals.renderRunnerWorkflowJUnit(report)),
writeFile(resolve(outputDirectory, "github-summary.md"), evals.renderRunnerWorkflowGitHubSummary(report)),
]);
process.stdout.write(`Runner workflow evals passed: ${report.aggregate.passed}/${report.aggregate.scoreable}; coverage ${report.coverage.canonicalOperations} operations, ${report.coverage.capabilityCases} capability cases, ${report.coverage.workflows} workflows.\n`);
packages/paperclip-runner/spec/evals/schemas/eval-scorecard.v2.schema.json
+55
View File
@@ -0,0 +1,55 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/eval-scorecard.v2.schema.json",
"title": "EvalScorecardV2",
"type": "object",
"additionalProperties": false,
"required": ["schema", "bundleId", "caseId", "candidateId", "classification", "dimensions", "overall"],
"properties": {
"schema": { "const": "paperclip.runner.eval-scorecard.v2" },
"bundleId": { "type": "string", "minLength": 1 },
"caseId": { "type": "string", "minLength": 1 },
"candidateId": { "type": "string", "minLength": 1 },
"classification": { "enum": ["completed", "candidate_failure", "infrastructure_failure", "skipped"] },
"dimensions": {
"type": "object",
"additionalProperties": false,
"required": ["hard_invariants", "lifecycle_integrity", "semantic_outcome", "trajectory_restraint", "trace_completeness", "quality_efficiency", "continuation_integrity", "presentation_fidelity"],
"properties": {
"hard_invariants": { "$ref": "#/$defs/dimension" },
"lifecycle_integrity": { "$ref": "#/$defs/dimension" },
"semantic_outcome": { "$ref": "#/$defs/dimension" },
"trajectory_restraint": { "$ref": "#/$defs/dimension" },
"trace_completeness": { "$ref": "#/$defs/dimension" },
"quality_efficiency": { "$ref": "#/$defs/dimension" },
"continuation_integrity": { "$ref": "#/$defs/dimension" },
"presentation_fidelity": { "$ref": "#/$defs/dimension" }
}
},
"overall": {
"type": "object",
"additionalProperties": false,
"required": ["score", "gatePassed", "passed"],
"properties": {
"score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
"gatePassed": { "type": ["boolean", "null"] },
"passed": { "type": ["boolean", "null"] }
}
}
},
"$defs": {
"dimension": {
"type": "object",
"additionalProperties": false,
"required": ["dimension", "score", "passed", "gate", "weight", "reasons"],
"properties": {
"dimension": { "type": "string" },
"score": { "type": ["number", "null"], "minimum": 0, "maximum": 1 },
"passed": { "type": ["boolean", "null"] },
"gate": { "type": "boolean" },
"weight": { "type": "number", "minimum": 0 },
"reasons": { "type": "array", "items": { "type": "string" } }
}
}
}
}
packages/paperclip-runner/spec/evals/schemas/runner-workflow-eval-case.v1.schema.json
+35
View File
@@ -0,0 +1,35 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/runner-workflow-eval-case.v1.schema.json",
"title": "RunnerWorkflowEvalCase v1",
"type": "object",
"additionalProperties": false,
"required": ["schema", "id", "title", "version", "lanes", "tags", "coreLiveRepeat", "providers", "steps", "assertions"],
"properties": {
"schema": { "const": "paperclip.runner.workflow-eval-case.v1" },
"id": { "enum": ["final-response", "rich-activity", "verification-policy", "governed-interaction", "steering-causality", "planning-lifecycle", "review-lifecycle", "delegation-return", "completion-robustness", "restart-recovery", "cancellation-permissions", "trace-lineage"] },
"title": { "type": "string", "minLength": 1 },
"version": { "const": 1 },
"lanes": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "enum": ["deterministic", "live", "chaos"] } },
"tags": { "type": "array", "uniqueItems": true, "items": { "type": "string", "minLength": 1 } },
"coreLiveRepeat": { "type": "boolean" },
"providers": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "enum": ["codex", "opencode", "acpx"] } },
"steps": {
"type": "array",
"minItems": 1,
"items": {
"oneOf": [
{ "type": "object", "additionalProperties": false, "required": ["kind", "taskMode"], "properties": { "kind": { "const": "run_start" }, "taskMode": { "enum": ["ask", "execute", "plan"] } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "interaction"], "properties": { "kind": { "const": "interaction_response" }, "interaction": { "enum": ["questions", "suggest_tasks", "checkbox", "item_verdicts"] }, "partial": { "type": "boolean" } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "delivery"], "properties": { "kind": { "const": "steer" }, "delivery": { "enum": ["queued", "active_turn"] }, "duplicate": { "type": "boolean" } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "review_decision" }, "decision": { "enum": ["approve", "reject"] } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "decision"], "properties": { "kind": { "const": "permission_decision" }, "decision": { "enum": ["allow_once", "accept_for_session", "deny"] } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "actor"], "properties": { "kind": { "const": "cancel" }, "actor": { "enum": ["operator", "control_plane"] } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "phase"], "properties": { "kind": { "const": "process_restart" }, "phase": { "enum": ["provider_turn", "semantic_result", "finalization"] } } },
{ "type": "object", "additionalProperties": false, "required": ["kind", "childProvider"], "properties": { "kind": { "const": "child_completion" }, "childProvider": { "enum": ["codex", "opencode", "acpx"] } } }
]
}
},
"assertions": { "type": "object" }
}
}
packages/paperclip-runner/spec/evals/schemas/runner-workflow-observation.v1.schema.json
+79
View File
@@ -0,0 +1,79 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://paperclip.dev/schemas/runner-workflow-observation.v1.schema.json",
"title": "RunnerWorkflowObservation v1",
"type": "object",
"required": ["schema", "caseId", "candidateId", "provider", "classification", "base", "lifecycle", "continuation", "presentation", "traceLineage", "metrics", "observedPrpEventTypes", "artifactDigests"],
"properties": {
"schema": { "const": "paperclip.runner.workflow-observation.v1" },
"caseId": { "enum": ["final-response", "rich-activity", "verification-policy", "governed-interaction", "steering-causality", "planning-lifecycle", "review-lifecycle", "delegation-return", "completion-robustness", "restart-recovery", "cancellation-permissions", "trace-lineage"] },
"candidateId": { "type": "string", "minLength": 1 },
"provider": { "enum": ["codex", "opencode", "acpx"] },
"classification": { "enum": ["completed", "candidate_failure", "infrastructure_failure", "skipped"] },
"base": { "type": "object" },
"lifecycle": { "$ref": "#/$defs/evidence" },
"continuation": { "$ref": "#/$defs/evidence" },
"presentation": { "$ref": "#/$defs/evidence" },
"traceLineage": {
"type": "object",
"required": ["capture", "frameCount", "byteCount", "digestVerified", "ordered", "dispositions", "lineage"],
"properties": {
"capture": { "enum": ["on", "off"] },
"frameCount": { "type": "integer", "minimum": 0 },
"byteCount": { "type": "integer", "minimum": 0 },
"digestVerified": { "type": "boolean" },
"ordered": { "type": "boolean" },
"dispositions": { "type": "array", "items": { "type": "string" } },
"lineage": { "type": "array", "items": { "type": "string" } },
"traceRef": { "type": "string" }
}
},
"metrics": {
"type": "object",
"required": ["attempts", "toolCount"],
"properties": {
"timeToFirstVisibleProgressMs": { "type": "number", "minimum": 0 },
"settlementMs": { "type": "number", "minimum": 0 },
"attempts": { "type": "integer", "minimum": 0 },
"toolCount": { "type": "integer", "minimum": 0 },
"totalTokens": { "type": "number", "minimum": 0 },
"costUsd": { "type": "number", "minimum": 0 }
}
},
"observedPrpEventTypes": { "type": "array", "items": { "type": "string" } },
"artifactDigests": { "type": "array", "items": { "type": "string" } },
"failure": {
"type": "object",
"required": ["code", "category", "retryable", "message"],
"properties": {
"code": { "type": "string", "minLength": 1 },
"category": { "enum": ["candidate", "provider", "qualification", "orchestration"] },
"retryable": { "type": "boolean" },
"message": { "type": "string" }
}
}
},
"allOf": [
{
"if": { "properties": { "classification": { "enum": ["infrastructure_failure", "skipped"] } }, "required": ["classification"] },
"then": { "required": ["failure"] }
}
],
"$defs": {
"check": {
"type": "object",
"required": ["id", "passed"],
"properties": {
"id": { "type": "string", "minLength": 1 },
"passed": { "type": "boolean" },
"reason": { "type": "string" },
"evidenceIds": { "type": "array", "items": { "type": "string" } }
}
},
"evidence": {
"type": "object",
"required": ["checks"],
"properties": { "checks": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/check" } } }
}
}
}
packages/paperclip-runner/spec/evals/stress-workflow-traceability.json
+65
View File
@@ -0,0 +1,65 @@
{
"schema": "paperclip.runner.stress-eval-traceability.v1",
"campaign": "2026-08-23-paperclip-runner-stress-campaign",
"expectedFindings": 44,
"workflows": [
"final-response",
"rich-activity",
"verification-policy",
"governed-interaction",
"steering-causality",
"planning-lifecycle",
"review-lifecycle",
"delegation-return",
"completion-robustness",
"restart-recovery",
"cancellation-permissions",
"trace-lineage"
],
"findings": [
{ "id": "STRESS-001", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/contracts/native-execution.test.ts"] },
{ "id": "STRESS-002", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["src/backends/harness-driver-backend.test.ts"] },
{ "id": "STRESS-003", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/contracts/native-execution.test.ts"] },
{ "id": "STRESS-004", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["../../server/src/__tests__/heartbeat-workspace-session.test.ts"] },
{ "id": "STRESS-005", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/acpx/acpx-runtime-driver.test.ts"] },
{ "id": "STRESS-006", "classification": "workflow_eval", "workflowIds": ["final-response", "verification-policy"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] },
{ "id": "STRESS-007", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["src/protocol/result-normalization.test.ts"] },
{ "id": "STRESS-008", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["src/native-session-runtime.test.ts"] },
{ "id": "STRESS-009", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["src/drivers/opencode/mcp-bridge.test.ts"] },
{ "id": "STRESS-010", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/native-finalization-recovery.test.ts"] },
{ "id": "STRESS-011", "classification": "workflow_eval", "workflowIds": ["rich-activity"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] },
{ "id": "STRESS-012", "classification": "workflow_eval", "workflowIds": ["final-response"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] },
{ "id": "STRESS-013", "classification": "workflow_eval", "workflowIds": ["final-response", "restart-recovery"], "regressionTests": ["../../server/src/__tests__/heartbeat-run-summary.test.ts"] },
{ "id": "STRESS-014", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/issue-recovery-actions.test.ts"] },
{ "id": "STRESS-015", "classification": "workflow_eval", "workflowIds": ["steering-causality"], "regressionTests": ["../../ui/src/components/TaskChatThread.test.tsx"] },
{ "id": "STRESS-016", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../ui/src/components/IssueThreadInteractionCard.test.tsx"] },
{ "id": "STRESS-017", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["../../ui/src/components/task-chat/transcript-adapter.test.ts"] },
{ "id": "STRESS-018", "classification": "workflow_eval", "workflowIds": ["restart-recovery", "completion-robustness"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] },
{ "id": "STRESS-019", "classification": "workflow_eval", "workflowIds": ["restart-recovery", "governed-interaction"], "regressionTests": ["src/backends/harness-driver-backend.test.ts"] },
{ "id": "STRESS-020", "classification": "workflow_eval", "workflowIds": ["completion-robustness"], "regressionTests": ["../../server/src/__tests__/heartbeat-process-recovery.test.ts", "../../server/src/__tests__/heartbeat-comment-wake-batching.test.ts"] },
{ "id": "STRESS-021", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] },
{ "id": "STRESS-022", "classification": "workflow_eval", "workflowIds": ["governed-interaction", "restart-recovery"], "regressionTests": ["src/drivers/acpx/acpx-runtime-driver.test.ts"] },
{ "id": "STRESS-023", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] },
{ "id": "STRESS-024", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/acpx/pi-paperclip-extension.test.ts"] },
{ "id": "STRESS-025", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../ui/src/context/LiveUpdatesProvider.test.ts"] },
{ "id": "STRESS-026", "classification": "workflow_eval", "workflowIds": ["planning-lifecycle"], "regressionTests": ["../../server/src/__tests__/issue-thread-interaction-routes.test.ts"] },
{ "id": "STRESS-027", "classification": "workflow_eval", "workflowIds": ["delegation-return"], "regressionTests": ["../../server/src/__tests__/heartbeat-dependency-scheduling.test.ts"] },
{ "id": "STRESS-028", "classification": "workflow_eval", "workflowIds": ["delegation-return"], "regressionTests": ["../../server/src/services/native-runtime/native-session-resume.test.ts"] },
{ "id": "STRESS-029", "classification": "workflow_eval", "workflowIds": ["restart-recovery"], "regressionTests": ["../../server/src/__tests__/native-finalization-recovery.test.ts"] },
{ "id": "STRESS-030", "classification": "workflow_eval", "workflowIds": ["review-lifecycle"], "regressionTests": ["../../server/src/services/native-runtime/evidence-classifier.test.ts"] },
{ "id": "STRESS-031", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/__tests__/native-interaction-bridge.test.ts"] },
{ "id": "STRESS-032", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["../../server/src/services/native-runtime/native-session-executor.test.ts"] },
{ "id": "STRESS-033", "classification": "workflow_eval", "workflowIds": ["governed-interaction", "final-response"], "regressionTests": ["../../server/src/__tests__/heartbeat-run-summary.test.ts"] },
{ "id": "STRESS-034", "classification": "workflow_eval", "workflowIds": ["governed-interaction"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] },
{ "id": "STRESS-035", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions"], "regressionTests": ["src/drivers/opencode/opencode-server-driver.test.ts"] },
{ "id": "STRESS-036", "classification": "workflow_eval", "workflowIds": ["final-response"], "regressionTests": ["src/drivers/acpx/acpx-runtime-driver.test.ts"] },
{ "id": "STRESS-037", "classification": "workflow_eval", "workflowIds": ["cancellation-permissions", "completion-robustness"], "regressionTests": ["src/drivers/acpx/acpx-runtime-host.test.ts"] },
{ "id": "STRESS-038", "classification": "workflow_eval", "workflowIds": ["trace-lineage"], "regressionTests": ["../../server/src/__tests__/provider-trace-store.test.ts"] },
{ "id": "STRESS-039", "classification": "workflow_eval", "workflowIds": ["trace-lineage"], "regressionTests": ["src/drivers/acpx/acpx-runtime-host.test.ts"] },
{ "id": "STRESS-040", "classification": "explicit_exclusion", "workflowIds": [], "regressionTests": [], "reason": "Campaign control cleanup is an operational hygiene check, not candidate behavior." },
{ "id": "STRESS-041", "classification": "workflow_eval", "workflowIds": ["final-response", "completion-robustness"], "regressionTests": ["../../server/src/__tests__/heartbeat-running-followup.test.ts"] },
{ "id": "STRESS-042", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/openapi-routes.test.ts"], "reason": "Mounted-route documentation parity is deterministic API inventory." },
{ "id": "STRESS-043", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/server-startup-feedback-export.test.ts"], "reason": "The HTTP listener test double is ordinary startup integration behavior." },
{ "id": "STRESS-044", "classification": "regression_test", "workflowIds": [], "regressionTests": ["../../server/src/__tests__/company-skills-service.test.ts"], "reason": "Filesystem path portability is deterministic service behavior." }
]
}
packages/paperclip-runner/spec/fixtures/status-authority-sdk.json
Vendored
+10 -10
View File
@@ -27,7 +27,7 @@
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["premature_done_claim", "incomplete_evidence", "partial_progress", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "missing_required_test", "trigger": "runner_finalizer" },
"expected": { "runStatus": "succeeded", "statusAction": "in_progress", "reasonCode": "completion_evidence_incomplete", "requiredEffects": ["enqueue_continuation"], "forbiddenEffects": ["release_checkout_as_done"], "livePathKind": "continuation", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "in_review", "reasonCode": "external_verification_required", "requiredEffects": ["bind_reviewer"], "forbiddenEffects": ["release_checkout_as_done", "enqueue_continuation"], "livePathKind": "review", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -36,7 +36,7 @@
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["incomplete_evidence", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "missing_required_test", "trigger": "runner_finalizer", "fault": "continuation_insert_failure" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["enqueue_continuation", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "in_review", "reasonCode": "external_verification_required", "requiredEffects": ["bind_reviewer"], "forbiddenEffects": ["enqueue_continuation"], "livePathKind": "review", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -45,7 +45,7 @@
"covers": { "decisionRows": ["SD-03"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-06"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["atomic_liveness", "reconciliation"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "no_durable_continuation", "trigger": "runner_finalizer" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "prior_status_preserved_no_live_path", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "enqueue_continuation", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "prior_status_preserved_no_live_path", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "enqueue_continuation", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -54,7 +54,7 @@
"covers": { "decisionRows": ["SD-04"], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-01"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["required_review", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "needs_review", "nativeFinalization": "present", "completionState": "named_reviewer_required", "trigger": "runner_finalizer" },
"expected": { "runStatus": "succeeded", "statusAction": "in_review", "reasonCode": "completion_review_required", "requiredEffects": ["bind_reviewer", "notify_owner"], "forbiddenEffects": ["bind_blocker"], "livePathKind": "review", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 1 },
"expected": { "runStatus": "succeeded", "statusAction": "in_review", "reasonCode": "external_verification_required", "requiredEffects": ["bind_reviewer"], "forbiddenEffects": ["bind_blocker", "notify_owner"], "livePathKind": "review", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -72,7 +72,7 @@
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-02"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["required_review", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "needs_review", "nativeFinalization": "present", "completionState": "review_required", "trigger": "runner_finalizer", "fault": "reviewer_insert_failure" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["bind_reviewer", "notify_owner", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["bind_reviewer", "notify_owner", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -81,7 +81,7 @@
"covers": { "decisionRows": ["SD-05"], "terminalRows": ["TC-08"], "attentionRows": [], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": ["COMP-02"], "migrationRows": [] },
"tags": ["native_legacy_distinction", "reconciliation"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": null, "nativeFinalization": "invalid", "completionState": "safe_partial_parse", "trigger": "runner_finalizer" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "native_finalization_invalid", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["legacy_finalizer", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 1, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "native_finalization_invalid", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["legacy_finalizer", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 1, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -117,7 +117,7 @@
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": ["LIVE-04"], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["real_blocker", "atomic_liveness"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "blocked", "nativeFinalization": "present", "completionState": "task_wide_owner_action_bound", "trigger": "runner_finalizer", "fault": "blocker_insert_failure" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["bind_blocker", "notify_owner", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "side_effect_planning_failed", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["bind_blocker", "notify_owner", "status_version_increment"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -162,7 +162,7 @@
"covers": { "decisionRows": ["SD-13"], "terminalRows": [], "attentionRows": ["ATT-07", "ATT-12"], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["excessive_human_request", "repeated_question", "reconciliation"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "resolver_budget_exhausted", "trigger": "attention_candidate" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "attention_budget_exhausted", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["create_interaction", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "attention_budget_exhausted", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["create_interaction", "bind_blocker", "bind_reviewer"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 1, "maxNotificationCount": 0 },
"replay": { "attempts": 8, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -261,7 +261,7 @@
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": ["ATT-09"], "livenessRows": [], "reconciliationRows": [], "compatibilityRows": [], "migrationRows": [] },
"tags": ["excessive_human_request"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "yielded", "nativeFinalization": "present", "completionState": "cross_company_target", "trigger": "attention_candidate" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "result_schema_rejected", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["create_interaction", "notify_owner", "enqueue_continuation"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 0, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "result_schema_rejected", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["create_interaction", "notify_owner", "enqueue_continuation"], "livePathKind": "recovery", "preserveClaim": true, "nativeRecords": true, "decisionCount": 1, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 1, "maxDomainEffectsPerKey": 1 }
},
{
@@ -297,7 +297,7 @@
"covers": { "decisionRows": [], "terminalRows": [], "attentionRows": [], "livenessRows": [], "reconciliationRows": ["REC-03"], "compatibilityRows": [], "migrationRows": [] },
"tags": ["deterministic_replay"],
"given": { "priorIssueStatus": "in_progress", "turnTerminalState": "completed", "runTerminalState": "succeeded", "reportedWorkDisposition": "done", "nativeFinalization": "present", "completionState": "reused_id_changed_material", "trigger": "runner_finalizer" },
"expected": { "runStatus": "failed", "statusAction": "preserve", "reasonCode": "structured_result_replay_conflict", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "notify_owner", "enqueue_continuation"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"expected": { "runStatus": "succeeded", "statusAction": "preserve", "reasonCode": "structured_result_replay_conflict", "requiredEffects": ["record_finalization_error"], "forbiddenEffects": ["set_issue_status", "notify_owner", "enqueue_continuation"], "livePathKind": null, "preserveClaim": true, "nativeRecords": true, "decisionCount": 0, "maxWakeCount": 0, "maxNotificationCount": 0 },
"replay": { "attempts": 2, "sameDecisionDigest": true, "maxSemanticDecisions": 0, "maxDomainEffectsPerKey": 1 }
},
{
packages/paperclip-runner/spec/paperclip-agent-operation-groups.md
+2 -2
View File
@@ -116,7 +116,7 @@ Grant groups are documentation/exposure bundles, not additional authority. The o
| `block_task` | `always_agent_tool` | none | `standard`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:block_task` | `scenario` + `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus active-task state diff, work-assessment, and issue-status-decision events<br>catalog PRP status: `audit_pending` |
| `comment_on_approval` | `optional_agent_tool` | `governance:approvals:comment` | `standard`<br>`ask`<br>`planning`<br>`skill_test` | `governance` | `required` | no | `semantic_command:comment_on_approval` | `scenario` + `live`<br>`live_codex` | `unbound`<br>approval lifecycle plus governed-wait continuation and audit events<br>catalog PRP status: `audit_pending` |
| `control_workspace_service` | `optional_agent_tool` | `workspace:control` | `standard`<br>`skill_test` | `workspace_control` | `required` | no | `semantic_command:control_workspace_service` | `scenario` + `live`<br>`live_codex` | `unbound`<br>workspace service lifecycle event<br>catalog PRP status: `audit_pending` |
| `create_task` | `optional_agent_tool` | `delegation:tasks:create` | `standard`<br>`skill_test` | `company_write` | `required` | no | `semantic_command:create_task` | `scenario` + `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus company-entity state diff and audit record<br>catalog PRP status: `audit_pending` |
| `create_task` | `optional_agent_tool` | `delegation:tasks:create` | `standard`<br>`skill_test` | `company_write` | `required` | no | `semantic_command:create_task` | `scenario` + `live`<br>`live_codex` | `issues.decomposeAcceptedPlan`<br>semantic-operation item event plus company-entity state diff and audit record<br>catalog PRP status: `bound` |
| `decide_approval` | `optional_agent_tool` | `governance:approvals:decide` | `standard`<br>`skill_test`<br>roles: `board`<br>`approver`<br>`security` | `governance` | `required` | no | `semantic_command:decide_approval` | `scenario` + `live`<br>`live_codex` | `unbound`<br>approval lifecycle plus governed-wait continuation and audit events<br>catalog PRP status: `audit_pending` |
| `export_company` | `optional_agent_tool` | `portability:export` | `standard`<br>`skill_test` | `admin` | `required` | no | `mock_extension:portability.export` | `scenario`<br>`scenario_mock` | `unbound`<br>company admin/portability item event plus audit record<br>catalog PRP status: `audit_pending` |
| `finish_task` | `always_agent_tool` | none | `standard`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:finish_task` | `scenario` + `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus active-task state diff, work-assessment, and issue-status-decision events<br>catalog PRP status: `audit_pending` |
@@ -144,7 +144,7 @@ Grant groups are documentation/exposure bundles, not additional authority. The o
| `register_deliverable` | `always_agent_tool` | none | `standard`<br>`planning`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:register_deliverable` | `scenario` + `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus active-task state diff, work-assessment, and issue-status-decision events<br>catalog PRP status: `audit_pending` |
| `report_progress` | `always_agent_tool` | none | `standard`<br>`ask`<br>`planning`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:report_progress` | `scenario` + `live`<br>`live_codex` | `issueService.addComment + heartbeatRuns semanticToolReceipts`<br>real service mutation + idempotent replay test<br>catalog PRP status: `bound` |
| `request_approval` | `optional_agent_tool` | `governance:approvals:request` | `standard`<br>`skill_test` | `governance` | `required` | no | `semantic_command:request_approval` | `scenario` + `live`<br>`live_codex` | `unbound`<br>approval lifecycle plus governed-wait continuation and audit events<br>catalog PRP status: `audit_pending` |
| `request_human_input` | `always_agent_tool` | none | `standard`<br>`planning`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:request_human_input` | `scenario` + `live`<br>`live_codex` | `issueThreadInteractionService.create`<br>real structured interaction + continuation projection test<br>catalog PRP status: `bound` |
| `request_human_input` | `always_agent_tool` | none | `standard`<br>`planning`<br>`ask`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:request_human_input` | `scenario` + `live`<br>`live_codex` | `issueThreadInteractionService.create`<br>real structured interaction + continuation projection test<br>catalog PRP status: `bound` |
| `request_review` | `always_agent_tool` | none | `standard`<br>`skill_test` | `task_write` | `required` | no | `semantic_command:request_review` | `scenario` + `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus active-task state diff, work-assessment, and issue-status-decision events<br>catalog PRP status: `audit_pending` |
| `schedule_wake` | `optional_agent_tool` | `control_plane:wakes` | `standard`<br>`skill_test` | `task_write` | `required` | no | inline/no mapping | `live`<br>`live_codex` | `unbound`<br>semantic-operation item event plus active-task state diff, work-assessment, and issue-status-decision events<br>catalog PRP status: `audit_pending` |
| `search_tasks` | `optional_agent_tool` | `discovery:tasks:read` | `standard`<br>`ask`<br>`planning`<br>`skill_test` | `read` | `none` | no | `snapshot_read:company_tasks` | `scenario` + `live`<br>`live_codex` | `issueService.list`<br>shared PRP route + company scope test<br>catalog PRP status: `bound` |
packages/paperclip-runner/src/backends/harness-driver-backend.test.ts
+55
View File
@@ -20,6 +20,18 @@ const result: PrpStructuredRunResult = {
artifacts: [],
};
const providerIdentity = {
kind: "acpx" as const,
normalizedSessionId: "session-1",
acpxRecordId: "driver-1",
backendSessionId: "backend-1",
agentSessionId: "provider-1",
profileDigest: "sha256:profile",
workspaceDigest: "sha256:workspace",
requestedModel: "claude-sonnet-4-20250514",
effectiveModel: "claude-sonnet-4-20250514",
};
function prpEvent(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record<string, unknown>): PrpEvent {
return {
schema: "paperclip.prp.event.v1",
@@ -38,6 +50,8 @@ function prpEvent(sourceSeq: number, eventType: PrpEvent["eventType"], payload:
};
}
const runtimeResolutions: unknown[] = [];
class FakeHarnessSession implements HarnessSession {
ids() { return { driverSessionId: "driver-1", providerSessionId: "provider-1" }; }
async *events() {
@@ -45,11 +59,16 @@ class FakeHarnessSession implements HarnessSession {
yield prpEvent(2, "turn.completed", { status: "completed" });
}
async startTurn() { return { turnId: "turn-1" }; }
async resolveRuntimeRequest(input: unknown) {
runtimeResolutions.push(structuredClone(input));
}
async snapshot(): Promise<PersistedHarnessSession> {
return {
driverKind: "fake",
driverSessionId: "driver-1",
providerSessionId: "provider-1",
providerIdentity,
providerRecoveryPolicy: "allow_replacement_after_governed_wait",
semanticResult: { result, fingerprint: "fingerprint", turnId: "turn-1" },
lastSourceSequence: 2,
};
@@ -90,15 +109,19 @@ describe("HarnessDriverBackend", () => {
driverKind: "fake",
sessionId: "driver-1",
providerSessionId: "provider-1",
providerIdentity,
providerRecoveryPolicy: "allow_replacement_after_governed_wait",
});
});
it("passes the persisted harness driver kind through recovery", async () => {
let recoveredDriverKind: string | null = null;
let recoveredProviderIdentity: PersistedHarnessSession["providerIdentity"];
const recoveryDriver: HarnessDriver = {
...driver,
async recoverSession(snapshot) {
recoveredDriverKind = snapshot.driverKind;
recoveredProviderIdentity = snapshot.providerIdentity;
return { recovered: true, session: new FakeHarnessSession() };
},
};
@@ -108,9 +131,41 @@ describe("HarnessDriverBackend", () => {
driverKind: "fake",
sessionId: "driver-1",
providerSessionId: "provider-1",
providerIdentity,
providerRecoveryPolicy: "allow_replacement_after_governed_wait",
identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" },
});
expect(recovery.recovered).toBe(true);
expect(recoveredDriverKind).toBe("fake");
expect(recoveredProviderIdentity).toEqual(providerIdentity);
});
it("delegates native runtime-request resolutions to the harness session", async () => {
runtimeResolutions.length = 0;
const backend = new HarnessDriverBackend(driver);
const session = await backend.openSession({
identity: {
runId: "run-1",
sessionId: "session-1",
companyId: "company-1",
issueId: "issue-1",
agentId: "agent-1",
},
workingDirectory: "/workspace",
});
await session.resolveRuntimeRequest?.({
requestId: "permission-1",
turnId: "turn-1",
resolution: { action: "accept_for_session" },
});
expect(runtimeResolutions).toEqual([
{
requestId: "permission-1",
turnId: "turn-1",
resolution: { action: "accept_for_session" },
},
]);
});
});
packages/paperclip-runner/src/backends/harness-driver-backend.ts
+23
View File
@@ -55,6 +55,12 @@ export class HarnessDriverBackend implements NativeSessionBackend {
driverKind: snapshot.driverKind ?? snapshot.backendKind,
driverSessionId: snapshot.sessionId,
providerSessionId: snapshot.providerSessionId,
...(snapshot.providerIdentity === undefined
? {}
: { providerIdentity: structuredClone(snapshot.providerIdentity) }),
...(snapshot.providerRecoveryPolicy === undefined
? {}
: { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }),
runId: snapshot.identity.runId,
normalizedSessionId: snapshot.identity.sessionId,
activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null,
@@ -170,6 +176,17 @@ class HarnessNativeSession implements NativeSession {
}
}
resolveRuntimeRequest(input: {
requestId: string;
turnId: string;
resolution: Parameters<NonNullable<HarnessSession["resolveRuntimeRequest"]>>[0]["resolution"];
}) {
if (this.#session.resolveRuntimeRequest === undefined) {
throw new Error("native_runtime_request_resolution_unavailable");
}
return this.#session.resolveRuntimeRequest(input);
}
async result() {
const snapshot = await this.#session.snapshot();
if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) {
@@ -193,6 +210,12 @@ class HarnessNativeSession implements NativeSession {
sessionId: snapshot.driverSessionId,
identity: structuredClone(this.#input.identity),
providerSessionId: snapshot.providerSessionId,
...(snapshot.providerIdentity === undefined
? {}
: { providerIdentity: structuredClone(snapshot.providerIdentity) }),
...(snapshot.providerRecoveryPolicy === undefined
? {}
: { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }),
cursor:
snapshot.lastSourceSequence === undefined
? null
packages/paperclip-runner/src/backends/runtime-context.ts
+14 -2
View File
@@ -11,6 +11,18 @@ export function nativeSystemInstructions(input: NativeExecutionInput): string {
}
export function nativeTaskConstraints(input: NativeExecutionInput): string[] {
if (!("runtimeContext" in input)) return ["Do not discover or invoke skills.", "Do not call a control-plane API."];
return ["Use only the assigned skills and provider-native tools.", "Use Paperclip semantic tools for coordination and finalization."];
const finalResponseConstraint =
"Write the complete user-facing final response exactly once before invoking paperclip_finish or paperclip_block. Treat that semantic completion tool as the last action and do not emit a trailing acknowledgement.";
if (!("runtimeContext" in input)) {
return [
"Do not discover or invoke skills.",
"Do not call a control-plane API.",
finalResponseConstraint,
];
}
return [
"Use only the assigned skills and provider-native tools.",
"Use Paperclip semantic tools for coordination and finalization.",
finalResponseConstraint,
];
}
packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts
+13 -6
View File
@@ -1,14 +1,15 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { createInterface } from "node:readline";
import { isAbsolute, relative, resolve } from "node:path";
import type { AcpPermissionDecision, AcpRuntimeEvent, AcpRuntimeStatus } from "acpx/runtime";
import {
CODEX_BLOCK_TOOL_NAME,
CODEX_COMPLETION_TOOL_NAME,
} from "../contracts/codex.js";
PRP_BLOCK_TOOL_NAME,
PRP_COMPLETION_TOOL_NAME,
} from "../contracts/completion-result.js";
import { AcpxRuntimeHost } from "../drivers/acpx/acpx-runtime-host.js";
import { resolveQualifiedAcpxProfile, type QualifiedAcpxAgent } from "../drivers/acpx/qualified-profiles.js";
import {
@@ -24,6 +25,7 @@ import {
type AcpxSidecarResponse,
} from "../drivers/acpx/sidecar-protocol.js";
import { validatePrpStructuredRunResult } from "../protocol/replay-contract.js";
import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
interface PendingResolution<T> {
turnId: string;
@@ -224,12 +226,12 @@ async function pumpTurn(currentTurnId: string, runtimeTurn: ReturnType<AcpxRunti
function waitForTool(call: { tool: string; callId: string; arguments: unknown }): Promise<unknown> {
if (!turnId || tools.has(call.callId)) throw new Error("ACPX tool call is unbound or duplicated");
if (call.tool === CODEX_COMPLETION_TOOL_NAME || call.tool === CODEX_BLOCK_TOOL_NAME) {
if (call.tool === PRP_COMPLETION_TOOL_NAME || call.tool === PRP_BLOCK_TOOL_NAME) {
const validation = validatePrpStructuredRunResult(call.arguments);
if (!validation.ok) throw new Error("ACPX semantic result failed PRP schema validation");
if (
(call.tool === CODEX_BLOCK_TOOL_NAME && validation.result.reportedWorkDisposition !== "blocked")
|| (call.tool === CODEX_COMPLETION_TOOL_NAME && validation.result.reportedWorkDisposition === "blocked")
(call.tool === PRP_BLOCK_TOOL_NAME && validation.result.reportedWorkDisposition !== "blocked")
|| (call.tool === PRP_COMPLETION_TOOL_NAME && validation.result.reportedWorkDisposition === "blocked")
) throw new Error("ACPX semantic result disposition does not match its terminal operation");
emit("runtime.event", {
type: "semantic_result",
@@ -380,12 +382,17 @@ function parseOpenParams(value: Record<string, unknown>): AcpxSidecarOpenParams
const agent = requiredAgent(value.agent);
const model = requiredText(value.model, "model");
resolveQualifiedAcpxProfile(agent, model);
const runtimeContextPath = process.env.PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH?.trim();
return {
runtimeDirectory: requiredText(value.runtimeDirectory, "runtimeDirectory"),
normalizedSessionId: requiredText(value.normalizedSessionId, "normalizedSessionId"),
workingDirectory: requiredText(value.workingDirectory, "workingDirectory"),
agent,
model,
systemInstructions: requiredText(value.systemInstructions, "systemInstructions"),
runtimeContext: runtimeContextPath
? parseNativeRuntimeContext(JSON.parse(readFileSync(runtimeContextPath, "utf8")))
: null,
tools: Array.isArray(value.tools) ? value.tools.slice(0, 512).map(record) : [],
...(value.expectedIdentity === undefined || value.expectedIdentity === null
? {}
packages/paperclip-runner/src/cli/eval-session.ts
+3
View File
@@ -49,6 +49,9 @@ interface EvalSessionRequest {
memoryArn: string;
memoryId: string;
invocationRoleArn: string;
contextBucket: string;
contextPrefix: string;
contextKmsKeyArn: string;
qualificationRevision: string;
eventExpiryDays: 90;
maxEstimatedSessionCostUsd: number;
packages/paperclip-runner/src/cli/opencode-app-server-proxy.ts
+24 -1
View File
@@ -1,10 +1,12 @@
#!/usr/bin/env node
import { createInterface } from "node:readline";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import type { HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
import type { HarnessRuntimeRequestResolution, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
import { createCodexTaskEnvelope } from "../contracts/codex.js";
import { OpenCodeServerDriver } from "../drivers/opencode/opencode-server-driver.js";
import { parseNativeRuntimeContext } from "../contracts/runtime-context.js";
type RpcMessage = { id?: string | number; method?: string; params?: unknown; result?: unknown; error?: unknown };
@@ -48,6 +50,10 @@ async function open(params: Record<string, unknown>, resume: boolean): Promise<R
const model = text(params.model);
if (!model.includes("/")) throw new Error("OpenCode proxy requires model in provider/model form");
const dynamicTools = Array.isArray(params.dynamicTools) ? params.dynamicTools.map(record) : [];
const runtimeContextPath = process.env.PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH?.trim();
const runtimeContext = runtimeContextPath
? parseNativeRuntimeContext(JSON.parse(readFileSync(runtimeContextPath, "utf8")))
: null;
driver = new OpenCodeServerDriver({
model,
command: process.env.PAPERCLIP_OPENCODE_COMMAND?.trim() || "opencode",
@@ -62,6 +68,8 @@ async function open(params: Record<string, unknown>, resume: boolean): Promise<R
"Use Paperclip MCP tools for semantic operations.",
],
}),
systemInstructions: text(params.baseInstructions, "Complete only the supplied task."),
runtimeContext,
dynamicTools,
dynamicToolHandler: async (call) => requestController("item/tool/call", {
threadId: session?.ids().driverSessionId ?? "opening",
@@ -169,6 +177,21 @@ async function pumpEvents(opened: HarnessSession): Promise<void> {
result: payload,
},
});
} else if (event.eventType === "runtime_request.created") {
const request = record(payload.request);
const requestId = text(request.requestId);
const turnId = text(request.turnId, text(event.turnId, activeTurnId ?? ""));
if (!requestId || !turnId || !opened.resolveRuntimeRequest) {
throw new Error("OpenCode runtime request is not resolvable");
}
const controllerResponse = record(await requestController("paperclip/runtimeRequest", {
request,
}));
await opened.resolveRuntimeRequest({
requestId,
turnId,
resolution: record(controllerResponse.resolution) as HarnessRuntimeRequestResolution,
});
} else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) {
const status = event.eventType.slice("turn.".length);
send({ method: "turn/completed", params: { threadId: opened.ids().driverSessionId, turnId: event.turnId, turn: { id: event.turnId, status } } });
packages/paperclip-runner/src/contracts/codex.ts
+21 -164
View File
@@ -4,15 +4,24 @@ import type {
PrpStructuredRunResult,
} from "../protocol/replay-contract.js";
import type { SessionSnapshot } from "../reducer/session-reducer.js";
import {
PRP_BLOCK_RESULT_OUTPUT_SCHEMA,
PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
PRP_BLOCK_TOOL_NAME,
PRP_COMPLETION_RESULT_OUTPUT_SCHEMA,
PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
PRP_COMPLETION_TOOL_NAME,
PRP_SEMANTIC_TOOL_NAMES,
} from "./completion-result.js";
export const CODEX_TASK_ENVELOPE_SCHEMA = "paperclip.skillless_task.v1" as const;
export const CODEX_CODEX_PROTOCOL_VERSION = "v2" as const;
export const CODEX_COMPLETION_TOOL_NAME = "paperclip_finish" as const;
export const CODEX_BLOCK_TOOL_NAME = "paperclip_block" as const;
export const CODEX_SEMANTIC_TOOL_NAMES = [
CODEX_COMPLETION_TOOL_NAME,
CODEX_BLOCK_TOOL_NAME,
] as const;
/** @deprecated Use the provider-neutral PRP completion contract exports. */
export const CODEX_COMPLETION_TOOL_NAME = PRP_COMPLETION_TOOL_NAME;
/** @deprecated Use the provider-neutral PRP completion contract exports. */
export const CODEX_BLOCK_TOOL_NAME = PRP_BLOCK_TOOL_NAME;
/** @deprecated Use the provider-neutral PRP completion contract exports. */
export const CODEX_SEMANTIC_TOOL_NAMES = PRP_SEMANTIC_TOOL_NAMES;
export const CODEX_SKILLLESS_BASE_INSTRUCTIONS =
"Complete only the supplied task envelope. Do not discover or invoke skills. Do not call a control-plane API. Return exactly one semantic completion result; use paperclip_finish when the work is done or needs review, and paperclip_block only when work cannot continue." as const;
@@ -141,164 +150,12 @@ export function createCodexTaskEnvelope(input: {
};
}
/**
* Strict provider-facing subset of the wider PRP result schema.
*
* OpenAI structured outputs require every object to reject additional
* properties. The accepted value is still validated against the canonical PRP
* schema before it is committed as a semantic result.
*/
const completionClaimSchema = {
type: "object",
additionalProperties: false,
required: ["contractRevision", "objectiveSatisfied", "criteria", "remainingWork"],
properties: {
contractRevision: { type: "string", minLength: 1 },
objectiveSatisfied: { type: "boolean" },
criteria: {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["criterionId", "status", "evidenceRefs"],
properties: {
criterionId: { type: "string", minLength: 1 },
status: { enum: ["satisfied", "not_satisfied", "unknown"] },
evidenceRefs: { type: "array", items: { type: "string" } },
},
},
},
remainingWork: {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["description", "blocksCompletion"],
properties: {
description: { type: "string", minLength: 1 },
blocksCompletion: { type: "boolean" },
},
},
},
},
} as const;
const evidenceSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["ref"],
properties: { ref: { type: "string", minLength: 1 } },
},
} as const;
const verificationSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["commandOrCheck", "status"],
properties: {
commandOrCheck: { type: "string", minLength: 1 },
status: { enum: ["passed", "failed", "not_run"] },
},
},
} as const;
const attentionRequestsSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["kind", "summary"],
properties: {
kind: { type: "string", minLength: 1 },
summary: { type: "string", minLength: 1 },
},
},
} as const;
const artifactsSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["kind", "ref"],
properties: {
kind: { type: "string", minLength: 1 },
ref: { type: "string", minLength: 1 },
},
},
} as const;
const commonResultProperties = {
schema: { type: "string", const: "paperclip.run_result.v1" },
summary: { type: "string", minLength: 1 },
completionClaim: completionClaimSchema,
evidence: evidenceSchema,
verification: verificationSchema,
attentionRequests: attentionRequestsSchema,
artifacts: artifactsSchema,
} as const;
export const CODEX_RESULT_OUTPUT_SCHEMA = {
type: "object",
additionalProperties: false,
required: [
"schema",
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
"attentionRequests",
"artifacts",
],
properties: {
...commonResultProperties,
reportedWorkDisposition: { enum: ["done", "needs_review"] },
},
} as const;
export const CODEX_BLOCK_RESULT_OUTPUT_SCHEMA = {
type: "object",
additionalProperties: false,
required: [
"schema",
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
"attentionRequests",
"artifacts",
"blocker",
],
properties: {
...commonResultProperties,
reportedWorkDisposition: { type: "string", const: "blocked" },
blocker: {
type: "object",
additionalProperties: false,
required: ["reasonCode", "owner", "unblockAction", "scope"],
properties: {
reasonCode: { type: "string", minLength: 1 },
owner: {
type: "object",
additionalProperties: false,
required: ["kind", "name"],
properties: {
kind: { enum: ["agent", "user", "system", "external"] },
name: { type: "string", minLength: 1 },
},
},
unblockAction: { type: "string", minLength: 1 },
scope: { enum: ["current_track", "task_wide"] },
},
},
},
} as const;
/** @deprecated Use PRP_COMPLETION_RESULT_OUTPUT_SCHEMA. */
export const CODEX_RESULT_OUTPUT_SCHEMA = PRP_COMPLETION_RESULT_OUTPUT_SCHEMA;
/** @deprecated Use PRP_BLOCK_RESULT_OUTPUT_SCHEMA. */
export const CODEX_BLOCK_RESULT_OUTPUT_SCHEMA = PRP_BLOCK_RESULT_OUTPUT_SCHEMA;
export const CODEX_RESULT_PROVIDER_INPUT_SCHEMA = PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA;
export const CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA = PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA;
export function isSkilllessCodexContext(
context: CodexModelContextSnapshot,
packages/paperclip-runner/src/contracts/completion-result.test.ts
+87
View File
@@ -0,0 +1,87 @@
import Ajv2020 from "ajv/dist/2020.js";
import { describe, expect, it } from "vitest";
import {
PRP_COMPLETION_RESULT_OUTPUT_SCHEMA,
PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
} from "./completion-result.js";
const baseResult = {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "done",
summary: "Completed the requested work.",
completionClaim: {
contractRevision: "1",
objectiveSatisfied: true,
criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }],
remainingWork: [],
},
evidence: [],
verification: [],
attentionRequests: [],
artifacts: [],
};
describe("provider-neutral completion result schema", () => {
const validate = new Ajv2020({ allErrors: true, strict: false })
.compile(PRP_COMPLETION_RESULT_OUTPUT_SCHEMA);
it("allows done with no verification and no actionable attention", () => {
expect(validate(structuredClone(baseResult))).toBe(true);
});
it("allows provider tool callers to omit the constant schema discriminator", () => {
const providerValidate = new Ajv2020({ allErrors: true, strict: false })
.compile(PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA);
const providerResult = structuredClone(baseResult) as Record<string, unknown>;
delete providerResult.schema;
expect(providerValidate(providerResult)).toBe(true);
});
it("admits known smaller-model aliases at the provider boundary for canonical normalization", () => {
const providerValidate = new Ajv2020({ allErrors: true, strict: false })
.compile(PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA);
const providerResult = structuredClone(baseResult);
providerResult.schema = "paperclip_paperclip_finish";
providerResult.reportedWorkDisposition = "completed";
providerResult.completionClaim.criteria[0]!.status = "passed";
providerResult.verification = [{ commandOrCheck: "model check", status: "pass" } as never];
expect(providerValidate(providerResult)).toBe(true);
});
it("requires a reason code for verification that was not run", () => {
const result = structuredClone(baseResult);
result.verification = [{ commandOrCheck: "Run tests", status: "not_run" } as never];
expect(validate(result)).toBe(false);
result.verification = [{
commandOrCheck: "Run tests",
status: "not_run",
reasonCode: "tool_unavailable",
} as never];
expect(validate(result)).toBe(true);
});
it("requires needs_review for actionable attention", () => {
const result = structuredClone(baseResult);
result.attentionRequests = [{
kind: "review",
summary: "Confirm the external result.",
ownerClass: "human",
} as never];
expect(validate(result)).toBe(false);
result.reportedWorkDisposition = "needs_review";
expect(validate(result)).toBe(true);
});
it("requires agent-owned attention to identify its target agent", () => {
const result = structuredClone(baseResult);
result.reportedWorkDisposition = "needs_review";
result.attentionRequests = [{
kind: "agent_handoff",
summary: "Ask the deployment agent to continue.",
ownerClass: "agent",
} as never];
expect(validate(result)).toBe(false);
});
});
packages/paperclip-runner/src/contracts/completion-result.ts
+350
View File
@@ -0,0 +1,350 @@
/** Provider-neutral semantic completion tools and their strict model-facing schemas. */
export const PRP_COMPLETION_TOOL_NAME = "paperclip_finish" as const;
export const PRP_BLOCK_TOOL_NAME = "paperclip_block" as const;
export const PRP_SEMANTIC_TOOL_NAMES = [
PRP_COMPLETION_TOOL_NAME,
PRP_BLOCK_TOOL_NAME,
] as const;
export const PRP_VERIFICATION_REASON_CODES = [
"environment_unavailable",
"tool_unavailable",
"dependency_missing",
"permission_denied",
"credential_missing",
"policy_restricted",
"external_service_unavailable",
"budget_exhausted",
"other",
] as const;
export const PRP_ACTIONABLE_ATTENTION_KINDS = [
"review",
"human_input",
"approval",
"credential",
"external_action",
"agent_handoff",
] as const;
export const PRP_ATTENTION_OWNER_CLASSES = [
"human",
"agent",
"external_system",
] as const;
const completionClaimSchema = {
type: "object",
additionalProperties: false,
required: ["contractRevision", "objectiveSatisfied", "criteria", "remainingWork"],
properties: {
contractRevision: { type: "string", minLength: 1 },
objectiveSatisfied: { type: "boolean" },
criteria: {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["criterionId", "status", "evidenceRefs"],
properties: {
criterionId: { type: "string", minLength: 1 },
status: { enum: ["satisfied", "not_satisfied", "unknown"] },
evidenceRefs: { type: "array", items: { type: "string" } },
},
},
},
remainingWork: {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["description", "blocksCompletion"],
properties: {
description: { type: "string", minLength: 1 },
blocksCompletion: { type: "boolean" },
},
},
},
},
} as const;
const evidenceSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["ref"],
properties: { ref: { type: "string", minLength: 1 } },
},
} as const;
const verificationSchema = {
type: "array",
description:
"Checks used to verify the work. Use failed only when the check actually ran and found a defect in the work. If a check could not run or complete because the environment, tool, dependency, permission, credential, policy, external service, or budget was unavailable, use not_run with reasonCode even when the attempted command exited non-zero.",
items: {
type: "object",
additionalProperties: false,
required: ["commandOrCheck", "status"],
properties: {
commandOrCheck: { type: "string", minLength: 1 },
status: {
enum: ["passed", "failed", "not_run"],
description:
"passed: the check ran and succeeded. failed: the check ran to a meaningful verdict and found the work incorrect. not_run: the check had no meaningful verdict because it was not attempted or an environment/tool/dependency/permission/credential/policy/service/budget limitation prevented it from completing; command launch or setup failures are not_run.",
},
reasonCode: {
enum: [...PRP_VERIFICATION_REASON_CODES],
description: "Required for not_run; identify why the check had no meaningful verdict.",
},
detail: { type: "string" },
artifactRef: { type: "string", minLength: 1 },
},
allOf: [{
if: { properties: { status: { const: "not_run" } }, required: ["status"] },
then: { required: ["reasonCode"] },
}],
},
} as const;
const attentionRequestsSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["kind", "summary", "ownerClass"],
properties: {
kind: { enum: [...PRP_ACTIONABLE_ATTENTION_KINDS] },
summary: { type: "string", minLength: 1 },
ownerClass: { enum: [...PRP_ATTENTION_OWNER_CLASSES] },
targetAgentId: { type: "string", minLength: 1 },
},
allOf: [{
if: { properties: { ownerClass: { const: "agent" } }, required: ["ownerClass"] },
then: { required: ["targetAgentId"] },
}],
},
} as const;
const artifactsSchema = {
type: "array",
items: {
type: "object",
additionalProperties: false,
required: ["kind", "ref"],
properties: {
kind: { type: "string", minLength: 1 },
ref: { type: "string", minLength: 1 },
},
},
} as const;
const commonResultProperties = {
schema: { type: "string", const: "paperclip.run_result.v1" },
summary: { type: "string", minLength: 1 },
completionClaim: completionClaimSchema,
evidence: evidenceSchema,
verification: verificationSchema,
attentionRequests: attentionRequestsSchema,
artifacts: artifactsSchema,
} as const;
export const PRP_COMPLETION_RESULT_OUTPUT_SCHEMA = {
type: "object",
additionalProperties: false,
required: [
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
"attentionRequests",
"artifacts",
],
properties: {
...commonResultProperties,
reportedWorkDisposition: { enum: ["done", "needs_review"] },
},
allOf: [
{
if: { properties: { reportedWorkDisposition: { const: "done" } }, required: ["reportedWorkDisposition"] },
then: { properties: { attentionRequests: { maxItems: 0 } } },
},
{
if: { properties: { reportedWorkDisposition: { const: "needs_review" } }, required: ["reportedWorkDisposition"] },
then: { properties: { attentionRequests: { minItems: 1 } } },
},
],
} as const;
export const PRP_BLOCK_RESULT_OUTPUT_SCHEMA = {
type: "object",
additionalProperties: false,
required: [
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
"attentionRequests",
"artifacts",
"blocker",
],
properties: {
...commonResultProperties,
reportedWorkDisposition: { type: "string", const: "blocked" },
attentionRequests: { type: "array", maxItems: 0, items: attentionRequestsSchema.items },
blocker: {
type: "object",
additionalProperties: false,
required: ["reasonCode", "owner", "unblockAction", "scope"],
properties: {
reasonCode: { type: "string", minLength: 1 },
owner: {
type: "object",
additionalProperties: false,
required: ["kind", "name"],
properties: {
kind: { enum: ["agent", "user", "system", "external"] },
name: { type: "string", minLength: 1 },
},
},
unblockAction: { type: "string", minLength: 1 },
scope: { enum: ["current_track", "task_wide"] },
},
},
},
} as const;
const providerVerificationCompatibilitySchema = {
type: "array",
description:
"Checks used to verify the work. Use failed only when the check actually ran and found a defect in the work. If a check could not run or complete because the environment, tool, dependency, permission, credential, policy, external service, or budget was unavailable, use not_run with reasonCode even when the attempted command exited non-zero.",
items: {
type: "object",
additionalProperties: true,
required: ["status"],
properties: {
commandOrCheck: { type: "string", minLength: 1 },
command: { type: "string", minLength: 1 },
status: {
enum: ["passed", "failed", "not_run", "blocked", "skipped", "pass", "success", "succeeded", "fail"],
description:
"passed: the check ran and succeeded. failed: the check ran to a meaningful verdict and found the work incorrect. not_run: no meaningful verdict because the check was not attempted or could not complete. Prefer not_run over legacy blocked/skipped, and include reasonCode for any unavailable check.",
},
reasonCode: {
enum: [...PRP_VERIFICATION_REASON_CODES],
description: "Required for not_run; identify why the check had no meaningful verdict.",
},
detail: { type: "string" },
result: { type: "string" },
cwd: { type: "string" },
artifactRef: { type: "string", minLength: 1 },
},
},
} as const;
const providerCompletionClaimCompatibilitySchema = {
...completionClaimSchema,
properties: {
...completionClaimSchema.properties,
criteria: {
...completionClaimSchema.properties.criteria,
items: {
...completionClaimSchema.properties.criteria.items,
properties: {
...completionClaimSchema.properties.criteria.items.properties,
status: {
enum: [
"satisfied",
"not_satisfied",
"unknown",
"pass",
"passed",
"complete",
"completed",
"fail",
"failed",
"incomplete",
],
description:
"Use satisfied when the criterion is met, not_satisfied when it is not met, or unknown when no verdict is available. Common provider aliases are accepted and normalized.",
},
},
},
},
},
} as const;
const providerCommonResultProperties = {
...commonResultProperties,
schema: {
enum: [
"paperclip.run_result.v1",
"paperclip_finish",
"paperclip_paperclip_finish",
"paperclip_finish.v1",
],
},
completionClaim: providerCompletionClaimCompatibilitySchema,
} as const;
const providerAttentionCompatibilitySchema = {
type: "array",
items: {
type: "object",
additionalProperties: true,
required: ["kind", "summary"],
properties: {
kind: { type: "string", minLength: 1 },
summary: { type: "string", minLength: 1 },
ownerClass: { enum: [...PRP_ATTENTION_OWNER_CLASSES] },
targetAgentId: { type: "string", minLength: 1 },
requestedCapability: { type: "string", minLength: 1 },
requiredAuthority: { type: "string", minLength: 1 },
target: { type: "object", additionalProperties: true },
},
},
} as const;
/**
* Provider-facing compatibility schemas. Canonical PRP validation still uses
* the strict schemas above after legacy aliases have been normalized.
*/
export const PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA = {
type: "object",
additionalProperties: true,
required: [
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
],
properties: {
...providerCommonResultProperties,
reportedWorkDisposition: { enum: ["done", "needs_review", "completed"] },
verification: providerVerificationCompatibilitySchema,
attentionRequests: providerAttentionCompatibilitySchema,
},
} as const;
export const PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA = {
type: "object",
additionalProperties: true,
required: [
"reportedWorkDisposition",
"summary",
"completionClaim",
"evidence",
"verification",
"blocker",
],
properties: {
...providerCommonResultProperties,
reportedWorkDisposition: { type: "string", const: "blocked" },
verification: providerVerificationCompatibilitySchema,
attentionRequests: providerAttentionCompatibilitySchema,
blocker: PRP_BLOCK_RESULT_OUTPUT_SCHEMA.properties.blocker,
},
} as const;
packages/paperclip-runner/src/contracts/harness-driver.ts
+52 -1
View File
@@ -3,6 +3,14 @@ import type {
PrpStructuredRunResult,
} from "../protocol/replay-contract.js";
import type { NativeSessionCapabilities, NativeUserMessage } from "./types.js";
import {
parsePaperclipQuestionResponse,
type PaperclipQuestionResponse,
type PaperclipQuestionSet,
type PaperclipRuntimeRequestOrigin,
} from "./question-set.js";
export * from "./question-set.js";
export const HARNESS_DRIVER_CONTRACT_VERSION = 1 as const;
@@ -98,6 +106,10 @@ export interface HarnessRuntimeRequest {
status: "pending";
prompt: string;
details: Record<string, unknown>;
/** Provider-neutral presentation model for structured input requests. */
input?: PaperclipQuestionSet;
/** Diagnostic origin only; provider response shapes never enter PRP. */
origin?: PaperclipRuntimeRequestOrigin;
}
export type HarnessRuntimeRequestResolution =
@@ -109,6 +121,10 @@ export type HarnessRuntimeRequestResolution =
| {
action: "submit";
content: Record<string, unknown>;
}
| {
action: "submit";
response: PaperclipQuestionResponse;
};
export type HarnessRuntimeRequestAction = HarnessRuntimeRequestResolution["action"];
@@ -160,6 +176,7 @@ function parseAnswers(value: unknown): Record<string, { answers: string[] }> | n
export function parseHarnessRuntimeRequestResolution(
requestKind: HarnessRuntimeRequestKind,
value: unknown,
questionSet?: PaperclipQuestionSet,
): HarnessRuntimeRequestResolution {
const candidate = plainRecord(value) ?? {};
const rawAction = candidate.action;
@@ -173,13 +190,45 @@ export function parseHarnessRuntimeRequestResolution(
);
}
const action = rawAction as HarnessRuntimeRequestAction;
if (action !== "submit" && ("answers" in candidate || "content" in candidate)) {
if (action !== "submit" && ("answers" in candidate || "content" in candidate || "response" in candidate)) {
throw new HarnessRuntimeRequestResolutionError(
requestKind,
`${action} does not carry submitted form data`,
);
}
if (action === "submit" && "response" in candidate) {
if (requestKind !== "user_input" && requestKind !== "elicitation") {
throw new HarnessRuntimeRequestResolutionError(
requestKind,
"approval requests do not accept submitted question responses",
);
}
if ("answers" in candidate || "content" in candidate) {
throw new HarnessRuntimeRequestResolutionError(
requestKind,
"canonical submissions cannot also carry provider-specific answers or content",
);
}
if (questionSet === undefined) {
throw new HarnessRuntimeRequestResolutionError(
requestKind,
"canonical submission requires the persisted question set",
);
}
try {
return {
action,
response: parsePaperclipQuestionResponse(questionSet, candidate.response),
};
} catch (error) {
throw new HarnessRuntimeRequestResolutionError(
requestKind,
error instanceof Error ? error.message : "invalid question response",
);
}
}
if (requestKind === "user_input") {
if (action === "accept" || action === "accept_for_session") {
throw new HarnessRuntimeRequestResolutionError(
@@ -336,6 +385,8 @@ export interface PersistedHarnessSession {
lastSourceSequence?: number;
/** Tagged provider identity used to reject cross-profile recovery. */
providerIdentity?: PersistedHarnessProviderIdentity;
/** Narrow escape hatch for a durable response-wake when a provider cannot reload its prior native session. */
providerRecoveryPolicy?: "same_session_only" | "allow_replacement_after_governed_wait";
}
export interface HarnessSessionRecoveryResult {
packages/paperclip-runner/src/contracts/native-execution.test.ts
+111
View File
@@ -1,6 +1,14 @@
import { describe, expect, it } from "vitest";
import { buildNativeModelEnvelope, parseNativeExecutionInput, type NativeExecutionInputV1 } from "./native-execution.js";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
PAPERCLIP_EXECUTION_PROMPT,
PAPERCLIP_EXECUTION_PROMPT_REVISION,
canonicalNativeRuntimeContextDigest,
composeNativeSystemInstructions,
nativeRuntimePromptDigest,
} from "./runtime-context.js";
const input: NativeExecutionInputV1 = {
schema: "paperclip.native-execution-input.v1",
@@ -47,6 +55,66 @@ const input: NativeExecutionInputV1 = {
};
describe("NativeExecutionInputV1", () => {
it("parses v3 immutable runtime context without changing the model task envelope", () => {
const digest = "0".repeat(64);
const context = {
prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() },
instructions: {
entryPath: "AGENTS.md",
bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/instructions", fileCount: 2, totalBytes: 42 },
},
skills: [{
key: "company/research",
runtimeName: "research",
versionId: "version-1",
bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/skills/research", fileCount: 2, totalBytes: 42 },
}],
mcp: { assignmentSetId: "sha256:test", digest, bindingId: "native-mcp:run-1" },
} as const;
const parsed = parseNativeExecutionInput({
...input,
schema: "paperclip.native-execution-input.v3",
executionMode: "default",
planningContext: null,
runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) },
});
expect(parsed.schema).toBe("paperclip.native-execution-input.v3");
const envelope = buildNativeModelEnvelope(parsed);
expect(envelope.task).toEqual(buildNativeModelEnvelope(input).task);
expect(envelope.completionContract).toEqual(buildNativeModelEnvelope(input).completionContract);
expect(JSON.stringify(envelope)).not.toContain("runtimeContext");
expect(JSON.stringify(envelope)).not.toContain(PAPERCLIP_EXECUTION_PROMPT);
expect(composeNativeSystemInstructions(parsed.runtimeContext, "Follow sibling.md")).toBe(
`${PAPERCLIP_EXECUTION_PROMPT}\n\nFollow sibling.md\n\nRead-only instruction sibling root: /runtime/instructions`,
);
expect(canonicalNativeRuntimeContextDigest({
...context,
mcp: { ...context.mcp, bindingId: "native-mcp:run-2" },
})).toBe(parsed.runtimeContext.aggregateDigest);
});
it("rejects runtime-context traversal and aggregate digest drift", () => {
const digest = "0".repeat(64);
const context = {
prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() },
instructions: {
entryPath: "../AGENTS.md",
bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/instructions", fileCount: 1, totalBytes: 1 },
},
skills: [],
mcp: { assignmentSetId: "none", digest, bindingId: null },
aggregateDigest: digest,
} as const;
expect(() => parseNativeExecutionInput({ ...input, schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, runtimeContext: context })).toThrow("bundle root");
expect(() => parseNativeExecutionInput({
...input,
schema: "paperclip.native-execution-input.v3",
executionMode: "default",
planningContext: null,
runtimeContext: { ...context, instructions: { ...context.instructions, entryPath: "AGENTS.md" } },
})).toThrow("aggregateDigest");
});
it("builds a model envelope without authority or credential bindings", () => {
const parsed = parseNativeExecutionInput(input);
const model = buildNativeModelEnvelope(parsed);
@@ -143,6 +211,9 @@ describe("NativeExecutionInputV1", () => {
memoryArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/memory-1",
memoryId: "memory-1",
invocationRoleArn: "arn:aws:iam::123456789012:role/paperclip-agentcore-runner",
contextBucket: "paperclip-agentcore-context",
contextPrefix: "paperclip/runtime",
contextKmsKeyArn: "arn:aws:kms:us-east-1:123456789012:key/test",
qualificationRevision: "aws-agentcore-harness-v1",
eventExpiryDays: 90,
},
@@ -273,3 +344,43 @@ describe("NativeExecutionInputV2 planning", () => {
})).toMatchObject({ task: { workMode: "planning" }, executionMode: "default" });
});
});
describe("NativeExecutionInputV2 ask mode", () => {
it("round-trips ask mode as a default execution without planning context", () => {
const parsed = parseNativeExecutionInput({
...input,
schema: "paperclip.native-execution-input.v2",
executionMode: "default",
task: { ...input.task, workMode: "ask" },
planningContext: null,
});
expect(parsed).toMatchObject({
schema: "paperclip.native-execution-input.v2",
executionMode: "default",
task: { workMode: "ask" },
});
expect(buildNativeModelEnvelope(parsed)).toMatchObject({
schema: "paperclip.native-model-envelope.v2",
task: { workMode: "ask" },
executionMode: "default",
planningContext: null,
});
});
it("rejects plan execution for ask mode", () => {
expect(() => parseNativeExecutionInput({
...input,
schema: "paperclip.native-execution-input.v2",
executionMode: "plan",
task: { ...input.task, workMode: "ask" },
planningContext: {
documentId: null,
baseRevisionId: null,
baseRevisionNumber: 0,
markdown: "",
sha256: "digest",
reviewContext: {},
},
})).toThrow("plan execution mode requires planning work mode");
});
});
packages/paperclip-runner/src/contracts/native-execution.ts
+26 -9
View File
@@ -26,7 +26,12 @@ export interface StrictCompletionContractInput {
export interface NativeInteractionResponseEnvelope {
interactionId: string;
kind: "request_confirmation" | "ask_user_questions";
kind:
| "suggest_tasks"
| "ask_user_questions"
| "request_confirmation"
| "request_checkbox_confirmation"
| "request_item_verdicts";
response: Record<string, unknown>;
}
@@ -62,6 +67,9 @@ export interface NativeAwsAgentCoreProfileSnapshot {
memoryArn: string;
memoryId: string;
invocationRoleArn: string;
contextBucket: string;
contextPrefix: string;
contextKmsKeyArn: string;
qualificationRevision: string;
eventExpiryDays: 90;
}
@@ -153,7 +161,7 @@ export interface NativeExecutionInputV2 extends Omit<NativeExecutionInputV1, "sc
schema: typeof NATIVE_EXECUTION_INPUT_SCHEMA_V2;
executionMode: NativeExecutionMode;
task: Omit<NativeExecutionInputV1["task"], "workMode"> & {
workMode: "standard" | "planning";
workMode: "standard" | "planning" | "ask";
};
planningContext: NativePlanningContext | null;
}
@@ -274,14 +282,14 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput
const contract = record(completionContract.contract, "input.completionContract.contract");
exactKeys(contract, ["revision", "objective", "criteria"], "input.completionContract.contract");
if (task.workMode !== "standard" && (!isV2 || task.workMode !== "planning")) {
throw new NativeExecutionInputError("input.task.workMode must be standard or planning");
if (task.workMode !== "standard" && (!isV2 || task.workMode !== "planning" && task.workMode !== "ask")) {
throw new NativeExecutionInputError("input.task.workMode must be standard, planning, or ask");
}
const executionMode: NativeExecutionMode = isV2 && input.executionMode === "plan" ? "plan" : "default";
if (isV2 && input.executionMode !== "default" && input.executionMode !== "plan") {
throw new NativeExecutionInputError("input.executionMode must be default or plan");
}
if (isV2 && task.workMode === "standard" && executionMode === "plan") {
if (isV2 && task.workMode !== "planning" && executionMode === "plan") {
throw new NativeExecutionInputError("plan execution mode requires planning work mode");
}
let planningContext: NativePlanningContext | null = null;
@@ -427,7 +435,7 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput
exactKeys(profile, [
"profileId", "region", "accountId", "harnessArn", "harnessVersion", "endpointArn",
"endpointQualifier", "agentRuntimeArn", "memoryArn", "memoryId", "invocationRoleArn",
"qualificationRevision", "eventExpiryDays",
"contextBucket", "contextPrefix", "contextKmsKeyArn", "qualificationRevision", "eventExpiryDays",
], "input.provider.agentCoreProfile");
if (profile.eventExpiryDays !== 90) {
throw new NativeExecutionInputError("input.provider.agentCoreProfile.eventExpiryDays must be 90");
@@ -468,6 +476,9 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput
memoryArn: text(profile.memoryArn, "input.provider.agentCoreProfile.memoryArn"),
memoryId: text(profile.memoryId, "input.provider.agentCoreProfile.memoryId"),
invocationRoleArn: text(profile.invocationRoleArn, "input.provider.agentCoreProfile.invocationRoleArn"),
contextBucket: text(profile.contextBucket, "input.provider.agentCoreProfile.contextBucket"),
contextPrefix: text(profile.contextPrefix, "input.provider.agentCoreProfile.contextPrefix"),
contextKmsKeyArn: text(profile.contextKmsKeyArn, "input.provider.agentCoreProfile.contextKmsKeyArn"),
qualificationRevision: text(profile.qualificationRevision, "input.provider.agentCoreProfile.qualificationRevision"),
eventExpiryDays: 90,
},
@@ -549,10 +560,16 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput
const interactionResponses = input.interactionResponses.map((entry, index) => {
const response = record(entry, `input.interactionResponses[${index}]`);
exactKeys(response, ["interactionId", "kind", "response"], `input.interactionResponses[${index}]`);
if (response.kind !== "request_confirmation" && response.kind !== "ask_user_questions") {
if (![
"suggest_tasks",
"ask_user_questions",
"request_confirmation",
"request_checkbox_confirmation",
"request_item_verdicts",
].includes(String(response.kind))) {
throw new NativeExecutionInputError(`input.interactionResponses[${index}].kind is unsupported`);
}
const kind: NativeInteractionResponseEnvelope["kind"] = response.kind;
const kind = response.kind as NativeInteractionResponseEnvelope["kind"];
return {
interactionId: text(response.interactionId, `input.interactionResponses[${index}].interactionId`),
kind,
@@ -587,7 +604,7 @@ export function parseNativeExecutionInput(value: unknown): NativeExecutionInput
// Recovery must retain those runs, so derive the same bounded model
// prompt from their task fields instead of making them unrecoverable.
prompt: text(task.prompt ?? task.description ?? task.title, "input.task.prompt"),
workMode: task.workMode as "standard" | "planning",
workMode: task.workMode as "standard" | "planning" | "ask",
},
workspace: {
cwd: text(workspace.cwd, "input.workspace.cwd"),
packages/paperclip-runner/src/contracts/native-session-backend.ts
+10
View File
@@ -10,8 +10,10 @@ import type {
} from "../protocol/replay-contract.js";
import type {
HarnessRuntimeRequest,
HarnessRuntimeRequestResolution,
HarnessThreadLineageEntry,
NativeRuntimeContextCapabilities,
PersistedHarnessProviderIdentity,
PersistedHarnessTurnTerminal,
} from "./harness-driver.js";
@@ -34,6 +36,9 @@ export interface PersistedNativeSession {
sessionId: string;
identity: NativeRunIdentity;
providerSessionId?: string | null;
/** Tagged provider-owned identity required for safe driver recovery. */
providerIdentity?: PersistedHarnessProviderIdentity;
providerRecoveryPolicy?: "same_session_only" | "allow_replacement_after_governed_wait";
cursor?: string | null;
semanticResult?: PrpStructuredRunResult | null;
terminal?: PrpTerminalState | null;
@@ -61,6 +66,11 @@ export interface NativeSession {
steer?(input: { turnId: string; message: NativeUserMessage; correlationId?: string }): Promise<void>;
interrupt?(input: { turnId?: string; reason?: string }): Promise<void>;
cancel?(input: { reason: string }): Promise<void>;
resolveRuntimeRequest?(input: {
requestId: string;
turnId: string;
resolution: HarnessRuntimeRequestResolution;
}): Promise<void>;
result(): Promise<{
result: PrpStructuredRunResult;
terminal: PrpTerminalState;
packages/paperclip-runner/src/contracts/provider-trace-file-sink.ts
+216
View File
@@ -0,0 +1,216 @@
import { createHash } from "node:crypto";
import { appendFile, chmod, readFile } from "node:fs/promises";
const DEFAULT_MAX_BYTES = 64 * 1024 * 1024;
const MAX_PENDING_RECORDS = 1_024;
type TraceRecord = Record<string, unknown>;
type ProviderTraceDirection =
| "client_to_provider"
| "provider_to_client"
| "provider_stderr";
type ProviderTraceDisposition =
| "mapped"
| "generic"
| "ignored"
| "rejected"
| "operator_only";
type ProviderTraceFieldMapping = {
inputPath?: string;
outputPath?: string;
action: "copied" | "renamed" | "normalized" | "derived" | "dropped" | "redacted";
reason?: string;
};
/**
* A bounded, fire-and-forget file spool for native providers implemented in
* TypeScript. Provider execution never awaits writes on its hot path; a slow
* or failed sidecar becomes an incomplete trace instead of run backpressure.
*/
export class ProviderTraceFileSink {
readonly #path: string;
readonly #provider: string;
readonly #channel: string;
readonly #maxBytes: number;
readonly #queue: string[] = [];
#nextFrameId: number;
#nextDebugSequence: number;
#capturedBytes: number;
#draining: Promise<void> | null = null;
#closed = false;
#truncated = false;
#incompleteReason: string | null = null;
constructor(input: {
path: string;
provider: string;
channel: string;
maxBytes?: number;
nextFrameId?: number;
nextDebugSequence?: number;
capturedBytes?: number;
}) {
this.#path = input.path;
this.#provider = input.provider;
this.#channel = input.channel;
this.#maxBytes = input.maxBytes ?? DEFAULT_MAX_BYTES;
this.#nextFrameId = input.nextFrameId ?? 1;
this.#nextDebugSequence = input.nextDebugSequence ?? 1;
this.#capturedBytes = input.capturedBytes ?? 0;
}
frame(input: {
direction: ProviderTraceDirection;
raw: Uint8Array | string;
transport: string;
nativeMethod?: string;
}): number | null {
if (this.#closed) return null;
const raw = typeof input.raw === "string" ? Buffer.from(input.raw) : Buffer.from(input.raw);
if (this.#capturedBytes + raw.byteLength > this.#maxBytes) {
this.#truncated = true;
return null;
}
const frameId = this.#nextFrameId++;
this.#capturedBytes += raw.byteLength;
this.#append({
kind: "frame",
schema: "paperclip.provider_trace_frame.v1",
frameId,
timestamp: new Date().toISOString(),
direction: input.direction,
transport: input.transport,
provider: this.#provider,
...(input.nativeMethod ? { nativeMethod: input.nativeMethod } : {}),
byteLength: raw.byteLength,
digest: `sha256:${createHash("sha256").update(raw).digest("hex")}`,
rawBase64: raw.toString("base64"),
});
return frameId;
}
interpretation(input: {
frameId: number;
stage: string;
ruleId: string;
disposition: ProviderTraceDisposition;
emittedEventIds?: string[];
droppedFields?: string[];
fieldMappings?: ProviderTraceFieldMapping[];
reason: string;
}): void {
if (this.#closed) return;
this.#append({
kind: "interpretation",
schema: "paperclip.provider_trace_interpretation.v1",
frameId: input.frameId,
stage: input.stage,
ruleId: input.ruleId,
disposition: input.disposition,
emittedEventIds: input.emittedEventIds ?? [],
droppedFields: input.droppedFields ?? [],
fieldMappings: input.fieldMappings ?? [],
reason: input.reason,
});
}
async finish(input: { reason?: string | null } = {}): Promise<void> {
if (this.#closed) {
await this.#draining;
return;
}
const reason = input.reason ?? this.#incompleteReason;
const status = reason
? "incomplete"
: this.#truncated
? "truncated"
: "complete";
const acknowledgedDebugSequence = this.#nextDebugSequence - 1;
this.#append({
kind: "trace_status",
status,
acknowledgedDebugSequence,
reason: reason ?? (this.#truncated ? "provider_trace_max_bytes_exceeded" : null),
});
this.#closed = true;
await this.#draining;
await chmod(this.#path, 0o600).catch(() => undefined);
}
#append(value: TraceRecord): void {
const record = {
...value,
debugChannel: this.#channel,
debugSequence: this.#nextDebugSequence++,
};
if (this.#queue.length >= MAX_PENDING_RECORDS) {
this.#incompleteReason ??= "trace_sidecar_spool_full";
return;
}
this.#queue.push(`${JSON.stringify(record)}\n`);
if (this.#draining === null) this.#scheduleDrain();
}
#scheduleDrain(): void {
this.#draining = this.#drain().finally(() => {
this.#draining = null;
if (this.#queue.length > 0) this.#scheduleDrain();
});
}
async #drain(): Promise<void> {
while (this.#queue.length > 0) {
const line = this.#queue.shift()!;
try {
await appendFile(this.#path, line, { encoding: "utf8", mode: 0o600 });
} catch {
this.#incompleteReason ??= "trace_sidecar_write_failed";
this.#queue.length = 0;
return;
}
}
}
}
export async function createProviderTraceFileSink(input: {
path?: string;
provider: string;
channel: string;
maxBytes?: string | number;
}): Promise<ProviderTraceFileSink | null> {
if (!input.path) return null;
const maxBytes = typeof input.maxBytes === "number"
? input.maxBytes
: Number.parseInt(input.maxBytes ?? "", 10);
const existing = await readFile(input.path, "utf8").catch(() => "");
let nextFrameId = 1;
let nextDebugSequence = 1;
let capturedBytes = 0;
for (const line of existing.split("\n")) {
if (!line.trim()) continue;
try {
const entry = JSON.parse(line) as Record<string, unknown>;
if (entry.kind === "frame") {
nextFrameId = Math.max(nextFrameId, Number(entry.frameId) + 1 || 1);
capturedBytes += Number(entry.byteLength) || 0;
}
if (entry.debugChannel === input.channel) {
nextDebugSequence = Math.max(
nextDebugSequence,
Number(entry.debugSequence) + 1 || 1,
);
}
} catch {
// The server-side integrity pass will classify a malformed prior record.
}
}
return new ProviderTraceFileSink({
path: input.path,
provider: input.provider,
channel: input.channel,
maxBytes: Number.isSafeInteger(maxBytes) && maxBytes > 0 ? maxBytes : DEFAULT_MAX_BYTES,
nextFrameId,
nextDebugSequence,
capturedBytes,
});
}
packages/paperclip-runner/src/contracts/question-set.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { describe, expect, it } from "vitest";
import {
PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
PAPERCLIP_QUESTION_SET_SCHEMA,
parsePaperclipQuestionResponse,
parsePaperclipQuestionSet,
type PaperclipQuestionSet,
} from "./question-set.js";
const questionSet: PaperclipQuestionSet = {
schema: PAPERCLIP_QUESTION_SET_SCHEMA,
title: "Release input",
questions: [
{
id: "environment",
prompt: "Where should we deploy?",
required: true,
answerMode: "single_select",
options: [
{ id: "staging", label: "Staging" },
{ id: "production", label: "Production" },
],
customAnswer: { enabled: true, label: "Other" },
},
{
id: "replicas",
prompt: "How many replicas?",
required: true,
answerMode: "text",
textValidation: { inputType: "integer", minimum: 1, maximum: 20 },
},
],
};
describe("Paperclip question-set contract", () => {
it("round-trips the portable presentation model", () => {
expect(parsePaperclipQuestionSet(questionSet)).toEqual(questionSet);
expect(parsePaperclipQuestionResponse(questionSet, {
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: {
environment: { selectedOptionIds: ["staging"] },
replicas: { text: "3" },
},
})).toEqual({
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: {
environment: { selectedOptionIds: ["staging"] },
replicas: { text: "3" },
},
});
});
it("rejects missing, unknown, and provider-shaped answers", () => {
expect(() => parsePaperclipQuestionResponse(questionSet, {
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: { environment: { selectedOptionIds: ["unknown"] }, replicas: { text: "3" } },
})).toThrow(/unknown option/);
expect(() => parsePaperclipQuestionResponse(questionSet, {
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: { environment: { selectedOptionIds: ["staging"] } },
})).toThrow(/replicas.*required/);
expect(() => parsePaperclipQuestionResponse(questionSet, {
answers: { environment: { answers: ["Staging"] } },
})).toThrow(/paperclip.question_response.v1/);
});
it("applies typed numeric validation before an adapter sees the answer", () => {
expect(() => parsePaperclipQuestionResponse(questionSet, {
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: {
environment: { customText: "Canary" },
replicas: { text: "3.5" },
},
})).toThrow(/valid integer/);
expect(() => parsePaperclipQuestionResponse(questionSet, {
schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA,
answers: {
environment: { customText: "Canary" },
replicas: { text: "21" },
},
})).toThrow(/at most 20/);
});
});
packages/paperclip-runner/src/contracts/question-set.ts
+344
View File
@@ -0,0 +1,344 @@
export const PAPERCLIP_QUESTION_SET_SCHEMA = "paperclip.question_set.v1" as const;
export const PAPERCLIP_QUESTION_RESPONSE_SCHEMA = "paperclip.question_response.v1" as const;
export const PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2 = "paperclip.runtime_request.v2" as const;
export type PaperclipQuestionAnswerMode = "single_select" | "multi_select" | "text";
export interface PaperclipQuestionOption {
id: string;
label: string;
description?: string;
}
export interface PaperclipQuestionCustomAnswer {
enabled: true;
label?: string;
placeholder?: string;
}
export interface PaperclipQuestionTextValidation {
minLength?: number;
maxLength?: number;
pattern?: string;
inputType?: "text" | "number" | "integer";
minimum?: number;
maximum?: number;
}
export interface PaperclipQuestion {
id: string;
header?: string;
prompt: string;
helpText?: string;
required: boolean;
answerMode: PaperclipQuestionAnswerMode;
options?: PaperclipQuestionOption[];
customAnswer?: PaperclipQuestionCustomAnswer;
textValidation?: PaperclipQuestionTextValidation;
}
export interface PaperclipQuestionSet {
schema: typeof PAPERCLIP_QUESTION_SET_SCHEMA;
title?: string;
description?: string;
submitLabel?: string;
questions: PaperclipQuestion[];
}
export interface PaperclipQuestionAnswer {
selectedOptionIds?: string[];
text?: string;
customText?: string;
}
export interface PaperclipQuestionResponse {
schema: typeof PAPERCLIP_QUESTION_RESPONSE_SCHEMA;
answers: Record<string, PaperclipQuestionAnswer>;
}
export interface PaperclipRuntimeRequestOrigin {
adapter: string;
provider?: string;
method?: string;
}
export interface PaperclipRuntimeInputRequest {
schema: typeof PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2;
requestKind: "runtime";
requestId: string;
type: "input";
status: "pending" | "resolved" | "expired" | "cancelled";
prompt: string;
input: PaperclipQuestionSet;
origin?: PaperclipRuntimeRequestOrigin;
turnId?: string;
itemId?: string;
}
export class PaperclipQuestionValidationError extends Error {
readonly code = "invalid_question_response" as const;
readonly path: string;
constructor(path: string, detail: string) {
super(`${path}: ${detail}`);
this.name = "PaperclipQuestionValidationError";
this.path = path;
}
}
function record(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: null;
}
function requiredText(value: unknown, path: string, maxLength = 4_000): string {
if (typeof value !== "string" || value.length === 0 || value.length > maxLength) {
throw new PaperclipQuestionValidationError(path, `must be a non-empty string of at most ${maxLength} characters`);
}
return value;
}
function optionalText(value: unknown, path: string, maxLength = 4_000): string | undefined {
if (value === undefined) return undefined;
if (typeof value !== "string" || value.length > maxLength) {
throw new PaperclipQuestionValidationError(path, `must be a string of at most ${maxLength} characters`);
}
return value;
}
function optionalFiniteNumber(value: unknown, path: string): number | undefined {
if (value === undefined) return undefined;
if (typeof value !== "number" || !Number.isFinite(value)) {
throw new PaperclipQuestionValidationError(path, "must be a finite number");
}
return value;
}
/** Parse and sanitize the provider-neutral presentation contract at an adapter boundary. */
export function parsePaperclipQuestionSet(value: unknown): PaperclipQuestionSet {
const candidate = record(value);
if (candidate === null || candidate.schema !== PAPERCLIP_QUESTION_SET_SCHEMA) {
throw new PaperclipQuestionValidationError("/input", `must use ${PAPERCLIP_QUESTION_SET_SCHEMA}`);
}
if (!Array.isArray(candidate.questions) || candidate.questions.length === 0 || candidate.questions.length > 64) {
throw new PaperclipQuestionValidationError("/input/questions", "must contain between 1 and 64 questions");
}
const questionIds = new Set<string>();
const questions = candidate.questions.map((rawQuestion, questionIndex): PaperclipQuestion => {
const path = `/input/questions/${questionIndex}`;
const question = record(rawQuestion);
if (question === null) throw new PaperclipQuestionValidationError(path, "must be an object");
const id = requiredText(question.id, `${path}/id`, 160);
if (questionIds.has(id)) throw new PaperclipQuestionValidationError(`${path}/id`, "must be unique");
questionIds.add(id);
const answerMode = question.answerMode;
if (answerMode !== "single_select" && answerMode !== "multi_select" && answerMode !== "text") {
throw new PaperclipQuestionValidationError(`${path}/answerMode`, "must be single_select, multi_select, or text");
}
if (typeof question.required !== "boolean") {
throw new PaperclipQuestionValidationError(`${path}/required`, "must be boolean");
}
const options: PaperclipQuestionOption[] | undefined = Array.isArray(question.options)
? question.options.map((rawOption, optionIndex) => {
const optionPath = `${path}/options/${optionIndex}`;
const option = record(rawOption);
if (option === null) throw new PaperclipQuestionValidationError(optionPath, "must be an object");
return {
id: requiredText(option.id, `${optionPath}/id`, 160),
label: requiredText(option.label, `${optionPath}/label`, 1_000),
...(optionalText(option.description, `${optionPath}/description`) !== undefined
? { description: optionalText(option.description, `${optionPath}/description`) }
: {}),
};
})
: undefined;
if (options !== undefined && new Set(options.map((option) => option.id)).size !== options.length) {
throw new PaperclipQuestionValidationError(`${path}/options`, "option IDs must be unique within a question");
}
if (answerMode !== "text" && (!options || options.length === 0)) {
throw new PaperclipQuestionValidationError(`${path}/options`, "select questions require at least one option");
}
if (answerMode === "text" && options !== undefined && options.length > 0) {
throw new PaperclipQuestionValidationError(`${path}/options`, "text questions cannot define options");
}
const custom = record(question.customAnswer);
const customAnswer = custom === null
? undefined
: {
enabled: true as const,
...(optionalText(custom.label, `${path}/customAnswer/label`, 1_000) !== undefined
? { label: optionalText(custom.label, `${path}/customAnswer/label`, 1_000) }
: {}),
...(optionalText(custom.placeholder, `${path}/customAnswer/placeholder`, 1_000) !== undefined
? { placeholder: optionalText(custom.placeholder, `${path}/customAnswer/placeholder`, 1_000) }
: {}),
};
if (custom !== null && custom.enabled !== true) {
throw new PaperclipQuestionValidationError(`${path}/customAnswer/enabled`, "must be true when customAnswer is present");
}
if (answerMode === "text" && customAnswer !== undefined) {
throw new PaperclipQuestionValidationError(`${path}/customAnswer`, "text questions do not use a separate custom answer");
}
const validation = record(question.textValidation);
const textValidation: PaperclipQuestionTextValidation | undefined = validation === null
? undefined
: {
...(typeof validation.minLength === "number" ? { minLength: validation.minLength } : {}),
...(typeof validation.maxLength === "number" ? { maxLength: validation.maxLength } : {}),
...(optionalText(validation.pattern, `${path}/textValidation/pattern`, 1_000) !== undefined
? { pattern: optionalText(validation.pattern, `${path}/textValidation/pattern`, 1_000) }
: {}),
...(validation.inputType === "number" || validation.inputType === "integer" || validation.inputType === "text"
? { inputType: validation.inputType }
: {}),
...(optionalFiniteNumber(validation.minimum, `${path}/textValidation/minimum`) !== undefined
? { minimum: optionalFiniteNumber(validation.minimum, `${path}/textValidation/minimum`) }
: {}),
...(optionalFiniteNumber(validation.maximum, `${path}/textValidation/maximum`) !== undefined
? { maximum: optionalFiniteNumber(validation.maximum, `${path}/textValidation/maximum`) }
: {}),
};
if (validation !== null) {
for (const key of ["minLength", "maxLength"] as const) {
const raw = validation[key];
if (raw !== undefined && (!Number.isSafeInteger(raw) || (raw as number) < 0 || (raw as number) > 100_000)) {
throw new PaperclipQuestionValidationError(`${path}/textValidation/${key}`, "must be an integer from 0 through 100000");
}
}
if (validation.inputType !== undefined && !["text", "number", "integer"].includes(String(validation.inputType))) {
throw new PaperclipQuestionValidationError(`${path}/textValidation/inputType`, "must be text, number, or integer");
}
if (textValidation?.minLength !== undefined && textValidation.maxLength !== undefined && textValidation.minLength > textValidation.maxLength) {
throw new PaperclipQuestionValidationError(`${path}/textValidation`, "minLength cannot exceed maxLength");
}
if (textValidation?.minimum !== undefined && textValidation.maximum !== undefined && textValidation.minimum > textValidation.maximum) {
throw new PaperclipQuestionValidationError(`${path}/textValidation`, "minimum cannot exceed maximum");
}
if (textValidation?.pattern !== undefined) {
try { new RegExp(textValidation.pattern); } catch {
throw new PaperclipQuestionValidationError(`${path}/textValidation/pattern`, "must be a valid regular expression");
}
}
}
return {
id,
...(optionalText(question.header, `${path}/header`, 1_000) !== undefined
? { header: optionalText(question.header, `${path}/header`, 1_000) }
: {}),
prompt: requiredText(question.prompt, `${path}/prompt`),
...(optionalText(question.helpText, `${path}/helpText`) !== undefined
? { helpText: optionalText(question.helpText, `${path}/helpText`) }
: {}),
required: question.required,
answerMode,
...(options !== undefined ? { options } : {}),
...(customAnswer !== undefined ? { customAnswer } : {}),
...(textValidation !== undefined ? { textValidation } : {}),
};
});
return {
schema: PAPERCLIP_QUESTION_SET_SCHEMA,
...(optionalText(candidate.title, "/input/title", 1_000) !== undefined
? { title: optionalText(candidate.title, "/input/title", 1_000) }
: {}),
...(optionalText(candidate.description, "/input/description") !== undefined
? { description: optionalText(candidate.description, "/input/description") }
: {}),
...(optionalText(candidate.submitLabel, "/input/submitLabel", 200) !== undefined
? { submitLabel: optionalText(candidate.submitLabel, "/input/submitLabel", 200) }
: {}),
questions,
};
}
function answerHasValue(answer: PaperclipQuestionAnswer): boolean {
return Boolean(answer.text?.trim() || answer.customText?.trim() || answer.selectedOptionIds?.length);
}
/** Revalidate untrusted UI input against the persisted question set. */
export function parsePaperclipQuestionResponse(
questionSetValue: unknown,
responseValue: unknown,
): PaperclipQuestionResponse {
const questionSet = parsePaperclipQuestionSet(questionSetValue);
const response = record(responseValue);
if (response === null || response.schema !== PAPERCLIP_QUESTION_RESPONSE_SCHEMA) {
throw new PaperclipQuestionValidationError("/response", `must use ${PAPERCLIP_QUESTION_RESPONSE_SCHEMA}`);
}
const rawAnswers = record(response.answers);
if (rawAnswers === null) throw new PaperclipQuestionValidationError("/response/answers", "must be an object keyed by question ID");
const questions = new Map(questionSet.questions.map((question) => [question.id, question]));
for (const questionId of Object.keys(rawAnswers)) {
if (!questions.has(questionId)) throw new PaperclipQuestionValidationError(`/response/answers/${questionId}`, "does not match a question in the persisted set");
}
const answers: Record<string, PaperclipQuestionAnswer> = {};
for (const question of questionSet.questions) {
const path = `/response/answers/${question.id}`;
const raw = rawAnswers[question.id];
if (raw === undefined) {
if (question.required) throw new PaperclipQuestionValidationError(path, "is required");
continue;
}
const answer = record(raw);
if (answer === null) throw new PaperclipQuestionValidationError(path, "must be an object");
const selectedOptionIds = answer.selectedOptionIds === undefined
? undefined
: Array.isArray(answer.selectedOptionIds) && answer.selectedOptionIds.every((entry) => typeof entry === "string")
? [...answer.selectedOptionIds]
: null;
if (selectedOptionIds === null) throw new PaperclipQuestionValidationError(`${path}/selectedOptionIds`, "must be an array of strings");
if (selectedOptionIds !== undefined && new Set(selectedOptionIds).size !== selectedOptionIds.length) {
throw new PaperclipQuestionValidationError(`${path}/selectedOptionIds`, "cannot contain duplicates");
}
const textValue = optionalText(answer.text, `${path}/text`, 100_000);
const customText = optionalText(answer.customText, `${path}/customText`, 100_000);
if (question.answerMode === "text") {
if (selectedOptionIds?.length || customText !== undefined) throw new PaperclipQuestionValidationError(path, "text answers only carry text");
} else {
if (textValue !== undefined) throw new PaperclipQuestionValidationError(path, "select answers do not carry text");
const allowed = new Set((question.options ?? []).map((option) => option.id));
for (const optionId of selectedOptionIds ?? []) {
if (!allowed.has(optionId)) throw new PaperclipQuestionValidationError(`${path}/selectedOptionIds`, `contains unknown option ${optionId}`);
}
if (question.answerMode === "single_select" && (selectedOptionIds?.length ?? 0) > 1) {
throw new PaperclipQuestionValidationError(`${path}/selectedOptionIds`, "single-select answers choose at most one option");
}
if (customText !== undefined && question.customAnswer?.enabled !== true) {
throw new PaperclipQuestionValidationError(`${path}/customText`, "custom answers are not enabled for this question");
}
if (customText?.trim() && (selectedOptionIds?.length ?? 0) > 0 && question.answerMode === "single_select") {
throw new PaperclipQuestionValidationError(path, "single-select answers cannot select an option and a custom answer");
}
}
const parsed: PaperclipQuestionAnswer = {
...(selectedOptionIds !== undefined ? { selectedOptionIds } : {}),
...(textValue !== undefined ? { text: textValue } : {}),
...(customText !== undefined ? { customText } : {}),
};
if (question.required && !answerHasValue(parsed)) throw new PaperclipQuestionValidationError(path, "is required");
const boundedText = question.answerMode === "text" ? parsed.text : parsed.customText;
if (boundedText !== undefined) {
const validation = question.textValidation;
if (validation?.minLength !== undefined && boundedText.length < validation.minLength) {
throw new PaperclipQuestionValidationError(path, `must contain at least ${validation.minLength} characters`);
}
if (validation?.maxLength !== undefined && boundedText.length > validation.maxLength) {
throw new PaperclipQuestionValidationError(path, `must contain at most ${validation.maxLength} characters`);
}
if (validation?.pattern !== undefined && !new RegExp(validation.pattern).test(boundedText)) {
throw new PaperclipQuestionValidationError(path, "does not match the required format");
}
if (validation?.inputType === "number" || validation?.inputType === "integer") {
const numeric = Number(boundedText);
if (!Number.isFinite(numeric) || (validation.inputType === "integer" && !Number.isInteger(numeric))) {
throw new PaperclipQuestionValidationError(path, `must be a valid ${validation.inputType}`);
}
if (validation.minimum !== undefined && numeric < validation.minimum) throw new PaperclipQuestionValidationError(path, `must be at least ${validation.minimum}`);
if (validation.maximum !== undefined && numeric > validation.maximum) throw new PaperclipQuestionValidationError(path, `must be at most ${validation.maximum}`);
}
}
if (answerHasValue(parsed)) answers[question.id] = parsed;
}
return { schema: PAPERCLIP_QUESTION_RESPONSE_SCHEMA, answers };
}
packages/paperclip-runner/src/contracts/runtime-context.ts
+4 -1
View File
@@ -78,7 +78,10 @@ function aggregatePayload(value: Omit<NativeRuntimeContextSnapshot, "aggregateDi
skills: [...value.skills].sort((a, b) => a.key.localeCompare(b.key)).map((skill) => ({
key: skill.key, runtimeName: skill.runtimeName, versionId: skill.versionId, bundleDigest: skill.bundle.digest,
})),
mcp: value.mcp,
// The binding is deliberately run-scoped. Compatibility is determined by the
// assigned access set so a fresh capability can be rebound without forcing a
// provider-session rotation when policy has not changed.
mcp: { assignmentSetId: value.mcp.assignmentSetId, digest: value.mcp.digest },
};
}
packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts
+10
View File
@@ -195,6 +195,9 @@ export interface DurablePrpControlPlaneOptions {
callId: string;
operationId: string;
input: unknown;
/** Internal trace lineage for the canonical semantic_tool.input event. */
sourceEventId?: string;
sourceEventType?: string;
}) => Promise<unknown>;
connectionLeaseTtlMs?: number;
}
@@ -1468,6 +1471,8 @@ export class DurablePrpControlPlane {
callId: semantic.callId,
operationId: semantic.operationId,
input: semantic.input,
sourceEventId,
sourceEventType: eventType,
};
void this.#onSemanticToolInput(call).then((value) => {
const outcome =
@@ -1553,6 +1558,7 @@ function runnerEnvironment(
for (const key of [
"PATH",
"HOME",
"CODEX_HOME",
"SystemRoot",
"WINDIR",
"PATHEXT",
@@ -1590,6 +1596,7 @@ function runnerEnvironment(
"PAPERCLIP_NATIVE_MCP_NAME",
"PAPERCLIP_NATIVE_MCP_URL",
"PAPERCLIP_NATIVE_MCP_TOKEN",
"PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH",
// Short-lived, controller-selected sidecar capture. These never enter the
// canonical PRP environment or provider payload; runnerd alone consumes them.
"PAPERCLIP_PROVIDER_TRACE_PATH",
@@ -2025,6 +2032,9 @@ export interface DurableEvalSessionInput {
memoryArn: string;
memoryId: string;
invocationRoleArn: string;
contextBucket: string;
contextPrefix: string;
contextKmsKeyArn: string;
qualificationRevision: string;
eventExpiryDays: 90;
maxEstimatedSessionCostUsd: number;
packages/paperclip-runner/src/drivers/acpx/acpx-runtime-driver.test.ts
+29
View File
@@ -0,0 +1,29 @@
import { describe, expect, it } from "vitest";
import { selectAcpxFinalAgentMessage } from "./acpx-runtime-driver.js";
describe("ACPX settled final response", () => {
it("uses the exact assistant prose present when the semantic result was accepted", () => {
expect(selectAcpxFinalAgentMessage({
semanticResultCommitted: true,
textAtSemanticResult: "- First fact\n- Second fact",
textAfterLastWorkTool: "- First fact\n- Second fact\nCompleted: task done.",
})).toBe("- First fact\n- Second fact");
});
it("does not promote a trailing acknowledgement when the semantic boundary had no answer", () => {
expect(selectAcpxFinalAgentMessage({
semanticResultCommitted: true,
textAtSemanticResult: null,
textAfterLastWorkTool: "Completed.",
})).toBeNull();
});
it("preserves a result-less provider final emitted after its last work tool", () => {
expect(selectAcpxFinalAgentMessage({
semanticResultCommitted: false,
textAtSemanticResult: null,
textAfterLastWorkTool: " Substantive answer. ",
})).toBe("Substantive answer.");
});
});
packages/paperclip-runner/src/drivers/acpx/acpx-runtime-driver.ts
+156 -41
View File
@@ -5,11 +5,13 @@ import { join, resolve } from "node:path";
import type { AcpPermissionDecision, AcpPermissionRequest, AcpRuntimeEvent } from "acpx/runtime";
import {
CODEX_BLOCK_TOOL_NAME,
CODEX_COMPLETION_TOOL_NAME,
createCodexTaskEnvelope,
type CodexTaskEnvelope,
} from "../../contracts/codex.js";
import {
PRP_BLOCK_TOOL_NAME,
PRP_COMPLETION_TOOL_NAME,
} from "../../contracts/completion-result.js";
import {
HarnessCapabilityUnavailableError,
HarnessStaleTurnError,
@@ -154,28 +156,55 @@ export class AcpxRuntimeDriver implements HarnessDriver {
join(sessionRoot(this.#options.runtimeDirectory, snapshot.normalizedSessionId), "workspace"),
"utf8",
)).trim();
const session = await this.#open({
const input = {
runId: snapshot.runId,
normalizedSessionId: snapshot.normalizedSessionId,
workingDirectory: workspace,
}, snapshot);
};
const replaceProviderSession =
snapshot.providerRecoveryPolicy ===
"allow_replacement_after_governed_wait";
if (replaceProviderSession) {
this.#options.onDiagnostic?.(
"ACPX is opening a replacement provider session for a durable governed-wait continuation while preserving the Paperclip session lineage.",
);
}
// ACPX 0.13 may not surface an unresumable persistent child until the
// first prompt, after ensureSession has already returned successfully.
// The explicit governed-wait policy therefore rotates proactively; all
// other recovery remains strict same-session-only.
const session = await this.#open(
input,
snapshot,
replaceProviderSession,
);
return { recovered: true, session };
} catch (error) {
return { recovered: false, reason: redact(String(error), this.#options.environment) };
}
}
async #open(input: OpenHarnessSessionInput, snapshot: PersistedHarnessSession | null): Promise<HarnessSession> {
async #open(
input: OpenHarnessSessionInput,
snapshot: PersistedHarnessSession | null,
replaceProviderSession = false,
): Promise<HarnessSession> {
let session: AcpxHarnessSession | null = null;
const host = await AcpxRuntimeHost.open({
runtimeDirectory: this.#options.runtimeDirectory,
normalizedSessionId: input.normalizedSessionId,
...(replaceProviderSession
? {
providerSessionKey:
`${input.normalizedSessionId}:governed-wait-replacement:${input.runId}`,
}
: {}),
workingDirectory: input.workingDirectory,
agent: this.#options.agent,
model: this.#options.model,
systemInstructions: this.#options.systemInstructions,
runtimeContext: this.#options.runtimeContext,
...(snapshot?.providerIdentity?.kind === "acpx"
...(!replaceProviderSession && snapshot?.providerIdentity?.kind === "acpx"
? { expectedIdentity: snapshot.providerIdentity }
: {}),
environment: this.#options.environment,
@@ -194,7 +223,7 @@ export class AcpxRuntimeDriver implements HarnessDriver {
onDiagnostic: this.#options.onDiagnostic,
});
try {
if (snapshot?.providerIdentity?.kind === "acpx") {
if (!replaceProviderSession && snapshot?.providerIdentity?.kind === "acpx") {
assertRecoveryIdentity(snapshot.providerIdentity, host.identity(), input.workingDirectory);
}
session = new AcpxHarnessSession({
@@ -224,6 +253,7 @@ class AcpxHarnessSession implements HarnessSession {
readonly #taskEnvelope: CodexTaskEnvelope;
readonly #dynamicToolHandler?: DynamicToolHandler;
readonly #runnerInstanceId: string;
readonly #providerRecoveryPolicy: PersistedHarnessSession["providerRecoveryPolicy"];
readonly #now: () => Date;
readonly #events = new AsyncQueue<PrpEvent>();
readonly #transcript: PrpEvent[] = [];
@@ -239,6 +269,9 @@ class AcpxHarnessSession implements HarnessSession {
#resultTurnId: string | null;
#usage: Record<string, unknown> | null = null;
#assistantText = "";
#assistantTextSinceLastWorkTool = "";
#assistantTextAtSemanticResult: string | null = null;
#traceCorrelationEventIds: string[] | null = null;
#thinking = false;
#closed = false;
@@ -260,6 +293,8 @@ class AcpxHarnessSession implements HarnessSession {
this.#taskEnvelope = input.taskEnvelope;
this.#dynamicToolHandler = input.dynamicToolHandler;
this.#runnerInstanceId = input.runnerInstanceId;
this.#providerRecoveryPolicy =
input.snapshot?.providerRecoveryPolicy ?? "same_session_only";
this.#now = input.now;
this.#sourceSequence = input.snapshot?.lastSourceSequence ?? 0;
this.#activeTurnId = input.snapshot?.activeTurnId ?? null;
@@ -270,6 +305,9 @@ class AcpxHarnessSession implements HarnessSession {
for (const terminal of input.snapshot?.terminalTurns ?? []) {
this.#terminalTurns.set(terminal.turnId, terminal.fingerprint);
}
if (this.#activeTurnId && this.#terminalTurns.has(this.#activeTurnId)) {
this.#activeTurnId = null;
}
}
ids() {
@@ -292,6 +330,8 @@ class AcpxHarnessSession implements HarnessSession {
this.#resultCallId = null;
this.#resultTurnId = null;
this.#assistantText = "";
this.#assistantTextSinceLastWorkTool = "";
this.#assistantTextAtSemanticResult = null;
this.#emit("run.attached", { runId: input.runId, sameSession: true });
}
@@ -301,6 +341,8 @@ class AcpxHarnessSession implements HarnessSession {
const turnId = `turn-${randomBytes(12).toString("hex")}`;
this.#activeTurnId = turnId;
this.#assistantText = "";
this.#assistantTextSinceLastWorkTool = "";
this.#assistantTextAtSemanticResult = null;
this.#activeTurnAbort = new AbortController();
this.#emit("turn.submitted", { envelopeSchema: this.#taskEnvelope.schema, text: input.message.text });
this.#emit("turn.accepted", { turnId }, { turnId });
@@ -399,37 +441,40 @@ class AcpxHarnessSession implements HarnessSession {
const turnId = this.#activeTurnId;
if (!turnId) throw new Error("ACPX semantic tool call is not bound to an active turn");
const tool = canonicalRunnerToolName(call.tool);
this.#emit("item.started", {
const traceFrameId = this.#host.latestInboundTraceFrameId();
const traceEventIds: string[] = [];
traceEventIds.push(this.#emit("item.started", {
kind: "dynamicToolCall",
item: { type: "tool_call", id: call.callId, name: tool, arguments: bounded(call.arguments) },
}, { turnId, itemId: call.callId });
if (tool === CODEX_COMPLETION_TOOL_NAME || tool === CODEX_BLOCK_TOOL_NAME) {
const validation = validatePrpStructuredRunResult(call.arguments);
if (!validation.ok) throw new Error("Invalid semantic result");
if (
(tool === CODEX_BLOCK_TOOL_NAME && validation.result.reportedWorkDisposition !== "blocked")
|| (tool === CODEX_COMPLETION_TOOL_NAME && validation.result.reportedWorkDisposition === "blocked")
) throw new Error("Semantic result disposition does not match the terminal tool");
if (validation.result.completionClaim.contractRevision !== this.#taskEnvelope.completionContract.revision) {
throw new Error("Semantic result completion contract revision does not match");
}
const fingerprint = canonicalJson(validation.result);
if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) throw new Error("A different semantic result was already committed");
if (!this.#resultFingerprint) {
this.#result = structuredClone(validation.result);
this.#resultFingerprint = fingerprint;
this.#resultCallId = call.callId;
this.#resultTurnId = turnId;
this.#emit("run.result.proposed", validation.result, { turnId, itemId: call.callId });
}
this.#emit("item.completed", {
kind: "dynamicToolCall",
item: { type: "tool_result", id: call.callId, tool_use_id: call.callId, result: "Semantic completion accepted." },
}, { turnId, itemId: call.callId });
return { accepted: true };
}
if (!this.#dynamicToolHandler) throw new Error("Unsupported Paperclip operation");
}, { turnId, itemId: call.callId }));
try {
if (tool === PRP_COMPLETION_TOOL_NAME || tool === PRP_BLOCK_TOOL_NAME) {
const validation = validatePrpStructuredRunResult(call.arguments);
if (!validation.ok) throw new Error("Invalid semantic result");
if (
(tool === PRP_BLOCK_TOOL_NAME && validation.result.reportedWorkDisposition !== "blocked")
|| (tool === PRP_COMPLETION_TOOL_NAME && validation.result.reportedWorkDisposition === "blocked")
) throw new Error("Semantic result disposition does not match the terminal tool");
if (validation.result.completionClaim.contractRevision !== this.#taskEnvelope.completionContract.revision) {
throw new Error("Semantic result completion contract revision does not match");
}
const fingerprint = canonicalJson(validation.result);
if (this.#resultFingerprint && this.#resultFingerprint !== fingerprint) throw new Error("A different semantic result was already committed");
if (!this.#resultFingerprint) {
this.#result = structuredClone(validation.result);
this.#resultFingerprint = fingerprint;
this.#resultCallId = call.callId;
this.#resultTurnId = turnId;
this.#assistantTextAtSemanticResult = this.#assistantTextSinceLastWorkTool.trim() || null;
traceEventIds.push(this.#emit("run.result.proposed", validation.result, { turnId, itemId: call.callId }));
}
traceEventIds.push(this.#emit("item.completed", {
kind: "dynamicToolCall",
item: { type: "tool_result", id: call.callId, tool_use_id: call.callId, result: "Semantic completion accepted." },
}, { turnId, itemId: call.callId }));
return { accepted: true };
}
if (!this.#dynamicToolHandler) throw new Error("Unsupported Paperclip operation");
const result = await this.#dynamicToolHandler({
tool,
callId: call.callId,
@@ -437,17 +482,19 @@ class AcpxHarnessSession implements HarnessSession {
turnId,
arguments: call.arguments,
});
this.#emit("item.completed", {
traceEventIds.push(this.#emit("item.completed", {
kind: "dynamicToolCall",
item: { type: "tool_result", id: call.callId, tool_use_id: call.callId, result: bounded(result) },
}, { turnId, itemId: call.callId });
}, { turnId, itemId: call.callId }));
return result;
} catch (error) {
this.#emit("item.completed", {
traceEventIds.push(this.#emit("item.completed", {
kind: "dynamicToolCall",
item: { type: "tool_result", id: call.callId, tool_use_id: call.callId, error: redact(String(error)), is_error: true },
}, { turnId, itemId: call.callId });
}, { turnId, itemId: call.callId }));
throw error;
} finally {
this.#host.correlateProviderFrame(traceFrameId, traceEventIds, `semantic_tool.${tool}`);
}
}
@@ -513,6 +560,7 @@ class AcpxHarnessSession implements HarnessSession {
requestedModel: identity.requestedModel,
effectiveModel: identity.effectiveModel,
},
providerRecoveryPolicy: this.#providerRecoveryPolicy,
runId: this.#runId,
normalizedSessionId: this.#normalizedSessionId,
activeTurnId: this.#activeTurnId,
@@ -543,7 +591,17 @@ class AcpxHarnessSession implements HarnessSession {
async #pumpTurn(turnId: string, turn: ReturnType<AcpxRuntimeHost["startTurn"]>): Promise<void> {
try {
let index = 0;
for await (const event of turn.events) this.#mapRuntimeEvent(event, turnId, ++index);
for await (const event of turn.events) {
const frameId = this.#host.latestInboundTraceFrameId();
const emittedEventIds: string[] = [];
this.#traceCorrelationEventIds = emittedEventIds;
try {
this.#mapRuntimeEvent(event, turnId, ++index);
} finally {
this.#traceCorrelationEventIds = null;
}
this.#host.correlateProviderFrame(frameId, emittedEventIds, event.type);
}
const result = await turn.result;
if (this.#thinking) {
this.#emit("item.completed", { kind: "thinking", text: "Reasoning completed." }, { turnId, itemId: `${turnId}:thinking` });
@@ -564,7 +622,27 @@ class AcpxHarnessSession implements HarnessSession {
reason: "ACP agent reported an effective model change",
}, { turnId, itemId: `${turnId}:model-route` });
}
if (this.#activeTurnId === turnId) this.#activeTurnId = null;
this.#activeTurnAbort = null;
if (result.status === "completed") {
const finalText = selectAcpxFinalAgentMessage({
semanticResultCommitted: this.#resultFingerprint !== null,
textAtSemanticResult: this.#assistantTextAtSemanticResult,
textAfterLastWorkTool: this.#assistantTextSinceLastWorkTool,
});
if (finalText) {
this.#emit("item.completed", {
kind: "agentMessage",
channel: "final",
providerPhase: "final_answer",
text: finalText,
item: {
type: "agentMessage",
phase: "final_answer",
text: finalText,
},
}, { turnId, itemId: `${turnId}:final-answer` });
}
this.#terminalTurns.set(turnId, canonicalJson({ status: "completed", semanticResult: this.#resultFingerprint }));
this.#emit("turn.completed", { status: "completed", stopReason: result.stopReason ?? null }, { turnId });
} else if (result.status === "cancelled") {
@@ -585,6 +663,8 @@ class AcpxHarnessSession implements HarnessSession {
this.#emit("turn.failed", { status: "failed", error: { code: result.error.code ?? null, message: redact(result.error.message) } }, { turnId });
}
} catch (error) {
if (this.#activeTurnId === turnId) this.#activeTurnId = null;
this.#activeTurnAbort = null;
this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" }));
this.#emit("turn.failed", { status: "failed", error: { message: redact(String(error)) } }, { turnId });
} finally {
@@ -597,6 +677,19 @@ class AcpxHarnessSession implements HarnessSession {
const itemId = event.type === "tool_call" && event.toolCallId
? safeId(event.toolCallId, `${turnId}:acp:${index}`)
: `${turnId}:acp:${index}`;
if (
event.type === "tool_call"
&& event.tag === "tool_call"
&& ![PRP_COMPLETION_TOOL_NAME, PRP_BLOCK_TOOL_NAME].includes(
canonicalAcpxRuntimeToolName(event.title) as typeof PRP_COMPLETION_TOOL_NAME,
)
) {
// ACP agents do not label commentary and final-answer channels. Text
// emitted after the last work tool is the only safe final candidate;
// the accepted completion-tool boundary below then excludes any trailing
// acknowledgement emitted after Paperclip already accepted the result.
this.#assistantTextSinceLastWorkTool = "";
}
if (event.type === "text_delta") {
if (event.stream === "thought" || event.tag === "agent_thought_chunk") {
if (!this.#thinking) {
@@ -607,6 +700,7 @@ class AcpxHarnessSession implements HarnessSession {
}
const output = event.text.slice(0, 64 * 1024);
this.#assistantText = `${this.#assistantText}${output}`.slice(-256 * 1024);
this.#assistantTextSinceLastWorkTool = `${this.#assistantTextSinceLastWorkTool}${output}`.slice(-256 * 1024);
this.#emit("item.delta", { kind: "agent_message", text: output }, { turnId, itemId });
for (const reference of paperclipWorkspaceFileReferencesFromText(this.#workingDirectory, output, turnId)) {
if (this.#emittedFileReferences.has(reference.referenceId)) continue;
@@ -679,7 +773,7 @@ class AcpxHarnessSession implements HarnessSession {
this.#emit("item.completed", { kind: "usage", usage: this.#usage }, { turnId, itemId });
}
#emit(eventType: PrpEvent["eventType"], payload: Record<string, unknown>, refs: { turnId?: string; itemId?: string } = {}): void {
#emit(eventType: PrpEvent["eventType"], payload: Record<string, unknown>, refs: { turnId?: string; itemId?: string } = {}): string {
const sourceSeq = ++this.#sourceSequence;
const event: PrpEvent = {
schema: "paperclip.prp.event.v1",
@@ -697,8 +791,10 @@ class AcpxHarnessSession implements HarnessSession {
emittedAt: this.#now().toISOString(),
payload,
};
this.#traceCorrelationEventIds?.push(event.sourceEventId);
this.#transcript.push(structuredClone(event));
this.#events.push(event);
return event.sourceEventId;
}
#optionsEnvironment(): NodeJS.ProcessEnv { return {}; }
@@ -753,6 +849,25 @@ function canonicalToolTerminal(status: unknown): boolean {
return ["completed", "failed", "cancelled", "canceled"].includes(text(status).toLowerCase());
}
function canonicalAcpxRuntimeToolName(value: unknown): string {
const nativeName = text(value).trim();
const bridgeName = nativeName
.replace(/^mcp__[^_]+__/, "")
.replace(/^mcp\.paperclip\./, "");
return canonicalRunnerToolName(bridgeName);
}
export function selectAcpxFinalAgentMessage(input: {
semanticResultCommitted: boolean;
textAtSemanticResult: string | null;
textAfterLastWorkTool: string;
}): string | null {
const candidate = input.semanticResultCommitted
? input.textAtSemanticResult
: input.textAfterLastWorkTool;
return candidate?.trim() || null;
}
function safeWorkspacePath(value: string): string | null {
const path = value.replaceAll("\\", "/");
if (!path || path.startsWith("/") || path.split("/").includes("..") || /^[a-z]+:\/\//i.test(path)) return null;
packages/paperclip-runner/src/drivers/acpx/acpx-runtime-host.test.ts
+235 -1
View File
@@ -4,6 +4,7 @@ import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import type {
AcpPermissionRequest,
AcpRuntime,
AcpRuntimeHandle,
AcpRuntimeOptions,
@@ -11,7 +12,18 @@ import type {
} from "acpx/runtime";
import { describe, expect, it } from "vitest";
import { AcpxRuntimeHost } from "./acpx-runtime-host.js";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
PAPERCLIP_EXECUTION_PROMPT,
PAPERCLIP_EXECUTION_PROMPT_REVISION,
canonicalNativeRuntimeContextDigest,
nativeRuntimePromptDigest,
type NativeRuntimeContextSnapshot,
} from "../../contracts/runtime-context.js";
import {
AcpxRuntimeHost,
shouldAutoApproveRunnerOwnedSemanticPermission,
} from "./acpx-runtime-host.js";
function completedTurn(requestId: string): AcpRuntimeTurn {
return {
@@ -24,7 +36,113 @@ function completedTurn(requestId: string): AcpRuntimeTurn {
};
}
async function createRuntimeContext(root: string): Promise<NativeRuntimeContextSnapshot> {
const instructionRoot = join(root, "instruction-source");
const skillRoot = join(root, "skill-source");
await Promise.all([
mkdir(instructionRoot, { recursive: true }),
mkdir(join(skillRoot, "references"), { recursive: true }),
]);
await Promise.all([
writeFile(join(instructionRoot, "AGENTS.md"), "Read sibling.md\n"),
writeFile(join(instructionRoot, "sibling.md"), "instruction sibling\n"),
writeFile(join(skillRoot, "SKILL.md"), "# Assigned\nRead references/support.md\n"),
writeFile(join(skillRoot, "references", "support.md"), "skill support\n"),
]);
const digest = "0".repeat(64);
const value = {
prompt: {
revision: PAPERCLIP_EXECUTION_PROMPT_REVISION,
text: PAPERCLIP_EXECUTION_PROMPT,
digest: nativeRuntimePromptDigest(),
},
instructions: {
entryPath: "AGENTS.md",
bundle: {
schema: NATIVE_RUNTIME_ASSET_SCHEMA,
digest,
manifestDigest: digest,
rootPath: instructionRoot,
fileCount: 2,
totalBytes: 2,
},
},
skills: [{
key: "company/assigned",
runtimeName: "assigned",
versionId: "version-1",
bundle: {
schema: NATIVE_RUNTIME_ASSET_SCHEMA,
digest,
manifestDigest: digest,
rootPath: skillRoot,
fileCount: 2,
totalBytes: 2,
},
}],
mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" },
} satisfies Omit<NativeRuntimeContextSnapshot, "aggregateDigest">;
return { ...value, aggregateDigest: canonicalNativeRuntimeContextDigest(value) };
}
describe("AcpxRuntimeHost", () => {
it("auto-approves only runner-owned semantic MCP calls for non-Pi ACP agents", () => {
const permission = (title: string) => ({
raw: { toolCall: { title } },
}) as unknown as AcpPermissionRequest;
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"claude",
permission("mcp__paperclip__paperclip_finish"),
)).toBe(true);
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"codex",
({
raw: {
_meta: { is_mcp_tool_approval: true },
toolCall: {
rawInput: { serverName: "paperclip" },
},
},
}) as unknown as AcpPermissionRequest,
)).toBe(true);
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"codex",
({
raw: {
toolCall: {
rawInput: { serverName: "paperclip-assigned" },
},
},
}) as unknown as AcpPermissionRequest,
new Set(["paperclip", "paperclip-assigned"]),
)).toBe(true);
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"codex",
({
raw: {
_meta: { is_mcp_tool_approval: true },
toolCall: {},
},
}) as unknown as AcpPermissionRequest,
new Set(["paperclip", "paperclip-assigned"]),
true,
)).toBe(true);
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"pi",
permission("mcp__paperclip__paperclip_finish"),
)).toBe(false);
expect(shouldAutoApproveRunnerOwnedSemanticPermission(
"codex",
({
raw: {
_meta: { is_mcp_tool_approval: true },
toolCall: {
rawInput: { serverName: "company-github" },
},
},
}) as unknown as AcpPermissionRequest,
)).toBe(false);
});
it("resolves a pinned profile, isolates state, and keeps credentials out of session records", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-acpx-host-"));
const workspace = join(root, "workspace");
@@ -130,6 +248,122 @@ describe("AcpxRuntimeHost", () => {
}
});
it.each([
{ agent: "pi" as const, model: "openrouter/deepseek/deepseek-v4-flash-0731" },
{ agent: "claude" as const, model: "claude-sonnet-5" },
{ agent: "codex" as const, model: "gpt-5.6-sol" },
])("realizes the complete Paperclip context for ACPX $agent", async ({ agent, model }) => {
const root = await mkdtemp(join(tmpdir(), `paperclip-acpx-${agent}-context-`));
const workspace = join(root, "workspace");
const tracePath = join(root, "provider-trace.ndjson");
await mkdir(workspace);
const runtimeContext = await createRuntimeContext(root);
const systemInstructions = `${PAPERCLIP_EXECUTION_PROMPT}\n\nRead sibling.md\n\nRead-only instruction sibling root: ${runtimeContext.instructions.bundle.rootPath}`;
let capturedOptions: AcpRuntimeOptions | null = null;
let ensureInput: Parameters<AcpRuntime["ensureSession"]>[0] | null = null;
const host = await AcpxRuntimeHost.open({
runtimeDirectory: root,
normalizedSessionId: `${agent}-runtime-context`,
workingDirectory: workspace,
agent,
model,
systemInstructions,
runtimeContext,
environment: {
PATH: process.env.PATH,
PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned",
PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp",
PAPERCLIP_NATIVE_MCP_TOKEN: "x".repeat(40),
PAPERCLIP_PROVIDER_TRACE_PATH: tracePath,
},
dynamicToolHandler: async () => ({}),
runtimeFactory: (options) => {
capturedOptions = options;
return {
ensureSession: async (input) => {
ensureInput = input;
return {
sessionKey: `${agent}-session-key`,
backend: "acpx",
runtimeSessionName: `${agent}-runtime`,
acpxRecordId: `${agent}-record`,
backendSessionId: `${agent}-backend`,
agentSessionId: `${agent}-agent`,
};
},
startTurn: (input) => completedTurn(input.requestId),
runTurn: async function* () {},
getStatus: async () => ({
acpxRecordId: `${agent}-record`,
backendSessionId: `${agent}-backend`,
agentSessionId: `${agent}-agent`,
models: {
currentModelId: agent === "claude" ? "sonnet" : model,
availableModelIds: [agent === "claude" ? "sonnet" : model],
},
}),
setConfigOption: async () => {},
cancel: async () => {},
close: async () => {},
};
},
});
try {
const options = capturedOptions as AcpRuntimeOptions & { spawnEnvironment?: () => Record<string, string> };
const environment = options.spawnEnvironment?.() ?? {};
options.onAcpMessage?.("outbound", {
jsonrpc: "2.0",
id: 1,
method: "session/new",
params: {},
} as never);
options.onAcpMessage?.("inbound", {
jsonrpc: "2.0",
id: 1,
result: { sessionId: "provider-session" },
} as never);
host.correlateProviderFrame(
host.latestInboundTraceFrameId(),
["event-1", "event-2", "event-3"],
"semantic_tool.paperclip_finish",
);
expect(environment.PAPERCLIP_ACPX_ISOLATED_CONTEXT).toBe("1");
expect(options.elicitationModes).toEqual(agent === "codex" ? [] : ["form", "url"]);
expect(ensureInput).toMatchObject({
sessionOptions: { systemPrompt: { append: systemInstructions } },
});
expect(JSON.stringify(ensureInput)).not.toContain("x".repeat(40));
await expect(readFile(join(host.runtimeRoot(), `${agent}-home`, "skills", "assigned", "references", "support.md"), "utf8"))
.resolves.toBe("skill support\n");
if (agent === "pi") {
expect(JSON.parse(environment.PI_ACP_PI_ARGS_JSON ?? "[]")).not.toContain("--no-skills");
expect(options.mcpServers).toEqual([]);
expect(environment.PAPERCLIP_NATIVE_MCP_URL).toBe("https://paperclip.example/mcp");
} else {
expect(options.mcpServers).toEqual(expect.arrayContaining([
expect.objectContaining({ name: "paperclip-assigned", url: "https://paperclip.example/mcp" }),
]));
}
} finally {
await host.close({ reason: "test complete" });
}
const traceEntries = (await readFile(tracePath, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as Record<string, unknown>);
expect(traceEntries.filter((entry) => entry.kind === "frame")).toHaveLength(2);
expect(traceEntries.filter((entry) => entry.kind === "interpretation")).toHaveLength(3);
expect(traceEntries).toContainEqual(expect.objectContaining({
kind: "interpretation",
frameId: 2,
stage: "typescript_acpx_runtime_normalization",
ruleId: "acpx.runtime.semantic_tool.paperclip_finish",
disposition: "mapped",
emittedEventIds: ["event-1", "event-2", "event-3"],
}));
expect(traceEntries.at(-1)).toMatchObject({ kind: "trace_status", status: "complete" });
});
it("rejects model fallback before starting the runtime", async () => {
await expect(AcpxRuntimeHost.open({
runtimeDirectory: await mkdtemp(join(tmpdir(), "paperclip-acpx-model-")),
packages/paperclip-runner/src/drivers/acpx/acpx-runtime-host.ts
+134 -5
View File
@@ -31,6 +31,10 @@ import {
import type { NativeRuntimeContextSnapshot } from "../../contracts/runtime-context.js";
import { nativeMcpLaunchBinding } from "../native-mcp.js";
import { materializeNativeRuntimeSkills } from "../runtime-context-materializer.js";
import {
createProviderTraceFileSink,
type ProviderTraceFileSink,
} from "../../contracts/provider-trace-file-sink.js";
const PI_PERMISSION_TOOL = "__paperclip_runtime_permission";
@@ -69,12 +73,16 @@ interface SecureAcpRuntimeOptions extends AcpRuntimeOptions {
/** Paperclip's 0.13.1 patch: never persisted in ACPX session records. */
onAgentSpawn?: (meta: { pid: number; startedAt: string }) => Promise<void> | void;
onAgentStderr?: (chunk: string) => void;
/** Paperclip's 0.13.1 patch: observes transient ACP frames without persisting them. */
onAcpMessage?: (direction: "inbound" | "outbound", message: unknown) => void;
spawnCwd?: string;
}
export interface AcpxRuntimeHostOptions {
runtimeDirectory: string;
normalizedSessionId: string;
/** Provider-record key may rotate while Paperclip's normalized session stays stable. */
providerSessionKey?: string;
workingDirectory: string;
agent: QualifiedAcpxAgent;
model: string;
@@ -143,6 +151,8 @@ export class AcpxRuntimeHost {
readonly #root: string;
readonly #ephemeralCredentialFiles: string[];
readonly #agentProcessIdentity: AcpxRuntimeProcessIdentity | null;
readonly #trace: ProviderTraceFileSink | null;
readonly #traceState: { latestInboundFrameId: number | null };
#activeTurn: AcpRuntimeTurn | null = null;
#closed = false;
@@ -154,6 +164,8 @@ export class AcpxRuntimeHost {
root: string;
ephemeralCredentialFiles: string[];
agentProcessIdentity: AcpxRuntimeProcessIdentity | null;
trace: ProviderTraceFileSink | null;
traceState: { latestInboundFrameId: number | null };
}) {
this.#runtime = input.runtime;
this.#handle = input.handle;
@@ -162,6 +174,8 @@ export class AcpxRuntimeHost {
this.#root = input.root;
this.#ephemeralCredentialFiles = input.ephemeralCredentialFiles;
this.#agentProcessIdentity = input.agentProcessIdentity;
this.#trace = input.trace;
this.#traceState = input.traceState;
}
static async open(options: AcpxRuntimeHostOptions): Promise<AcpxRuntimeHost> {
@@ -178,6 +192,13 @@ export class AcpxRuntimeHost {
await mkdir(directory, { recursive: true, mode: 0o700 });
await chmod(directory, 0o700);
}
const trace = await createProviderTraceFileSink({
path: options.environment?.PAPERCLIP_PROVIDER_TRACE_PATH,
provider: `acpx:${options.agent}`,
channel: "typescript_acpx_native",
maxBytes: options.environment?.PAPERCLIP_PROVIDER_TRACE_MAX_BYTES,
});
const traceState = { latestInboundFrameId: null as number | null };
if (options.expectedIdentity) {
await verifyExpectedIdentity(options.expectedIdentity, root, options.normalizedSessionId, cwd, profile, options.model);
}
@@ -246,9 +267,14 @@ export class AcpxRuntimeHost {
...(options.agent === "claude" ? { CLAUDE_CONFIG_DIR: agentHome } : {}),
...(options.agent === "codex" ? { CODEX_HOME: agentHome } : {}),
PAPERCLIP_ACPX_PROFILE: options.agent,
PAPERCLIP_ACPX_ISOLATED_CONTEXT: "1",
}, options.agent);
const safeSessionEnvironment = withoutCredentials(agentEnvironment);
const nativeMcp = nativeMcpLaunchBinding(options.environment ?? process.env);
const runnerOwnedMcpServerNames = new Set([
"paperclip",
...(nativeMcp ? [nativeMcp.name] : []),
]);
const mcpServers: NonNullable<AcpRuntimeOptions["mcpServers"]> = options.agent === "pi" ? [] : [{
type: "http",
name: "paperclip",
@@ -281,16 +307,28 @@ export class AcpxRuntimeHost {
escalate: ["execute", "write", "edit", "delete", "move"],
defaultAction: "escalate",
},
elicitationModes: ["form", "url"],
// Paperclip routes durable user interaction through PRP semantic tools.
// Advertising ACP form elicitation to Codex diverts MCP-tool approvals
// into a separate UI callback that this headless runner does not own.
// With no generic elicitation capability advertised, Codex-ACP uses the
// governed permission channel where the runner-owned server identity is
// correlated and external tool calls remain subject to policy.
elicitationModes: options.agent === "codex" ? [] : ["form", "url"],
onPermissionRequest: async (request, context) => {
// Claude Code asks its ACP client to authorize every MCP invocation,
// Claude Code and Codex ask their ACP client to authorize MCP
// invocations,
// including calls to Paperclip's own authenticated semantic bridge.
// Those calls must reach PRP, whose attached run catalog and schema
// checks are the authority. Treating this duplicate provider prompt as
// a human approval prevents even read-only semantic tools from running.
// Built-in filesystem/process calls do not have this closed namespace
// and continue through the durable interactive permission flow.
if (options.agent === "claude" && isRunnerOwnedSemanticPermission(request)) {
if (shouldAutoApproveRunnerOwnedSemanticPermission(
options.agent,
request,
runnerOwnedMcpServerNames,
true,
)) {
return { outcome: "allow_once" };
}
return options.onPermissionRequest
@@ -299,6 +337,37 @@ export class AcpxRuntimeHost {
},
spawnEnvironment: () => ({ ...agentEnvironment }),
onAgentStderr: routeAgentStderr,
onAcpMessage: (direction, message) => {
try {
const raw = `${JSON.stringify(message)}\n`;
const nativeMethod = message && typeof message === "object" && !Array.isArray(message)
&& typeof (message as Record<string, unknown>).method === "string"
? (message as Record<string, unknown>).method as string
: undefined;
const frameId = trace?.frame({
direction: direction === "outbound" ? "client_to_provider" : "provider_to_client",
raw,
transport: "acp_json_rpc_object",
...(nativeMethod ? { nativeMethod } : {}),
});
if (direction === "inbound" && frameId !== null && frameId !== undefined) {
traceState.latestInboundFrameId = frameId;
}
if (frameId !== null && frameId !== undefined) {
trace?.interpretation({
frameId,
stage: "typescript_acpx_transport",
ruleId: direction === "outbound" ? "acpx.transport.outbound" : "acpx.transport.inbound",
disposition: direction === "outbound" ? "operator_only" : "generic",
reason: direction === "outbound"
? "Captured the ACP client request before provider delivery."
: "Captured the parsed ACP provider message before runtime-event normalization.",
});
}
} catch {
// Debug capture must never alter provider execution.
}
},
onAgentSpawn: async (meta) => {
agentProcessIdentity = { pid: meta.pid, processGroupId: null, startedAt: meta.startedAt };
await options.onSpawn?.(agentProcessIdentity);
@@ -309,7 +378,12 @@ export class AcpxRuntimeHost {
let handle: AcpRuntimeHandle | null = null;
try {
handle = await runtime.ensureSession({
sessionKey: profileSessionKey(options.normalizedSessionId, cwd, profile, options.model),
sessionKey: profileSessionKey(
options.providerSessionKey ?? options.normalizedSessionId,
cwd,
profile,
options.model,
),
agent: options.agent,
mode: "persistent",
cwd,
@@ -351,12 +425,15 @@ export class AcpxRuntimeHost {
root,
ephemeralCredentialFiles,
agentProcessIdentity,
trace,
traceState,
});
} catch (error) {
if (handle) {
await runtime.close({ handle, reason: "ACPX startup failed", discardPersistentState: false }).catch(() => {});
}
await bridge.close().catch(() => {});
await trace?.finish({ reason: "acpx_session_start_failed" });
await removeEphemeralCredentialFiles(ephemeralCredentialFiles);
throw error;
}
@@ -366,6 +443,22 @@ export class AcpxRuntimeHost {
return structuredClone(this.#identity);
}
latestInboundTraceFrameId(): number | null {
return this.#traceState.latestInboundFrameId;
}
correlateProviderFrame(frameId: number | null, eventIds: string[], runtimeEventType: string): void {
if (frameId === null || eventIds.length === 0) return;
this.#trace?.interpretation({
frameId,
stage: "typescript_acpx_runtime_normalization",
ruleId: `acpx.runtime.${runtimeEventType}`,
disposition: "mapped",
emittedEventIds: eventIds,
reason: `Normalized the ACP runtime ${runtimeEventType} event into canonical PRP.`,
});
}
async status(): Promise<AcpRuntimeStatus> {
if (!this.#runtime.getStatus) return {};
return normalizeQualifiedModelStatus(
@@ -425,6 +518,7 @@ export class AcpxRuntimeHost {
}
} finally {
await this.#bridge.close().catch(() => {});
await this.#trace?.finish();
await removeEphemeralCredentialFiles(this.#ephemeralCredentialFiles);
}
}
@@ -448,6 +542,15 @@ function isRunnerOwnedSemanticPermission(request: AcpPermissionRequest): boolean
const toolCall = raw.toolCall;
if (!toolCall || typeof toolCall !== "object" || Array.isArray(toolCall)) return false;
const call = toolCall as Record<string, unknown>;
const requestMeta = raw._meta;
const isMcpToolApproval = requestMeta && typeof requestMeta === "object" && !Array.isArray(requestMeta)
? (requestMeta as Record<string, unknown>).is_mcp_tool_approval === true
: false;
const rawInput = call.rawInput;
const serverName = rawInput && typeof rawInput === "object" && !Array.isArray(rawInput)
? (rawInput as Record<string, unknown>).serverName
: null;
if (isMcpToolApproval && serverName === "paperclip") return true;
const meta = call._meta;
const claudeCode = meta && typeof meta === "object" && !Array.isArray(meta)
? (meta as Record<string, unknown>).claudeCode
@@ -456,10 +559,35 @@ function isRunnerOwnedSemanticPermission(request: AcpPermissionRequest): boolean
? (claudeCode as Record<string, unknown>).toolName
: null;
return [call.name, call.title, metadataToolName].some((value) => (
typeof value === "string" && value.startsWith("mcp__paperclip__")
typeof value === "string"
&& (value.startsWith("mcp__paperclip__") || value.startsWith("mcp.paperclip."))
));
}
export function shouldAutoApproveRunnerOwnedSemanticPermission(
agent: QualifiedAcpxAgent,
request: AcpPermissionRequest,
runnerOwnedMcpServerNames: ReadonlySet<string> = new Set(["paperclip"]),
allConfiguredMcpServersAreRunnerOwned = false,
): boolean {
if (agent === "pi") return false;
const raw = request.raw as unknown as Record<string, unknown>;
const requestMeta = raw._meta;
const isMcpToolApproval = requestMeta && typeof requestMeta === "object" && !Array.isArray(requestMeta)
? (requestMeta as Record<string, unknown>).is_mcp_tool_approval === true
: false;
if (agent === "codex" && isMcpToolApproval && allConfiguredMcpServersAreRunnerOwned) return true;
const toolCall = raw.toolCall;
if (toolCall && typeof toolCall === "object" && !Array.isArray(toolCall)) {
const rawInput = (toolCall as Record<string, unknown>).rawInput;
if (rawInput && typeof rawInput === "object" && !Array.isArray(rawInput)) {
const serverName = (rawInput as Record<string, unknown>).serverName;
if (typeof serverName === "string" && runnerOwnedMcpServerNames.has(serverName)) return true;
}
}
return isRunnerOwnedSemanticPermission(request);
}
async function stageManagedCodexCredential(
agentHome: string,
options: AcpxRuntimeHostOptions,
@@ -689,6 +817,7 @@ function sanitizedAgentEnvironment(
const allowedExact = new Set([
"PATH", "LANG", "LANGUAGE", "TZ", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY",
"ALL_PROXY", "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS",
"PAPERCLIP_NATIVE_MCP_NAME", "PAPERCLIP_NATIVE_MCP_URL", "PAPERCLIP_NATIVE_MCP_TOKEN",
]);
const credentialNames = agent === "pi"
? ["OPENROUTER_API_KEY"]
packages/paperclip-runner/src/drivers/acpx/pi-paperclip-extension.test.ts
+8 -1
View File
@@ -40,7 +40,7 @@ describe("paperclip Pi extension", () => {
return new Response(JSON.stringify({
jsonrpc: "2.0",
id: request.id,
result: { content: [{ type: "text", text: JSON.stringify({ decision: "accept" }) }] },
result: { content: [{ type: "text", text: JSON.stringify({ decision: "accept_for_session" }) }] },
}), { status: 200 });
}
return request.id === undefined
@@ -61,6 +61,8 @@ describe("paperclip Pi extension", () => {
expect(JSON.stringify(tools)).not.toContain("__paperclip_runtime_permission");
expect(await toolCallHandler!({ toolName: "read", toolCallId: "read-1", input: { path: "/etc/passwd" } })).toMatchObject({ block: true });
expect(await toolCallHandler!({ toolName: "bash", toolCallId: "bash-1", input: { command: "printf hello" } })).toBeUndefined();
expect(await toolCallHandler!({ toolName: "bash", toolCallId: "bash-relative", input: { command: "mkdir -p nested/dir && printf hello > nested/dir/file.txt" } })).toBeUndefined();
expect(await toolCallHandler!({ toolName: "write", toolCallId: "write-nested", input: { path: "new/nested/file.txt" } })).toBeUndefined();
const permissionCall = requests.find((request) =>
request.method === "tools/call"
@@ -68,7 +70,12 @@ describe("paperclip Pi extension", () => {
);
const permissionArguments = ((permissionCall?.params as Record<string, unknown>)?.arguments ?? {}) as Record<string, unknown>;
expect(permissionArguments.target).toMatch(/^command-sha256:[a-f0-9]{64}$/);
expect(requests.filter((request) =>
request.method === "tools/call"
&& (request.params as Record<string, unknown>)?.name === "__paperclip_runtime_permission"
)).toHaveLength(1);
expect(JSON.stringify(requests)).not.toContain("printf hello");
expect(await toolCallHandler!({ toolName: "bash", toolCallId: "bash-2", input: { command: "cat /etc/passwd" } })).toMatchObject({ block: true });
expect(await toolCallHandler!({ toolName: "bash", toolCallId: "bash-parent", input: { command: "cat safe/../../etc/passwd" } })).toMatchObject({ block: true });
});
});
packages/paperclip-runner/src/drivers/acpx/pi-paperclip-extension.ts
+42 -5
View File
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import { existsSync, realpathSync } from "node:fs";
import { lstatSync, realpathSync } from "node:fs";
import { dirname, isAbsolute, relative, resolve } from "node:path";
import type { ExtensionAPI, ToolDefinition } from "@earendil-works/pi-coding-agent";
@@ -38,6 +38,7 @@ export default async function paperclipPiExtension(pi: ExtensionAPI): Promise<vo
.filter((value): value is string => typeof value === "string" && value.trim().length > 0)
.map((value) => resolve(value));
const rpc = createMcpClient(endpoint, token);
let sessionAllowsGovernedTools = false;
await initializeMcpClient(rpc, "paperclip-pi-extension");
await registerMcpTools(pi, rpc, { privatePermissionTool: true, prefix: "" });
const assignedUrl = process.env.PAPERCLIP_NATIVE_MCP_URL?.trim();
@@ -62,12 +63,14 @@ export default async function paperclipPiExtension(pi: ExtensionAPI): Promise<vo
if (command !== null && unsafeCommandPath(command, workspace, runtimeRoot)) {
return { block: true, reason: "Paperclip denied a command that names a protected or outside-workspace path." };
}
if (sessionAllowsGovernedTools) return undefined;
const target = command === null ? pathTarget : `command-sha256:${createHash("sha256").update(command).digest("hex")}`;
const response = await rpc("tools/call", {
name: PRIVATE_PERMISSION_TOOL,
arguments: { operation: event.toolName, target: target === null ? null : relative(workspace, resolve(workspace, target)), inputClass: event.toolName === "bash" ? "command" : "workspace_mutation" },
}, `permission:${event.toolCallId}`);
const decision = decodedResult(response).decision;
if (decision === "accept_for_session") sessionAllowsGovernedTools = true;
return decision === "accept" || decision === "accept_for_session" ? undefined : { block: true, reason: decision === "cancel" ? "Paperclip cancelled this operation." : "Paperclip declined this operation." };
});
}
@@ -167,14 +170,42 @@ function inside(root: string, candidate: string): boolean {
return result === "" || (!result.startsWith("..") && !isAbsolute(result));
}
function pathEntryExists(candidate: string): boolean {
try {
lstatSync(candidate);
return true;
} catch {
return false;
}
}
function nearestExistingAncestor(candidate: string): string | null {
let current = candidate;
while (!pathEntryExists(current)) {
const parent = dirname(current);
if (parent === current) return null;
current = parent;
}
return current;
}
function safeWorkspaceTarget(workspace: string, runtimeRoot: string, readOnlyRoots: string[], candidate: string, mayCreate: boolean): boolean {
const absolute = isAbsolute(candidate) ? resolve(candidate) : resolve(workspace, candidate);
const authorizedRoot = [workspace, ...readOnlyRoots].find((root) => inside(root, absolute));
if (!authorizedRoot || (inside(runtimeRoot, absolute) && !readOnlyRoots.some((root) => inside(root, absolute)))) return false;
try {
const existing = existsSync(absolute) ? absolute : mayCreate ? dirname(absolute) : absolute;
const existing = pathEntryExists(absolute)
? absolute
: mayCreate
? nearestExistingAncestor(dirname(absolute))
: absolute;
if (existing === null) return false;
const physical = realpathSync(existing);
return inside(authorizedRoot, physical) && (!inside(runtimeRoot, physical) || readOnlyRoots.some((root) => inside(root, physical)));
const physicalAuthorizedRoot = realpathSync(authorizedRoot);
const physicalRuntimeRoot = realpathSync(runtimeRoot);
const physicalReadOnlyRoots = readOnlyRoots.map((root) => realpathSync(root));
return inside(physicalAuthorizedRoot, physical)
&& (!inside(physicalRuntimeRoot, physical) || physicalReadOnlyRoots.some((root) => inside(root, physical)));
} catch {
return false;
}
@@ -188,8 +219,14 @@ function commandText(input: unknown): string | null {
}
function unsafeCommandPath(command: string, workspace: string, runtimeRoot: string): boolean {
if (/(^|[\s'"`=])~(?:\/|\s|$)/.test(command) || /(^|[\s'"`=])\.\.(?:\/|\s|$)/.test(command)) return true;
const absolutePaths = command.match(/\/(?:[^\s'"`;&|()<>]|\\.)+/g) ?? [];
const shellBoundary = String.raw`[\s'"\x60=;&|()<>]`;
if (
new RegExp(`(^|${shellBoundary})~(?:/|${shellBoundary}|$)`).test(command)
|| new RegExp(`(^|${shellBoundary}|/)\\.\\.(?:/|${shellBoundary}|$)`).test(command)
) return true;
const absolutePaths = [...command.matchAll(new RegExp(`(?:^|${shellBoundary})(/(?:[^\\s'"\\x60;&|()<>]|\\\\.)+)`, "g"))]
.map((match) => match[1]!)
.filter((candidate) => candidate.length > 0);
return absolutePaths.some((candidate) => !inside(workspace, candidate) || inside(runtimeRoot, candidate));
}
packages/paperclip-runner/src/drivers/acpx/qualified-profiles.ts
+1 -1
View File
@@ -74,7 +74,7 @@ export const QUALIFIED_ACPX_PROFILES: Readonly<Record<QualifiedAcpxAgent, Qualif
agentServerVersion: "1.6.2",
agentRuntimePackage: null,
agentRuntimeVersion: null,
commandDigest: "sha256:8c7fc8af156596668a95ce23d52309f70ad576e75bac6dc209d30378bdbb8ebe",
commandDigest: "sha256:94049b3e3c3aee87de62703786e4fa81d031d7bd979f99bdf516d84f28791a79",
qualificationModel: "gpt-5.6-sol",
reportedModelId: "gpt-5.6-sol",
permissionPolicy: "interactive",
packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.ts
+3
View File
@@ -1,4 +1,5 @@
import type { QualifiedAcpxAgent } from "./qualified-profiles.js";
import type { NativeRuntimeContextSnapshot } from "../../contracts/runtime-context.js";
import {
GENERATED_ACPX_SIDECAR_COMMANDS,
GENERATED_ACPX_SIDECAR_PROTOCOL_VERSION,
@@ -39,6 +40,8 @@ export interface AcpxSidecarOpenParams {
workingDirectory: string;
agent: QualifiedAcpxAgent;
model: string;
systemInstructions: string;
runtimeContext: NativeRuntimeContextSnapshot | null;
tools: readonly Readonly<Record<string, unknown>>[];
expectedIdentity?: AcpxExpectedSessionIdentity;
}
packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test.ts
+143
View File
@@ -449,6 +449,35 @@ describe("Codex app-server Codex driver", () => {
});
});
it("places Paperclip runtime instructions in Codex's system channel and enables only selected skill instructions", async () => {
const transport = new FakeCodexTransport();
const baseInstructions = [
"You are running as a Paperclip agent.",
"Follow the attached AGENTS.md instructions.",
"Read-only instruction sibling root: /paperclip/context/instructions",
].join("\n\n");
const driver = makeDriver([transport], {
baseInstructions,
includeSkillInstructions: true,
});
await driver.openSession({
runId: "run-runtime-context",
normalizedSessionId: "normalized-runtime-context",
workingDirectory: "/workspace",
});
const threadStart = transport.calls.find((call) => call.method === "thread/start");
expect(threadStart?.params).toMatchObject({
baseInstructions,
config: {
"skills.include_instructions": true,
include_apps_instructions: false,
},
});
expect(JSON.stringify(threadStart?.params.input ?? null)).not.toContain(baseInstructions);
});
it("passes the common typed-event contract and reports one provider turn terminal", async () => {
const transport = new FakeCodexTransport();
const driver = makeDriver([transport]);
@@ -1169,6 +1198,12 @@ describe("Codex app-server Codex driver", () => {
message: { role: "user", text: "Stay concise." },
correlationId: "queued-comment-1",
});
transport.push("item/completed", {
kind: "steering_acknowledgement",
status: "acknowledged",
commandId: "runnerd-steer-command",
});
await new Promise((resolve) => setTimeout(resolve, 0));
await session.steer?.({
turnId,
message: { role: "user", text: "Stay concise." },
@@ -1257,6 +1292,8 @@ describe("Codex app-server Codex driver", () => {
}),
]),
);
expect(observed.some((event) => event.eventType === "session.failed"))
.toBe(false);
await session.close({ reason: "fixture complete" });
});
@@ -1508,6 +1545,93 @@ describe("Codex app-server Codex driver", () => {
await session.close({ reason: "fixture complete" });
});
it("replays the DOT-185 requestUserInput shape as one canonical three-question request", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({
runId: "run-dot-185",
normalizedSessionId: "normalized-dot-185",
workingDirectory: "/workspace",
});
const { turnId } = await session.startTurn({
message: { role: "user", text: "Ask the deployment questions." },
});
const nativeResponse = transport.invoke({
id: "dot-185-request",
method: "item/tool/requestUserInput",
params: {
threadId: "thread-1",
turnId,
itemId: "dot-185-item",
questions: [
{
id: "environment",
header: "Environment",
question: "Where should we deploy?",
options: [
{ label: "Staging", description: "Deploy to staging first." },
{ label: "Production", description: "Deploy directly to production." },
],
isOther: true,
},
{
id: "regions",
header: "Regions",
question: "Which regions should receive the release?",
options: [{ label: "US" }, { label: "EU" }],
multiSelect: true,
},
{
id: "notes",
header: "Notes",
question: "Anything else we should know?",
required: false,
},
],
},
});
await new Promise((resolve) => setTimeout(resolve, 0));
const [pending] = session.pendingRuntimeRequests?.() ?? [];
expect(pending?.input).toMatchObject({
schema: "paperclip.question_set.v1",
questions: [
{
id: "environment",
answerMode: "single_select",
customAnswer: { enabled: true },
options: [
{ id: "option-1", label: "Staging", description: "Deploy to staging first." },
{ id: "option-2", label: "Production", description: "Deploy directly to production." },
],
},
{ id: "regions", answerMode: "multi_select" },
{ id: "notes", answerMode: "text", required: false },
],
});
await session.resolveRuntimeRequest?.({
requestId: "dot-185-request",
turnId,
resolution: {
action: "submit",
response: {
schema: "paperclip.question_response.v1",
answers: {
environment: { selectedOptionIds: ["option-1"] },
regions: { selectedOptionIds: ["option-1", "option-2"] },
notes: { text: "Ship during the maintenance window." },
},
},
},
});
expect(await nativeResponse).toEqual({
answers: {
environment: { answers: ["Staging"] },
regions: { answers: ["US", "EU"] },
notes: { answers: ["Ship during the maintenance window."] },
},
});
await session.close({ reason: "fixture complete" });
});
it("emits one canonical outcome payload for every terminal runtime request fact", async () => {
const transport = new FakeCodexTransport();
const session = await makeDriver([transport]).openSession({
@@ -1643,6 +1767,10 @@ describe("Codex app-server Codex driver", () => {
const request = {
id: 1,
method: "item/tool/call",
paperclipTrace: {
sourceEventId: "event_runner_000101",
sourceEventType: "semantic_tool.input",
},
params: {
threadId: "thread-1",
turnId: "turn-1",
@@ -1652,6 +1780,21 @@ describe("Codex app-server Codex driver", () => {
},
};
expect(await transport.invoke(request)).toMatchObject({ success: true });
const resultMapping = transport.traceInterpretations.find(
(entry) =>
entry.sourceEventId === "event_runner_000101" &&
entry.providerMethod === "item/tool/call",
);
expect(resultMapping).toMatchObject({
sourceEventType: "semantic_tool.input",
disposition: "mapped",
});
expect(resultMapping?.emittedEventIds).toHaveLength(2);
expect(resultMapping?.emittedEventIds).toEqual(
expect.arrayContaining([
expect.stringContaining(":run-result:"),
]),
);
expect(
await transport.invoke({
...request,
packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts
+288 -7
View File
@@ -10,6 +10,9 @@ import type {
HarnessRuntimeRequest,
HarnessRuntimeRequestKind,
HarnessRuntimeRequestResolution,
PaperclipQuestion,
PaperclipQuestionResponse,
PaperclipQuestionSet,
HarnessThreadGoal,
HarnessThreadLineageEntry,
OpenHarnessSessionInput,
@@ -23,14 +26,17 @@ import {
HarnessReconciliationError,
HarnessStaleTurnError,
harnessRuntimeRequestOutcome,
PAPERCLIP_QUESTION_SET_SCHEMA,
PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2,
parseHarnessRuntimeRequestResolution,
} from "../../contracts/harness-driver.js";
import {
CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
CODEX_BLOCK_TOOL_NAME,
CODEX_CODEX_PROTOCOL_VERSION,
CODEX_COMPLETION_TOOL_NAME,
CODEX_RESULT_OUTPUT_SCHEMA,
CODEX_RESULT_PROVIDER_INPUT_SCHEMA,
CODEX_SEMANTIC_TOOL_NAMES,
CODEX_SKILLLESS_BASE_INSTRUCTIONS,
type CodexModelContextSnapshot,
@@ -246,7 +252,7 @@ function finishToolSpec(): Record<string, unknown> {
return {
name: CODEX_COMPLETION_TOOL_NAME,
description: "Return the one semantic completion result for this task.",
inputSchema: CODEX_RESULT_OUTPUT_SCHEMA,
inputSchema: CODEX_RESULT_PROVIDER_INPUT_SCHEMA,
};
}
@@ -255,7 +261,7 @@ function blockToolSpec(): Record<string, unknown> {
name: CODEX_BLOCK_TOOL_NAME,
description:
"Return the one semantic result when the task cannot continue.",
inputSchema: CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
inputSchema: CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
};
}
@@ -1013,8 +1019,13 @@ class CodexHarnessSession implements HarnessSession {
}
this.#terminalTurns.set(terminal.turnId, terminal.fingerprint);
}
if (this.#activeTurnId && this.#terminalTurns.has(this.#activeTurnId)) {
this.#activeTurnId = null;
}
this.#terminal =
this.#conversationMode === "task" && this.#terminalTurns.size > 0;
this.#conversationMode === "task"
&& this.#terminalTurns.size > 0
&& this.#result !== null;
this.#transport.setServerRequestHandler((request) =>
this.#handleServerRequest(request),
);
@@ -1330,6 +1341,7 @@ class CodexHarnessSession implements HarnessSession {
const resolution = parseHarnessRuntimeRequestResolution(
pending.request.requestKind,
input.resolution,
pending.request.input,
);
const response = runtimeRequestResponse(pending.request, resolution);
this.#pendingRuntimeRequests.delete(input.requestId);
@@ -1749,6 +1761,17 @@ class CodexHarnessSession implements HarnessSession {
pending.settle(safeRequestResponse(pending.request.method, "cancel"));
return;
}
if (
notification.method === "item/completed"
&& text(params.kind) === "steering_acknowledgement"
&& Object.keys(item).length === 0
) {
// runnerd persists its own command acknowledgement as a canonical PRP
// item. The request() call is already the authoritative acknowledgement
// and steer() emits the user-visible item with the active turn binding.
// Do not reinterpret this transport-level echo as an unbound Codex item.
return;
}
if (threadId !== this.#opened.threadId) {
this.#failProtocol(
"thread_binding_mismatch",
@@ -2006,6 +2029,55 @@ class CodexHarnessSession implements HarnessSession {
async #handleServerRequest(
request: CodexRpcServerRequest,
): Promise<Record<string, unknown>> {
const sourceSequenceBefore = this.#sourceSequence;
let rejected = false;
try {
const response = await this.#handleServerRequestBody(request);
rejected = response.success === false;
return response;
} catch (error) {
rejected = true;
throw error;
} finally {
const correlation = request.paperclipTrace;
if (correlation !== undefined) {
const emittedEventIds: string[] = [];
for (
let sourceSeq = sourceSequenceBefore + 1;
sourceSeq <= this.#sourceSequence;
sourceSeq += 1
) {
emittedEventIds.push(
`${this.#runnerInstanceId}:${this.#runId}:${sourceSeq}`,
);
}
try {
this.#transport.recordTraceInterpretation?.({
sourceEventId: correlation.sourceEventId,
sourceEventType: correlation.sourceEventType,
providerMethod: request.method,
disposition: rejected
? "rejected"
: emittedEventIds.length > 0
? "mapped"
: "ignored",
emittedEventIds,
reason: rejected
? "Codex driver rejected the correlated provider server request"
: emittedEventIds.length > 0
? "Codex driver mapped the correlated provider server request into canonical PRP events"
: "Codex driver accepted the correlated provider server request without emitting a canonical PRP event",
});
} catch {
// Trace delivery is deliberately outside run authority.
}
}
}
}
async #handleServerRequestBody(
request: CodexRpcServerRequest,
): Promise<Record<string, unknown>> {
if (request.method === "item/tool/call") {
const tool = text(request.params.tool);
@@ -2165,6 +2237,7 @@ class CodexHarnessSession implements HarnessSession {
);
return safeRequestResponse(request.method);
}
const input = codexQuestionSet(request.method, request.params);
const runtimeRequest: HarnessRuntimeRequest = {
requestId,
requestKind,
@@ -2174,11 +2247,17 @@ class CodexHarnessSession implements HarnessSession {
status: "pending",
prompt: runtimeRequestPrompt(requestKind, request.params),
details: record(redactCodexValue(boundedCodexValue(request.params))),
...(input !== null ? { input } : {}),
origin: {
adapter: "codex-app-server",
provider: "codex",
method: request.method,
},
};
this.#emit(
"runtime_request.created",
{
request: { ...runtimeRequest, type: runtimeRequest.method },
request: runtimeRequestProtocolPayload(runtimeRequest),
},
{
turnId: requestTurnId,
@@ -2324,6 +2403,8 @@ class CodexHarnessSession implements HarnessSession {
? "turn.cancelled"
: "turn.completed";
this.#cancelPendingRequests("turn_terminal");
this.#activeTurnId = null;
this.#turnStarted = false;
this.#emit(
eventType,
boundedPayload({
@@ -2332,8 +2413,6 @@ class CodexHarnessSession implements HarnessSession {
}),
{ turnId },
);
this.#activeTurnId = null;
this.#turnStarted = false;
this.#finalize(status);
}
@@ -2710,6 +2789,198 @@ function runtimeRequestPrompt(
return labels[kind];
}
function stableQuestionId(value: unknown, index: number): string {
const candidate = text(value).trim();
return candidate.length > 0 ? candidate.slice(0, 160) : `question-${index + 1}`;
}
function codexOptions(value: unknown): NonNullable<PaperclipQuestion["options"]> | undefined {
if (!Array.isArray(value)) return undefined;
return value.slice(0, 128).map((rawOption, index) => {
const option = record(rawOption);
const label = text(option.label, text(option.value, text(rawOption))).slice(0, 1_000);
return {
id: stableQuestionId(option.id, index).replace(/^question-/, "option-"),
label: label || `Option ${index + 1}`,
...(text(option.description).length > 0
? { description: boundedText(redactCodexDiagnostic(text(option.description))) }
: {}),
};
});
}
function jsonSchemaOptions(schema: Record<string, unknown>): NonNullable<PaperclipQuestion["options"]> {
const values = Array.isArray(schema.enum)
? schema.enum
: Array.isArray(schema.oneOf)
? schema.oneOf.map((entry) => record(entry).const)
: [];
return values.slice(0, 128).map((value, index) => {
const oneOf = Array.isArray(schema.oneOf) ? record(schema.oneOf[index]) : {};
return {
id: `option-${index + 1}`,
label: text(oneOf.title, typeof value === "string" ? value : JSON.stringify(value)).slice(0, 1_000),
...(text(oneOf.description).length > 0
? { description: boundedText(text(oneOf.description)) }
: {}),
};
});
}
/** Codex-native requests are converted once, before they enter PRP. */
function codexQuestionSet(method: string, params: Record<string, unknown>): PaperclipQuestionSet | null {
if (method === "item/tool/requestUserInput" || method === "tool/requestUserInput") {
if (!Array.isArray(params.questions) || params.questions.length === 0) return null;
const questions = params.questions.slice(0, 64).map((rawQuestion, index): PaperclipQuestion => {
const question = record(rawQuestion);
const options = codexOptions(question.options);
return {
id: stableQuestionId(question.id, index),
...(text(question.header).length > 0 ? { header: boundedText(text(question.header), "", 1_000) } : {}),
prompt: boundedText(text(question.question, text(question.prompt, `Question ${index + 1}`))),
...(text(question.description).length > 0 ? { helpText: boundedText(text(question.description)) } : {}),
required: question.required !== false,
answerMode: options && options.length > 0
? question.multiSelect === true || question.multiple === true ? "multi_select" : "single_select"
: "text",
...(options && options.length > 0 ? { options } : {}),
...(question.isOther === true || question.allowOther === true
? { customAnswer: { enabled: true, label: "Other", placeholder: "Enter another answer" } }
: {}),
...(!(options && options.length > 0) && (typeof question.minLength === "number" || typeof question.maxLength === "number")
? { textValidation: {
...(typeof question.minLength === "number" ? { minLength: question.minLength } : {}),
...(typeof question.maxLength === "number" ? { maxLength: question.maxLength } : {}),
} }
: {}),
};
});
return {
schema: PAPERCLIP_QUESTION_SET_SCHEMA,
title: text(params.title, "Codex needs your input"),
...(text(params.description).length > 0 ? { description: boundedText(text(params.description)) } : {}),
submitLabel: text(params.submitLabel, "Submit answers"),
questions,
};
}
if (method !== "mcpServer/elicitation/request") return null;
const requestedSchema = record(params.requestedSchema ?? params.schema);
const properties = record(requestedSchema.properties);
const required = new Set(Array.isArray(requestedSchema.required) ? requestedSchema.required.filter((entry): entry is string => typeof entry === "string") : []);
const questions = Object.entries(properties).slice(0, 64).map(([id, rawProperty]): PaperclipQuestion => {
const property = record(rawProperty);
const propertyType = text(property.type);
const itemSchema = record(property.items);
const selectSchema = propertyType === "array" ? itemSchema : property;
const options = propertyType === "boolean"
? [{ id: "true", label: "Yes" }, { id: "false", label: "No" }]
: jsonSchemaOptions(selectSchema);
const answerMode: PaperclipQuestion["answerMode"] = propertyType === "array" && options.length > 0
? "multi_select"
: options.length > 0
? "single_select"
: "text";
const inputType = propertyType === "integer" ? "integer" : propertyType === "number" ? "number" : "text";
return {
id,
...(text(property.title).length > 0 ? { header: boundedText(text(property.title), "", 1_000) } : {}),
prompt: boundedText(text(property.title, id)),
...(text(property.description).length > 0 ? { helpText: boundedText(text(property.description)) } : {}),
required: required.has(id),
answerMode,
...(options.length > 0 ? { options } : {}),
...(answerMode === "text" ? { textValidation: {
inputType,
...(typeof property.minLength === "number" ? { minLength: property.minLength } : {}),
...(typeof property.maxLength === "number" ? { maxLength: property.maxLength } : {}),
...(typeof property.minimum === "number" ? { minimum: property.minimum } : {}),
...(typeof property.maximum === "number" ? { maximum: property.maximum } : {}),
...(typeof property.pattern === "string" ? { pattern: property.pattern } : {}),
} } : {}),
};
});
if (questions.length === 0) return null;
return {
schema: PAPERCLIP_QUESTION_SET_SCHEMA,
title: "A tool needs your input",
...(text(params.message).length > 0 ? { description: boundedText(text(params.message)) } : {}),
submitLabel: "Submit",
questions,
};
}
function runtimeRequestProtocolPayload(request: HarnessRuntimeRequest): Record<string, unknown> {
if (request.input !== undefined) {
return {
schema: PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2,
requestKind: "runtime",
requestId: request.requestId,
type: "input",
status: request.status,
prompt: request.prompt,
input: structuredClone(request.input),
origin: structuredClone(request.origin),
turnId: request.turnId,
itemId: request.itemId,
};
}
return { ...request, type: request.method };
}
function canonicalCodexAnswers(
request: HarnessRuntimeRequest,
response: PaperclipQuestionResponse,
): Record<string, { answers: string[] }> {
const result: Record<string, { answers: string[] }> = {};
for (const question of request.input?.questions ?? []) {
const answer = response.answers[question.id];
if (answer === undefined) continue;
const labels = (answer.selectedOptionIds ?? []).map((optionId) =>
question.options?.find((option) => option.id === optionId)?.label,
).filter((label): label is string => typeof label === "string");
if (answer.text !== undefined) labels.push(answer.text);
if (answer.customText !== undefined) labels.push(answer.customText);
result[question.id] = { answers: labels };
}
return result;
}
function jsonSchemaOptionValue(schema: Record<string, unknown>, optionId: string): unknown {
if (optionId === "true") return true;
if (optionId === "false") return false;
const index = Number(optionId.match(/^option-(\d+)$/)?.[1] ?? "0") - 1;
if (index < 0) return optionId;
if (Array.isArray(schema.enum)) return schema.enum[index];
if (Array.isArray(schema.oneOf)) return record(schema.oneOf[index]).const;
return optionId;
}
function canonicalElicitationContent(
request: HarnessRuntimeRequest,
response: PaperclipQuestionResponse,
): Record<string, unknown> {
const requestedSchema = record(request.details.requestedSchema ?? request.details.schema);
const properties = record(requestedSchema.properties);
const content: Record<string, unknown> = {};
for (const question of request.input?.questions ?? []) {
const answer = response.answers[question.id];
if (answer === undefined) continue;
const property = record(properties[question.id]);
const itemSchema = record(property.items);
if (question.answerMode === "text") {
const value = answer.text ?? "";
content[question.id] = property.type === "integer" || property.type === "number" ? Number(value) : value;
} else if (question.answerMode === "multi_select") {
content[question.id] = (answer.selectedOptionIds ?? []).map((optionId) => jsonSchemaOptionValue(itemSchema, optionId));
} else {
const optionId = answer.selectedOptionIds?.[0];
if (optionId !== undefined) content[question.id] = jsonSchemaOptionValue(property, optionId);
else if (answer.customText !== undefined) content[question.id] = answer.customText;
}
}
return content;
}
/**
* Maps an already-validated resolution onto the provider's response shape.
* `parseHarnessRuntimeRequestResolution` is the only gate on shape, so every
@@ -2748,6 +3019,9 @@ function runtimeRequestResponse(
};
}
if (request.requestKind === "user_input") {
if (resolution.action === "submit" && "response" in resolution) {
return { answers: canonicalCodexAnswers(request, resolution.response) };
}
if (resolution.action !== "submit" || !("answers" in resolution)) {
// Declines and cancels are the only non-submit answers the validator
// lets through, and neither carries form data.
@@ -2755,6 +3029,13 @@ function runtimeRequestResponse(
}
return { answers: structuredClone(resolution.answers) };
}
if (resolution.action === "submit" && "response" in resolution) {
return {
action: "accept",
content: canonicalElicitationContent(request, resolution.response),
_meta: null,
};
}
if (resolution.action === "submit" && "content" in resolution) {
return {
action: "accept",
packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts
+1
View File
@@ -92,6 +92,7 @@ describe("OpenCode MCP bridge", () => {
expect(canonicalOpenCodeMcpToolName("paperclip_documents.read")).toBe("documents.read");
expect(canonicalOpenCodeMcpToolName("paperclip__documents.read")).toBe("documents.read");
expect(canonicalOpenCodeMcpToolName("paperclip.documents.read")).toBe("documents.read");
expect(canonicalOpenCodeMcpToolName("paperclip_paperclip_finish")).toBe("paperclip_finish");
});
it("validates inputs before dispatch", async () => {
packages/paperclip-runner/src/drivers/opencode/mcp-bridge.ts
+8 -8
View File
@@ -3,11 +3,11 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse }
import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js";
import {
CODEX_BLOCK_RESULT_OUTPUT_SCHEMA,
CODEX_BLOCK_TOOL_NAME,
CODEX_COMPLETION_TOOL_NAME,
CODEX_RESULT_OUTPUT_SCHEMA,
} from "../../contracts/codex.js";
PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA,
PRP_BLOCK_TOOL_NAME,
PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA,
PRP_COMPLETION_TOOL_NAME,
} from "../../contracts/completion-result.js";
export interface OpenCodeMcpToolDefinition {
name: string;
@@ -43,7 +43,7 @@ const DEFAULT_MAX_BODY_BYTES = 1_048_576;
export function canonicalOpenCodeMcpToolName(name: string): string {
const trimmed = name.trim();
if (trimmed === CODEX_COMPLETION_TOOL_NAME || trimmed === CODEX_BLOCK_TOOL_NAME) return trimmed;
if (trimmed === PRP_COMPLETION_TOOL_NAME || trimmed === PRP_BLOCK_TOOL_NAME) return trimmed;
for (const prefix of ["paperclip__", "paperclip_", "paperclip."]) {
if (trimmed.startsWith(prefix)) return trimmed.slice(prefix.length);
}
@@ -113,8 +113,8 @@ function normalizeTools(
seen.add(name);
}
if (includeTerminalTools) for (const terminal of [
{ name: CODEX_COMPLETION_TOOL_NAME, description: "Return the semantic completion result.", inputSchema: CODEX_RESULT_OUTPUT_SCHEMA },
{ name: CODEX_BLOCK_TOOL_NAME, description: "Return the semantic blocked result.", inputSchema: CODEX_BLOCK_RESULT_OUTPUT_SCHEMA },
{ name: PRP_COMPLETION_TOOL_NAME, description: "Return the semantic completion result.", inputSchema: PRP_COMPLETION_RESULT_PROVIDER_INPUT_SCHEMA },
{ name: PRP_BLOCK_TOOL_NAME, description: "Return the semantic blocked result.", inputSchema: PRP_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA },
]) {
if (!seen.has(terminal.name)) tools.push(terminal);
}
packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts
+461 -6
View File
@@ -1,15 +1,74 @@
import { chmod, mkdtemp, readFile, rm, stat } from "node:fs/promises";
import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
PAPERCLIP_EXECUTION_PROMPT,
PAPERCLIP_EXECUTION_PROMPT_REVISION,
canonicalNativeRuntimeContextDigest,
nativeRuntimePromptDigest,
type NativeRuntimeContextSnapshot,
} from "../../contracts/runtime-context.js";
import { OpenCodeServerDriver, openCodeServerDriverInternals } from "./opencode-server-driver.js";
const roots: string[] = [];
const fixture = resolve("test/fixtures/fake-opencode-server.mjs");
function runtimeContext(skillRoot: string, instructionRoot: string): NativeRuntimeContextSnapshot {
const digest = "0".repeat(64);
const value = {
prompt: {
revision: PAPERCLIP_EXECUTION_PROMPT_REVISION,
text: PAPERCLIP_EXECUTION_PROMPT,
digest: nativeRuntimePromptDigest(),
},
instructions: {
entryPath: "AGENTS.md",
bundle: {
schema: NATIVE_RUNTIME_ASSET_SCHEMA,
digest,
manifestDigest: digest,
rootPath: instructionRoot,
fileCount: 2,
totalBytes: 2,
},
},
skills: [{
key: "company/assigned",
runtimeName: "assigned",
versionId: "version-1",
bundle: {
schema: NATIVE_RUNTIME_ASSET_SCHEMA,
digest,
manifestDigest: digest,
rootPath: skillRoot,
fileCount: 2,
totalBytes: 2,
},
}],
mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" },
} satisfies Omit<NativeRuntimeContextSnapshot, "aggregateDigest">;
return { ...value, aggregateDigest: canonicalNativeRuntimeContextDigest(value) };
}
async function makeWritable(root: string): Promise<void> {
const info = await lstat(root).catch(() => null);
if (!info) return;
if (info.isDirectory()) {
await chmod(root, 0o700);
for (const entry of await readdir(root)) await makeWritable(join(root, entry));
} else {
await chmod(root, 0o600);
}
}
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
await Promise.all(roots.splice(0).map(async (root) => {
await makeWritable(root);
await rm(root, { recursive: true, force: true });
}));
});
describe("OpenCodeServerDriver", () => {
@@ -18,6 +77,7 @@ describe("OpenCodeServerDriver", () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-"));
roots.push(root, workspace);
const tracePath = join(root, "provider-trace.ndjson");
const spawns: Array<{ pid: number; processGroupId: number | null }> = [];
const diagnostics: string[] = [];
const driver = new OpenCodeServerDriver({
@@ -29,6 +89,8 @@ describe("OpenCodeServerDriver", () => {
OPENROUTER_API_KEY: "test-openrouter-key",
PAPERCLIP_API_KEY: "must-not-leak",
UNRELATED_SECRET: "must-not-leak",
PAPERCLIP_PROVIDER_TRACE_PATH: tracePath,
PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024),
},
onSpawn: async (meta) => { spawns.push(meta); },
onDiagnostic: (message) => diagnostics.push(message),
@@ -43,6 +105,16 @@ describe("OpenCodeServerDriver", () => {
expect(events).toContainEqual(expect.objectContaining({ eventType: "run.result.proposed" }));
expect(events.filter((event) => event.eventType === "item.delta")).toHaveLength(1);
expect(events.find((event) => event.eventType === "item.delta")?.payload.text).toBe("done [guide](guide.md)");
expect(events.find(
(event) => event.eventType === "item.completed" && event.payload.kind === "agentMessage",
)?.payload).toMatchObject({
channel: "final",
providerPhase: "final_answer",
text: "done [guide](guide.md)",
});
expect(events.filter(
(event) => event.eventType === "item.completed" && event.payload.kind === "usage",
)).toHaveLength(1);
expect(events.find((event) => event.eventType === "workspace.change.updated")?.payload)
.toMatchObject({ schema: "paperclip.workspace.diff.v1", source: "harness_reported", totals: { files: 1 } });
expect(events.find((event) => event.eventType === "workspace.diff.recorded")?.payload)
@@ -60,23 +132,44 @@ describe("OpenCodeServerDriver", () => {
await session.interrupt?.({ turnId: turn.turnId });
const snapshot = await session.snapshot();
expect(snapshot.driverKind).toBe("opencode_server");
const sessionRoot = join(root, "normalized_1");
expect((await stat(sessionRoot)).mode & 0o777).toBe(0o700);
expect((await stat(join(sessionRoot, "config", "opencode", "opencode.json"))).mode & 0o777).toBe(0o600);
const config = await readFile(join(sessionRoot, "config", "opencode", "opencode.json"), "utf8");
await session.close({ reason: "test" });
const recovered = await driver.recoverSession?.(snapshot);
expect(recovered).toMatchObject({ recovered: true });
await expect(recovered?.session?.read?.()).resolves.toMatchObject({ sessionId: "ses_fake_1", messages: [] });
await recovered?.session?.close({ reason: "recovery-test" });
const sessionRoot = join(root, "normalized_1");
expect((await stat(sessionRoot)).mode & 0o777).toBe(0o700);
expect((await stat(join(sessionRoot, "config", "opencode", "opencode.json"))).mode & 0o777).toBe(0o600);
const traceEntries = (await readFile(tracePath, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as Record<string, unknown>);
const traceFrames = traceEntries.filter((entry) => entry.kind === "frame");
expect(traceFrames.map((entry) => entry.direction)).toEqual(
expect.arrayContaining(["client_to_provider", "provider_to_client", "provider_stderr"]),
);
expect(traceFrames.some((entry) => entry.nativeMethod === "SSE /event")).toBe(true);
expect(traceEntries.some(
(entry) => entry.stage === "typescript_opencode_driver_normalization"
&& Array.isArray(entry.emittedEventIds)
&& entry.emittedEventIds.length > 0,
)).toBe(true);
expect(traceEntries.at(-1)).toMatchObject({
kind: "trace_status",
debugChannel: "typescript_opencode_native",
status: "complete",
});
const environment = JSON.parse(await readFile(join(sessionRoot, "data", "fake-environment.json"), "utf8"));
const mcpEvidence = JSON.parse(await readFile(join(sessionRoot, "data", "fake-mcp-evidence.json"), "utf8"));
expect(environment.keys).toContain("OPENROUTER_API_KEY");
expect(environment.keys).not.toContain("PAPERCLIP_API_KEY");
expect(environment.keys).not.toContain("UNRELATED_SECRET");
expect(environment.keys).not.toContain("PAPERCLIP_PROVIDER_TRACE_PATH");
expect(environment.projectConfigDisabled).toBe("true");
expect(mcpEvidence.tools).toEqual(expect.arrayContaining(["paperclip_finish", "paperclip_block"]));
const config = await readFile(join(sessionRoot, "config", "opencode", "opencode.json"), "utf8");
expect(config).toContain("openrouter/deepseek/deepseek-v4-flash-0731");
expect(config).toContain('"external_directory": "deny"');
expect(config).not.toContain("test-openrouter-key");
@@ -84,6 +177,31 @@ describe("OpenCodeServerDriver", () => {
expect(diagnostics.join("\n")).toContain("[REDACTED]");
});
it("maps OpenCode's normal abort error to cancellation without a false provider failure notice", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-abort-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-abort-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-aborted",
normalizedSessionId: "aborted",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "session-aborted" } });
const events = [];
for await (const event of session.events()) events.push(event);
expect(events.map((event) => event.eventType)).toContain("turn.cancelled");
expect(events.map((event) => event.eventType)).not.toContain("turn.failed");
expect(events.map((event) => event.eventType)).not.toContain("provider.notice.recorded");
await session.close({ reason: "test" });
});
it("keeps OpenCode in an outer supervisor process group when requested", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
@@ -103,6 +221,217 @@ describe("OpenCodeServerDriver", () => {
await session.close({ reason: "test" });
});
it("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-context-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-context-workspace-"));
const skillRoot = join(root, "skill-source");
const instructionRoot = join(root, "instruction-source");
roots.push(root, workspace);
await Promise.all([
mkdir(join(skillRoot, "references"), { recursive: true }),
mkdir(instructionRoot, { recursive: true }),
]);
await Promise.all([
writeFile(join(skillRoot, "SKILL.md"), "# Assigned\nRead references/support.md\n"),
writeFile(join(skillRoot, "references", "support.md"), "skill support\n"),
writeFile(join(instructionRoot, "AGENTS.md"), "Read sibling.md\n"),
writeFile(join(instructionRoot, "sibling.md"), "instruction sibling\n"),
]);
const systemInstructions = `${PAPERCLIP_EXECUTION_PROMPT}\n\nRead sibling.md\n\nRead-only instruction sibling root: ${instructionRoot}`;
let submittedPrompt: Record<string, unknown> | null = null;
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
systemInstructions,
runtimeContext: runtimeContext(skillRoot, instructionRoot),
environment: {
PATH: process.env.PATH,
OPENROUTER_API_KEY: "fixture-key",
PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned",
PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp",
PAPERCLIP_NATIVE_MCP_TOKEN: "x".repeat(40),
},
fetch: async (input, init) => {
if (String(input).endsWith("/prompt_async")) submittedPrompt = JSON.parse(String(init?.body));
return fetch(input, init);
},
});
const session = await driver.openSession({
runId: "run-context",
normalizedSessionId: "context",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "finish" } });
for await (const event of session.events()) {
if (event.eventType === "turn.completed") break;
}
expect(submittedPrompt).toMatchObject({
system: systemInstructions,
tools: { question: true },
});
expect(JSON.stringify(submittedPrompt?.parts ?? null)).not.toContain(PAPERCLIP_EXECUTION_PROMPT);
const sessionRoot = join(root, "context");
await expect(readFile(join(sessionRoot, "home", ".claude", "skills", "assigned", "references", "support.md"), "utf8"))
.resolves.toBe("skill support\n");
const config = JSON.parse(await readFile(join(sessionRoot, "config", "opencode", "opencode.json"), "utf8"));
expect(config).toMatchObject({
instructions: [],
plugin: [],
tools: { question: true },
permission: { external_directory: { "*": "deny", [`${instructionRoot}/**`]: "allow" } },
mcp: {
paperclip: { enabled: true },
"paperclip-assigned": { url: "https://paperclip.example/mcp", enabled: true },
},
});
await expect(readFile(join(instructionRoot, "sibling.md"), "utf8")).resolves.toBe("instruction sibling\n");
await session.close({ reason: "test" });
});
it("sends only the authoritative wake envelope when resuming an existing provider session", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-resume-context-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-resume-workspace-"));
roots.push(root, workspace);
let submittedPrompt: Record<string, unknown> | null = null;
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
systemInstructions: "large original system context",
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
fetch: async (input, init) => {
if (String(input).endsWith("/prompt_async")) submittedPrompt = JSON.parse(String(init?.body));
return fetch(input, init);
},
});
const seed = await driver.openSession({
runId: "run-seed",
normalizedSessionId: "resume-context",
workingDirectory: workspace,
});
const snapshot = await seed.snapshot();
await seed.close({ reason: "seed complete" });
const recovered = await driver.recoverSession?.(snapshot);
expect(recovered).toMatchObject({ recovered: true });
await recovered!.session!.startTurn({
message: { role: "user", text: "{\"interactionResponses\":[{\"answer\":\"friendly\"}]}" },
});
for await (const event of recovered!.session!.events()) {
if (event.eventType === "turn.completed") break;
}
expect(submittedPrompt).toMatchObject({
providerID: "openrouter",
modelID: "deepseek/deepseek-v4-flash-0731",
tools: { question: true },
parts: [{ type: "text", text: "{\"interactionResponses\":[{\"answer\":\"friendly\"}]}" }],
});
expect(submittedPrompt).not.toHaveProperty("system");
await recovered!.session!.close({ reason: "recovery-test" });
});
it("normalizes native question.asked events and replies with ordered OpenCode answer arrays", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-question-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-native-question",
normalizedSessionId: "native-question",
workingDirectory: workspace,
});
const { turnId } = await session.startTurn({ message: { role: "user", text: "native-question" } });
const iterator = session.events()[Symbol.asyncIterator]();
let requestEvent: Awaited<ReturnType<typeof iterator.next>>["value"] | null = null;
for (let count = 0; count < 30; count += 1) {
const event = await iterator.next();
if (event.done) break;
if (event.value.eventType === "runtime_request.created") {
requestEvent = event.value;
break;
}
}
expect(requestEvent?.payload).toMatchObject({
request: {
schema: "paperclip.runtime_request.v2",
requestKind: "runtime",
type: "input",
input: {
schema: "paperclip.question_set.v1",
questions: [
{ id: "environment", answerMode: "single_select", customAnswer: { enabled: true } },
{ id: "regions", answerMode: "multi_select" },
],
},
},
});
await session.resolveRuntimeRequest?.({
requestId: "question-native-1",
turnId,
resolution: {
action: "submit",
response: {
schema: "paperclip.question_response.v1",
answers: {
environment: { selectedOptionIds: ["option-1"] },
regions: { selectedOptionIds: ["option-1", "option-2"] },
},
},
},
});
const reply = JSON.parse(await readFile(join(root, "native-question", "data", "fake-question-reply.json"), "utf8"));
expect(reply).toEqual({
url: expect.stringContaining(`directory=${encodeURIComponent(workspace)}`),
body: { answers: [["Staging"], ["US", "EU"]] },
});
await session.close({ reason: "test" });
});
it("clears a stale active turn that already has a persisted terminal fingerprint", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-stale-turn-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-stale-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const seed = await driver.openSession({
runId: "run-stale-turn",
normalizedSessionId: "stale-turn",
workingDirectory: workspace,
});
const snapshot = await seed.snapshot();
await seed.close({ reason: "seed complete" });
const recoveryDriver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const recovery = await recoveryDriver.recoverSession?.({
...snapshot,
activeTurnId: "turn-already-terminal",
terminalTurns: [{ turnId: "turn-already-terminal", fingerprint: "terminal-fingerprint" }],
});
expect(recovery).toMatchObject({ recovered: true });
await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ activeTurnId: null });
await recovery!.session!.close({ reason: "recovery-test" });
});
it("validates provider/model form", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
roots.push(root);
@@ -131,6 +460,132 @@ describe("OpenCodeServerDriver", () => {
await session.close({ reason: "test" });
});
it("designates the substantive pre-finish response instead of a trailing acknowledgement", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-final-selection",
normalizedSessionId: "final-selection",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "text-before-finish" } });
const events = [];
for await (const event of session.events()) events.push(event);
const finalMessages = events.filter(
(event) => event.eventType === "item.completed"
&& event.payload.kind === "agentMessage"
&& event.payload.channel === "final",
);
expect(finalMessages).toHaveLength(1);
expect(finalMessages[0]?.payload.text).toBe("Substantive answer before finish.");
expect(events.filter((event) => event.eventType === "item.delta").map((event) => event.payload.text))
.toEqual(expect.arrayContaining(["Substantive answer before finish.", "Done."]));
await session.close({ reason: "test" });
});
it("does not promote opening commentary to the final response when work follows it", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-commentary-only",
normalizedSessionId: "commentary-only",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "commentary-only-before-work" } });
const events = [];
for await (const event of session.events()) events.push(event);
expect(events.filter(
(event) => event.eventType === "item.completed"
&& event.payload.kind === "agentMessage"
&& event.payload.channel === "final",
)).toHaveLength(0);
expect(events.filter((event) => event.eventType === "item.delta").map((event) => event.payload.text))
.toContain("I will inspect the workspace first.");
expect(events).toContainEqual(expect.objectContaining({ eventType: "run.result.proposed" }));
await session.close({ reason: "test" });
});
it("correlates the final response to the native message containing the semantic tool", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-correlated-final-selection",
normalizedSessionId: "correlated-final-selection",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "correlated-final-message" } });
const events = [];
for await (const event of session.events()) events.push(event);
const finalMessages = events.filter(
(event) => event.eventType === "item.completed"
&& event.payload.kind === "agentMessage"
&& event.payload.channel === "final",
);
expect(finalMessages).toHaveLength(1);
expect(finalMessages[0]?.payload.text).toBe("Correlated substantive final answer.");
expect(events.filter((event) => event.eventType === "item.delta").map((event) => event.payload.text))
.toEqual(expect.arrayContaining([
"I will write the result now.",
"Correlated substantive final answer.",
"Done.",
]));
await session.close({ reason: "test" });
});
it("prefers a substantive post-tool answer over schema commentary in the tool message", async () => {
await chmod(fixture, 0o755);
const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-"));
const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-"));
roots.push(root, workspace);
const driver = new OpenCodeServerDriver({
model: "openrouter/deepseek/deepseek-v4-flash-0731",
runtimeDirectory: root,
command: fixture,
environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" },
});
const session = await driver.openSession({
runId: "run-post-tool-final-selection",
normalizedSessionId: "post-tool-final-selection",
workingDirectory: workspace,
});
await session.startTurn({ message: { role: "user", text: "final-after-tool-commentary" } });
const events = [];
for await (const event of session.events()) events.push(event);
const finalMessages = events.filter(
(event) => event.eventType === "item.completed"
&& event.payload.kind === "agentMessage"
&& event.payload.channel === "final",
);
expect(finalMessages).toHaveLength(1);
expect(finalMessages[0]?.payload.text)
.toBe("This is the complete substantive answer emitted after the accepted completion tool call.");
await session.close({ reason: "test" });
});
it("rejects malformed and oversized SSE frames", async () => {
const stream = (value: string) => new ReadableStream<Uint8Array>({
start(controller) {
packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
+659 -40
View File
File diff suppressed because it is too large. Load diff
packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts
+84
View File
@@ -0,0 +1,84 @@
import { chmod, lstat, mkdtemp, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
PAPERCLIP_EXECUTION_PROMPT,
PAPERCLIP_EXECUTION_PROMPT_REVISION,
canonicalNativeRuntimeContextDigest,
nativeRuntimePromptDigest,
type NativeRuntimeContextSnapshot,
} from "../contracts/runtime-context.js";
import { prepareIsolatedCodexHome } from "./runtime-context-materializer.js";
const roots: string[] = [];
async function makeWritable(root: string): Promise<void> {
const info = await lstat(root).catch(() => null);
if (!info) return;
if (info.isDirectory()) {
await chmod(root, 0o700);
for (const entry of await readdir(root)) await makeWritable(join(root, entry));
} else {
await chmod(root, 0o600);
}
}
afterEach(async () => {
const { rm } = await import("node:fs/promises");
await Promise.all(roots.splice(0).map(async (root) => {
await makeWritable(root);
await rm(root, { recursive: true, force: true });
}));
});
function context(skillRoot: string, instructionRoot: string): NativeRuntimeContextSnapshot {
const digest = "0".repeat(64);
const value = {
prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() },
instructions: {
entryPath: "AGENTS.md",
bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: instructionRoot, fileCount: 2, totalBytes: 2 },
},
skills: [{
key: "company/assigned",
runtimeName: "assigned",
versionId: "version-1",
bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: skillRoot, fileCount: 2, totalBytes: 2 },
}],
mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" },
} satisfies Omit<NativeRuntimeContextSnapshot, "aggregateDigest">;
return { ...value, aggregateDigest: canonicalNativeRuntimeContextDigest(value) };
}
describe("runtime context materialization", () => {
it("copies only assigned skills with their supporting files and an isolated MCP config", async () => {
const root = await mkdtemp(join(tmpdir(), "paperclip-runtime-context-"));
roots.push(root);
const assigned = join(root, "assigned-source");
const instructions = join(root, "instructions-source");
const hostSkills = join(root, "host-home", "skills", "unassigned");
const codexHome = join(root, "codex-home");
await Promise.all([mkdir(join(assigned, "references"), { recursive: true }), mkdir(instructions), mkdir(hostSkills, { recursive: true })]);
await writeFile(join(assigned, "SKILL.md"), "# Assigned\nRead references/support.md\n");
await writeFile(join(assigned, "references", "support.md"), "support\n");
await writeFile(join(instructions, "AGENTS.md"), "Read sibling.md\n");
await writeFile(join(instructions, "sibling.md"), "sibling\n");
await writeFile(join(hostSkills, "SKILL.md"), "# Must not leak\n");
await prepareIsolatedCodexHome({
context: context(assigned, instructions),
codexHome,
sourceCodexHome: join(root, "host-home"),
nativeMcp: { name: "paperclip-assigned", url: "https://paperclip.example/mcp", token: "x".repeat(40) },
});
await expect(readFile(join(codexHome, "skills", "assigned", "references", "support.md"), "utf8")).resolves.toBe("support\n");
await expect(stat(join(codexHome, "skills", "unassigned"))).rejects.toThrow();
expect((await stat(join(codexHome, "skills", "assigned", "SKILL.md"))).mode & 0o222).toBe(0);
const config = await readFile(join(codexHome, "config.toml"), "utf8");
expect(config).toContain("paperclip-assigned");
expect(config).toContain("Bearer ");
expect(config).not.toContain("unassigned");
});
});
packages/paperclip-runner/src/eval/fixtures/acpx-sanitized-provider.json
+17
View File
@@ -0,0 +1,17 @@
{
"schema": "paperclip.runner.sanitized-provider-fixture.v1",
"provider": "acpx",
"events": [
{
"type": "tool_call",
"tag": "tool_call",
"toolCallId": "fixture-tool",
"title": "node --test",
"kind": "execute",
"status": "completed",
"text": "Test completed",
"rawOutput": "1 test passed",
"locations": []
}
]
}
packages/paperclip-runner/src/eval/fixtures/codex-sanitized-provider.json
+36
View File
@@ -0,0 +1,36 @@
{
"schema": "paperclip.runner.sanitized-provider-fixture.v1",
"provider": "codex",
"records": [
{
"method": "item/completed",
"params": {
"item": {
"id": "fixture-search",
"type": "webSearch",
"query": "sanitized protocol query",
"results": [
{
"ref_id": "safe-source",
"title": "Protocol reference",
"url": "https://example.com/protocol",
"snippet": "Sanitized fixture"
}
]
}
}
},
{
"method": "item/completed",
"params": {
"item": {
"id": "fixture-tool",
"type": "commandExecution",
"command": "node --test",
"status": "completed",
"output": "1 test passed"
}
}
}
]
}
packages/paperclip-runner/src/eval/fixtures/opencode-sanitized-provider.json
+19
View File
@@ -0,0 +1,19 @@
{
"schema": "paperclip.runner.sanitized-provider-fixture.v1",
"provider": "opencode",
"parts": [
{
"id": "fixture-search",
"type": "websearch",
"query": "sanitized protocol query",
"state": { "status": "completed" }
},
{
"id": "fixture-tool",
"type": "tool",
"tool": "node_test",
"callID": "functions.node_test:0",
"state": { "status": "completed", "output": "1 test passed", "exit": 0 }
}
]
}
packages/paperclip-runner/src/eval/index.ts
+8
View File
@@ -2,3 +2,11 @@ export * from "./eval-bundle.js";
export * from "./eval-execution.js";
export * from "./eval-scoring.js";
export * from "./eval-slice.js";
export * from "./workflow-contracts.js";
export * from "./workflow-catalog.js";
export * from "./workflow-scoring.js";
export * from "./workflow-harness.js";
export * from "./workflow-traceability.js";
export * from "./live-workflow-matrix.js";
export * from "./live-workflow-executor.js";
export * from "./workflow-report.js";
packages/paperclip-runner/src/eval/live-workflow-executor.ts
+452
View File
@@ -0,0 +1,452 @@
import { createHash } from "node:crypto";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js";
import { capabilityFixtureRunCapabilities } from "../scenarios/fixture-run-capabilities.js";
import {
CapabilityLiveSessionService,
InMemoryCapabilityLiveSessionStore,
type CapabilityLiveSession,
type CapabilityLiveSessionSnapshot,
type CapabilityLiveTurnEvent,
type CapabilityLiveTurnResult,
} from "../live/live-session.js";
import type { EvalObservation } from "./eval-scoring.js";
import {
RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
type RunnerWorkflowCheck,
type RunnerWorkflowEvalCase,
type RunnerWorkflowObservation,
} from "./workflow-contracts.js";
import {
RunnerWorkflowInfrastructureError,
type RunnerLiveEvalCandidate,
type RunnerLiveScheduleEntry,
} from "./live-workflow-matrix.js";
const LIVE_GRANTS = [
"delegation:tasks:create",
"discovery:agents:read",
"discovery:tasks:read",
"governance:approvals:request",
] as const;
function promptFor(evalCase: RunnerWorkflowEvalCase): string {
const common = "Use only the typed Paperclip tools. Do not call REST APIs. Reply with concise user-facing prose after the final tool result.";
const prompts: Record<RunnerWorkflowEvalCase["id"], string> = {
"final-response": "Write a detailed answer of at least 1200 characters, then call finish_task exactly once with a faithful summary. Do not replace the answer with a completion acknowledgement.",
"rich-activity": "Immediately report one short progress update, inspect the workspace with a read-only command or search if available, then call finish_task exactly once. Preserve visible tool activity.",
"verification-policy": "Call finish_task exactly once. Include one verification entry that is not_run with reasonCode tool_unavailable and explain the caveat without treating it as a blocker.",
"governed-interaction": "Call request_human_input once to ask a required single-select question. After the typed answer arrives, use it exactly and call finish_task once. Do not create a provider-native question.",
"steering-causality": "A later message will narrow the requested format. On this turn provide a short draft and do not repeat work; after the follow-up, call finish_task exactly once.",
"planning-lifecycle": "Write a plan document, then request human confirmation. Do not create implementation work until confirmation is accepted. If rejected, revise the plan and request confirmation again. After acceptance create one child task and call finish_task.",
"review-lifecycle": "Request human confirmation for an external verification check. If rejected, address only the requested verification and ask again. After acceptance call finish_task exactly once.",
"delegation-return": "Create exactly one child task with a complete handoff, then call finish_task exactly once. Do not search for work already supplied in the handoff.",
"completion-robustness": "Call finish_task exactly once with canonical values. If the tool returns a schema error, correct the arguments in this turn. Do not print completion JSON as prose.",
"restart-recovery": "Call report_progress exactly once, state that disposition will be completed after recovery, and do not call finish_task on this turn.",
"cancellation-permissions": "Begin a read-only inspection and continue working until interrupted. Do not claim completion and do not call a semantic completion tool.",
"trace-lineage": "Call finish_task exactly once with a concise summary so the semantic tool frame can be correlated through PRP and presentation.",
};
return `${common} ${prompts[evalCase.id]}`;
}
function continuationPrompt(evalCase: RunnerWorkflowEvalCase): string {
if (evalCase.id === "restart-recovery") {
return "The provider session was restored. Use the existing progress result, do not repeat it, and call finish_task exactly once.";
}
if (evalCase.id === "steering-causality") {
return "Steering update: use bullet points only. Apply only this formatting change and call finish_task exactly once without repeating prior work.";
}
return "Continue from the authoritative typed interaction result. Do not redo completed work.";
}
function acpxAgent(candidate: RunnerLiveEvalCandidate): "claude" | "codex" | "pi" | undefined {
if (candidate.adapter !== "acpx_runtime") return undefined;
if (candidate.qualification.profile === "claude") return "claude";
if (candidate.qualification.profile === "codex") return "codex";
return "pi";
}
function check(id: string, passed: boolean, reason: string, evidenceIds: string[] = []): RunnerWorkflowCheck {
return { id, passed, ...(passed ? {} : { reason }), ...(evidenceIds.length === 0 ? {} : { evidenceIds }) };
}
function interactionResolution(evalCase: RunnerWorkflowEvalCase, index: number): { outcome: "answered" | "accepted" | "rejected"; result: CapabilityJsonValue } {
const decisions = evalCase.steps.flatMap((step) => step.kind === "review_decision" ? [step.decision] : []);
const decision = decisions[index];
if (decision === "reject") return { outcome: "rejected", result: { note: "Eval reviewer requests the narrow verification-only revision." } };
if (decision === "approve") return { outcome: "accepted", result: { confirmed: true } };
return { outcome: "answered", result: { selectedOptionIds: ["eval-choice"], answer: "eval-choice" } };
}
function providerEventTypes(snapshot: CapabilityLiveSessionSnapshot): string[] {
return snapshot.evidence.flatMap((entry) => {
if (entry.kind !== "provider_event") return [];
const value = entry.data.canonicalEventType;
return typeof value === "string" ? [value] : [];
});
}
function observedCalls(snapshot: CapabilityLiveSessionSnapshot): string[] {
return snapshot.evidence.flatMap((entry) =>
entry.kind === "tool_call" && typeof entry.data.operationId === "string"
? [entry.data.operationId]
: []);
}
function duplicateEffectSignals(snapshot: CapabilityLiveSessionSnapshot): string[] {
return snapshot.evidence.flatMap((entry) => {
if (entry.kind !== "tool_result") return [];
const envelope = typeof entry.data.result === "object" && entry.data.result !== null && !Array.isArray(entry.data.result)
? entry.data.result as Record<string, CapabilityJsonValue>
: {};
const result = typeof envelope.result === "object" && envelope.result !== null && !Array.isArray(envelope.result)
? envelope.result as Record<string, CapabilityJsonValue>
: {};
return result.disposition === "duplicate" && typeof entry.data.operationId === "string"
? [entry.data.operationId]
: [];
});
}
function traceMetadata(raw: string): {
frameCount: number;
byteCount: number;
digest: string;
digestVerified: boolean;
ordered: boolean;
dispositions: string[];
lineage: string[];
} {
const records = raw.split("\n").filter(Boolean).flatMap((line) => {
try {
return [JSON.parse(line) as Record<string, unknown>];
} catch {
return [];
}
});
const frames = records.filter((entry) => entry.kind === "frame");
const frameIds = frames.map((entry) => Number(entry.frameId));
const debugSequences = records.map((entry) => Number(entry.debugSequence)).filter(Number.isFinite);
const digestVerified = frames.length > 0 && frames.every((entry) => {
if (typeof entry.rawBase64 !== "string" || typeof entry.digest !== "string") return false;
const bytes = Buffer.from(entry.rawBase64, "base64");
return bytes.byteLength === Number(entry.byteLength)
&& `sha256:${createHash("sha256").update(bytes).digest("hex")}` === entry.digest;
});
const strictlyIncreasing = (values: number[]): boolean => values.every((value, index) =>
Number.isFinite(value) && (index === 0 || value > values[index - 1]!));
const interpretations = records.filter((entry) => entry.kind === "interpretation");
const emittedCounts = interpretations.map((entry) => Array.isArray(entry.emittedEventIds) ? entry.emittedEventIds.length : 0);
const eventParents = new Map<string, number>();
for (const interpretation of interpretations) {
if (!Array.isArray(interpretation.emittedEventIds)) continue;
for (const eventId of interpretation.emittedEventIds) {
if (typeof eventId === "string") eventParents.set(eventId, (eventParents.get(eventId) ?? 0) + 1);
}
}
const lineage = new Set<string>();
if (interpretations.length > 0) lineage.add("one_to_one");
if (emittedCounts.some((count) => count > 1)) lineage.add("one_to_many");
if ([...eventParents.values()].some((count) => count > 1)) lineage.add("many_to_one");
return {
frameCount: frames.length,
byteCount: frames.reduce((sum, entry) => sum + (Number(entry.byteLength) || 0), 0),
digest: `sha256:${createHash("sha256").update(raw).digest("hex")}`,
digestVerified,
ordered: strictlyIncreasing(frameIds) && strictlyIncreasing(debugSequences),
dispositions: [...new Set(interpretations.flatMap((entry) => typeof entry.disposition === "string" ? [entry.disposition] : []))],
lineage: [...lineage],
};
}
function digestJson(value: unknown): string {
return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`;
}
function safeFailureMessage(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/\b(?:sk-[A-Za-z0-9_-]{8,}|Bearer\s+\S+|AKIA[0-9A-Z]{16})\b/gi, "[REDACTED]")
.slice(0, 1_000);
}
/** Builds an unscored observation without exposing provider credentials or raw trace payloads. */
export function unavailableLiveRunnerWorkflowObservation(input: {
entry: RunnerLiveScheduleEntry;
candidate: RunnerLiveEvalCandidate;
evalCase: RunnerWorkflowEvalCase;
classification: "infrastructure_failure" | "skipped";
code: string;
category: "provider" | "qualification" | "orchestration";
retryable: boolean;
message: string;
}): RunnerWorkflowObservation {
const unavailable = check("execution-unavailable", false, input.message);
return {
schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
caseId: input.evalCase.id,
candidateId: input.candidate.id,
provider: input.candidate.provider,
classification: input.classification,
base: {
caseId: input.evalCase.id,
provenance: { source: "live_model", behavior: input.evalCase.id },
controlPlaneOwned: false,
expectedCalls: input.evalCase.assertions.requiredOperationIds ?? [],
observedCalls: [],
forbiddenCalls: [],
finalState: { expected: "mutated", observed: "unchanged" },
authorization: { expected: "allowed", observed: "absent" },
trace: { terminalPresent: false, receiptIds: [] },
efficiency: { latencyMs: 0, totalTokens: 0, costUsd: 0, attempts: 0 },
budget: input.candidate.budget,
},
lifecycle: { checks: [unavailable], attempts: 0, runs: 0, recoveryOwner: "none" },
continuation: { checks: [unavailable], repeatedWorkSignals: [] },
presentation: { checks: [unavailable], responseSource: "none", commentCount: 0 },
traceLineage: {
capture: "off", frameCount: 0, byteCount: 0, digestVerified: false,
ordered: false, dispositions: [], lineage: [],
},
metrics: { settlementMs: 0, attempts: 0, toolCount: 0, totalTokens: 0, costUsd: 0 },
observedPrpEventTypes: [],
artifactDigests: [],
failure: {
code: input.code,
category: input.category,
retryable: input.retryable,
message: safeFailureMessage(input.message),
},
};
}
async function settleInteractions(
evalCase: RunnerWorkflowEvalCase,
session: CapabilityLiveSession,
turns: CapabilityLiveTurnResult[],
): Promise<void> {
let index = 0;
while (session.pendingInteractions().length > 0 && index < 6) {
const pending = session.pendingInteractions()[0]!;
const resolution = interactionResolution(evalCase, index);
turns.push(await session.resolveInteraction({ interactionId: pending.id, ...resolution }));
index += 1;
}
}
/** Executes one real-provider workflow against the controlled mock authority. */
export async function executeLiveRunnerWorkflow(input: {
entry: RunnerLiveScheduleEntry;
candidate: RunnerLiveEvalCandidate;
evalCase: RunnerWorkflowEvalCase;
workingDirectory?: string;
}): Promise<RunnerWorkflowObservation> {
const runtimeRoot = await mkdtemp(join(tmpdir(), "paperclip-runner-live-eval-"));
const tracePath = join(runtimeRoot, "provider-trace.ndjson");
const store = new InMemoryCapabilityLiveSessionStore();
const transportOptions = {
environment: { ...process.env, PAPERCLIP_PROVIDER_TRACE_PATH: tracePath },
};
let service = new CapabilityLiveSessionService({ store, transportOptions });
let session: CapabilityLiveSession | null = null;
const turns: CapabilityLiveTurnResult[] = [];
const streamed: CapabilityLiveTurnEvent[] = [];
const startedAt = Date.now();
let firstVisibleAt: number | null = null;
let infrastructureError: unknown;
try {
session = await service.create({
workingDirectory: input.workingDirectory ?? runtimeRoot,
provider: input.candidate.provider,
...(acpxAgent(input.candidate) === undefined ? {} : { acpxAgent: acpxAgent(input.candidate) }),
requestedModel: input.candidate.model,
seed: {
actors: [{
id: "actor-1", companyId: "company-1", name: "Workflow eval actor", role: "engineer",
status: "active", budgetId: "budget-actor-1", capabilityGrants: [...LIVE_GRANTS],
}],
},
capabilities: capabilityFixtureRunCapabilities(LIVE_GRANTS),
explicitClaims: [...LIVE_GRANTS],
runId: input.entry.executionId,
sessionId: `session-${input.entry.executionId}`,
attemptId: `attempt-${input.entry.executionId}`,
turnTimeoutMs: input.candidate.budget.maxLatencyMs,
lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 },
scenario: { id: `runner-workflow-${input.evalCase.id}` },
});
const unsubscribe = session.subscribe((event) => {
streamed.push(event);
if (firstVisibleAt === null && (event.kind === "delta" || event.kind === "activity")) firstVisibleAt = Date.now();
});
try {
if (input.evalCase.id === "cancellation-permissions") {
const pending = session.sendMessage(promptFor(input.evalCase));
await new Promise<void>((resolve) => setTimeout(resolve, 250));
await session.interrupt("workflow eval cancellation");
const settled = await Promise.allSettled([pending]);
if (settled[0]?.status === "fulfilled") turns.push(settled[0].value);
} else {
turns.push(await session.sendMessage(promptFor(input.evalCase)));
await settleInteractions(input.evalCase, session, turns);
if (input.evalCase.id === "steering-causality") {
turns.push(await session.sendMessage(continuationPrompt(input.evalCase)));
}
if (input.evalCase.id === "restart-recovery") {
const sessionId = session.id;
await session.suspend("workflow eval simulated worker restart");
service = new CapabilityLiveSessionService({ store, transportOptions });
session = await service.restore(sessionId);
turns.push(await session.sendMessage(continuationPrompt(input.evalCase)));
}
}
} finally {
unsubscribe();
}
} catch (error) {
infrastructureError = error;
}
const snapshot = session?.snapshot();
if (session !== null) await service.shutdown(session.id, "Runner live workflow eval complete").catch(() => undefined);
if (infrastructureError !== undefined) {
await rm(runtimeRoot, { recursive: true, force: true });
throw new RunnerWorkflowInfrastructureError(
"live_provider_execution_failed",
true,
safeFailureMessage(infrastructureError),
);
}
const calls = snapshot === undefined ? [] : observedCalls(snapshot);
const expectedCalls = input.evalCase.assertions.requiredOperationIds ?? [];
const taskState = snapshot === undefined ? null : (JSON.parse(snapshot.mockState) as { tasks?: Array<{ id: string; status: string }> }).tasks?.find((task) => task.id === "task-1") ?? null;
const terminalStatuses = turns.map((turn) => turn.status);
const cancellationExpected = input.evalCase.id === "cancellation-permissions";
const terminalOkay = cancellationExpected
? terminalStatuses.some((status) => status === "cancelled" || status === "interrupted")
: terminalStatuses.length > 0 && terminalStatuses.every((status) => status === "completed");
const missingCalls = expectedCalls.filter((operationId) => !calls.includes(operationId));
const extraCalls = calls.filter((operationId) => !expectedCalls.includes(operationId));
const duplicateSignals = snapshot === undefined ? [] : duplicateEffectSignals(snapshot);
const pendingInteractions = session?.pendingInteractions().length ?? 0;
let rawTrace = "";
try {
rawTrace = await readFile(tracePath, "utf8");
} catch {
rawTrace = "";
}
const trace = traceMetadata(rawTrace);
const usage = snapshot?.usageLedger ?? [];
const totalTokens = usage.reduce((sum, receipt) => sum + receipt.inputTokens + receipt.outputTokens + receipt.reasoningTokens, 0);
const costUsd = usage.reduce((sum, receipt) => sum + receipt.costNanodollars, 0) / 1_000_000_000;
const receiptIds = snapshot?.evidence.filter((entry) => entry.kind === "tool_result").map((entry) => entry.id) ?? [];
const assistantTexts = snapshot?.transcript.filter((entry) => entry.role === "assistant" && entry.text.trim().length > 0).map((entry) => entry.text) ?? [];
const stateMutated = (snapshot?.stateHistory?.length ?? 0) > 1;
const base: EvalObservation = {
caseId: input.evalCase.id,
provenance: { source: "live_model", behavior: input.evalCase.id },
controlPlaneOwned: expectedCalls.length === 0,
expectedCalls,
observedCalls: calls,
forbiddenCalls: [],
finalState: { expected: expectedCalls.length === 0 ? "unchanged" : "mutated", observed: stateMutated ? "mutated" : "unchanged" },
authorization: {
expected: expectedCalls.length === 0 ? "absent" : "allowed",
observed: calls.length === 0 ? "absent" : snapshot?.authorizationRecords.some((record) => record.phase === "invocation" && !record.allowed) ? "denied" : "allowed",
},
trace: {
runId: snapshot?.authority.runId,
sessionId: snapshot?.sessionId,
turnId: turns.at(-1)?.turnId,
itemId: snapshot?.evidence.find((entry) => entry.kind === "tool_call")?.id,
receiptIds,
terminalPresent: terminalStatuses.length > 0,
},
efficiency: { latencyMs: Date.now() - startedAt, totalTokens, costUsd, attempts: snapshot?.attempts?.length ?? 1 },
budget: {
maxLatencyMs: input.candidate.budget.maxLatencyMs,
maxTotalTokens: input.candidate.budget.maxTotalTokens,
maxCostUsd: input.candidate.budget.maxCostUsd,
maxAttempts: input.candidate.budget.maxAttempts,
},
};
const lifecycleChecks = [
check("terminal-authority", terminalOkay, `unexpected terminal statuses: ${terminalStatuses.join(",") || "none"}`),
check("attempt-bound", (snapshot?.attempts?.length ?? 1) <= input.candidate.budget.maxAttempts, "attempt budget exceeded"),
check("semantic-disposition", cancellationExpected || calls.includes("finish_task") || calls.includes("request_review"), "no authoritative semantic disposition"),
check("owned-wait", pendingInteractions === 0, `${pendingInteractions} governed interaction(s) remain pending`),
];
const continuationChecks = [
check("required-calls", missingCalls.length === 0, `missing required calls: ${missingCalls.join(", ")}`),
check("trajectory-restraint", extraCalls.length === 0, `unexpected calls: ${extraCalls.join(", ")}`),
check(
"no-repeated-work",
input.evalCase.id === "restart-recovery" || duplicateSignals.length === 0,
`duplicate semantic effects detected: ${[...new Set(duplicateSignals)].join(", ")}`,
),
];
const presentationChecks = [
check("first-visible-progress", cancellationExpected || firstVisibleAt !== null, "provider emitted no visible progress"),
check("substantive-response", cancellationExpected || assistantTexts.some((text) => text.trim().length >= 2), "provider emitted no user-facing response"),
check("no-empty-comment", assistantTexts.every((text) => text.trim().length > 0), "empty assistant output was retained"),
check("terminal-presentation", terminalOkay, "terminal presentation would remain unsettled"),
];
const checksPassed = [...lifecycleChecks, ...continuationChecks, ...presentationChecks].every((entry) => entry.passed);
const observation: RunnerWorkflowObservation = {
schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
caseId: input.evalCase.id,
candidateId: input.candidate.id,
provider: input.candidate.provider,
classification: checksPassed ? "completed" : "candidate_failure",
base,
lifecycle: {
checks: lifecycleChecks,
issueStatus: taskState?.status,
runStatus: terminalStatuses.at(-1),
semanticDisposition: calls.includes("finish_task") ? "done" : calls.includes("request_review") ? "needs_review" : undefined,
attempts: snapshot?.attempts?.length ?? 1,
runs: turns.length,
recoveryOwner: pendingInteractions > 0 ? "human" : "none",
},
continuation: {
checks: continuationChecks,
wakeReasons: snapshot?.evidence.filter((entry) => entry.kind === "interaction").map(() => "interaction_resolved") ?? [],
consumedInputIds: snapshot?.evidence.filter((entry) => entry.kind === "interaction").map((entry) => entry.id) ?? [],
sessionPolicy: input.evalCase.id === "restart-recovery" ? "same_session" : "same_session",
repeatedWorkSignals: [...new Set([...extraCalls, ...duplicateSignals])],
},
presentation: {
checks: presentationChecks,
responseSource: assistantTexts.length > 0 ? "final_agent_message" : "none",
commentCount: assistantTexts.length > 0 ? 1 : 0,
orderedMarkers: streamed.map((event) => event.kind),
visibleActivityFamilies: snapshot === undefined ? [] : providerEventTypes(snapshot).map((eventType) => eventType.split(".")[0]!),
terminalLabel: cancellationExpected ? "Stopped" : terminalOkay ? "Completed" : "Needs attention",
},
traceLineage: {
capture: rawTrace.length > 0 ? "on" : "off",
frameCount: trace.frameCount,
byteCount: trace.byteCount,
digestVerified: trace.digestVerified,
ordered: trace.ordered,
dispositions: trace.dispositions,
lineage: trace.lineage,
...(rawTrace.length > 0 ? { traceRef: trace.digest } : {}),
},
metrics: {
timeToFirstVisibleProgressMs: firstVisibleAt === null ? undefined : firstVisibleAt - startedAt,
settlementMs: Date.now() - startedAt,
attempts: snapshot?.attempts?.length ?? 1,
toolCount: calls.length,
totalTokens,
costUsd,
},
observedPrpEventTypes: snapshot === undefined ? [] : providerEventTypes(snapshot),
artifactDigests: snapshot?.workspaceDiffs?.map((entry) => digestJson(entry.diff)) ?? [],
};
await rm(runtimeRoot, { recursive: true, force: true });
return observation;
}
packages/paperclip-runner/src/eval/live-workflow-matrix.ts
+296
View File
@@ -0,0 +1,296 @@
import { createHash } from "node:crypto";
import type {
RunnerWorkflowEvalCase,
RunnerWorkflowLane,
RunnerWorkflowObservation,
RunnerWorkflowProvider,
} from "./workflow-contracts.js";
import { RUNNER_WORKFLOW_CATALOG } from "./workflow-catalog.js";
export const RUNNER_LIVE_CANDIDATE_SCHEMA = "paperclip.runner.live-eval-candidate.v1" as const;
export const RUNNER_LIVE_SCHEDULE_SCHEMA = "paperclip.runner.live-eval-schedule.v1" as const;
export type RunnerLiveAdapter = "codex_app_server" | "opencode_server" | "acpx_runtime";
export type RunnerLiveTier = "strong" | "inexpensive";
export interface RunnerLiveEvalCandidate {
schema: typeof RUNNER_LIVE_CANDIDATE_SCHEMA;
id: string;
slotId: string;
adapter: RunnerLiveAdapter;
provider: RunnerWorkflowProvider;
model: string;
tier: RunnerLiveTier;
reasoningEffort?: string;
qualification: { requiredEnvironment: string[]; profile?: string };
budget: { maxLatencyMs: number; maxAttempts: number; maxCostUsd: number; maxTotalTokens: number };
}
interface RunnerLiveCandidateSlot {
id: string;
candidates: readonly RunnerLiveEvalCandidate[];
}
function candidate(input: Omit<RunnerLiveEvalCandidate, "schema">): RunnerLiveEvalCandidate {
return Object.freeze({ schema: RUNNER_LIVE_CANDIDATE_SCHEMA, ...input });
}
export const RUNNER_LIVE_CANDIDATE_SLOTS: readonly RunnerLiveCandidateSlot[] = Object.freeze([
{
id: "codex-strong",
candidates: [candidate({
id: "codex-luna", slotId: "codex-strong", adapter: "codex_app_server", provider: "codex",
model: "gpt-5.6-luna", tier: "strong", reasoningEffort: "medium",
qualification: { requiredEnvironment: [] },
budget: { maxLatencyMs: 180_000, maxAttempts: 3, maxCostUsd: 1.50, maxTotalTokens: 100_000 },
})],
},
{
id: "opencode-strong",
candidates: [
candidate({
id: "opencode-kimi", slotId: "opencode-strong", adapter: "opencode_server", provider: "opencode",
model: "openrouter/moonshotai/kimi-k2.5", tier: "strong",
qualification: { requiredEnvironment: ["OPENROUTER_API_KEY"], profile: "openrouter" },
budget: { maxLatencyMs: 180_000, maxAttempts: 3, maxCostUsd: 0.80, maxTotalTokens: 100_000 },
}),
candidate({
id: "opencode-glm", slotId: "opencode-strong", adapter: "opencode_server", provider: "opencode",
model: "openrouter/z-ai/glm-4.6", tier: "strong",
qualification: { requiredEnvironment: ["OPENROUTER_API_KEY"], profile: "openrouter" },
budget: { maxLatencyMs: 180_000, maxAttempts: 3, maxCostUsd: 0.80, maxTotalTokens: 100_000 },
}),
],
},
{
id: "opencode-inexpensive",
candidates: [
candidate({
id: "opencode-gpt-oss", slotId: "opencode-inexpensive", adapter: "opencode_server", provider: "opencode",
model: "openrouter/openai/gpt-oss-20b", tier: "inexpensive",
qualification: { requiredEnvironment: ["OPENROUTER_API_KEY"], profile: "openrouter" },
budget: { maxLatencyMs: 150_000, maxAttempts: 3, maxCostUsd: 0.20, maxTotalTokens: 64_000 },
}),
candidate({
id: "opencode-glm-flash", slotId: "opencode-inexpensive", adapter: "opencode_server", provider: "opencode",
model: "openrouter/z-ai/glm-4.5-air", tier: "inexpensive",
qualification: { requiredEnvironment: ["OPENROUTER_API_KEY"], profile: "openrouter" },
budget: { maxLatencyMs: 150_000, maxAttempts: 3, maxCostUsd: 0.20, maxTotalTokens: 64_000 },
}),
],
},
{
id: "acpx-claude",
candidates: [candidate({
id: "acpx-claude-sonnet", slotId: "acpx-claude", adapter: "acpx_runtime", provider: "acpx",
model: "claude-sonnet-5", tier: "strong",
qualification: { requiredEnvironment: ["ANTHROPIC_API_KEY"], profile: "claude" },
budget: { maxLatencyMs: 180_000, maxAttempts: 3, maxCostUsd: 1.50, maxTotalTokens: 100_000 },
})],
},
{
id: "acpx-codex",
candidates: [candidate({
id: "acpx-codex-sol", slotId: "acpx-codex", adapter: "acpx_runtime", provider: "acpx",
model: "gpt-5.6-sol", tier: "strong", reasoningEffort: "medium",
qualification: { requiredEnvironment: [], profile: "codex" },
budget: { maxLatencyMs: 180_000, maxAttempts: 3, maxCostUsd: 1.50, maxTotalTokens: 100_000 },
})],
},
{
id: "acpx-pi",
candidates: [candidate({
id: "acpx-pi-deepseek-flash", slotId: "acpx-pi", adapter: "acpx_runtime", provider: "acpx",
model: "openrouter/deepseek/deepseek-v4-flash-0731", tier: "inexpensive",
qualification: { requiredEnvironment: ["OPENROUTER_API_KEY"], profile: "pi" },
budget: { maxLatencyMs: 150_000, maxAttempts: 3, maxCostUsd: 0.20, maxTotalTokens: 64_000 },
})],
},
]);
export interface RunnerLiveScheduleEntry {
executionId: string;
caseId: RunnerWorkflowEvalCase["id"];
candidateId: string;
slotId: string;
repetition: number;
providerTrace: "raw";
budget: RunnerLiveEvalCandidate["budget"];
}
export interface RunnerLiveEvalSchedule {
schema: typeof RUNNER_LIVE_SCHEDULE_SCHEMA;
seed: string;
rotationDay: number;
generatedAt: string;
candidates: RunnerLiveEvalCandidate[];
entries: RunnerLiveScheduleEntry[];
expectedExecutions: number;
}
export function assertRunnerLiveCandidateManifest(): void {
const slots = RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.id);
const candidates = RUNNER_LIVE_CANDIDATE_SLOTS.flatMap((slot) => slot.candidates);
const ids = candidates.map((entry) => entry.id);
if (new Set(slots).size !== 6 || new Set(ids).size !== ids.length) {
throw new Error("Runner live candidate slots or candidate ids are not unique");
}
const serialized = JSON.stringify(candidates);
if (/\b(?:sk-[A-Za-z0-9]{16,}|Bearer\s+\S{16,}|AKIA[0-9A-Z]{16})\b/.test(serialized)) {
throw new Error("Runner live candidate manifest contains secret-shaped data");
}
for (const slot of RUNNER_LIVE_CANDIDATE_SLOTS) {
if (slot.candidates.some((entry) => entry.slotId !== slot.id || entry.model.trim().length === 0)) {
throw new Error(`Runner live candidate slot ${slot.id} is invalid`);
}
}
}
export function resolvedNightlyCandidates(rotationDay: number): RunnerLiveEvalCandidate[] {
assertRunnerLiveCandidateManifest();
const normalizedDay = ((rotationDay % 7) + 7) % 7;
return RUNNER_LIVE_CANDIDATE_SLOTS.map((slot) => slot.candidates[normalizedDay % slot.candidates.length]!);
}
function digestId(value: string): string {
return createHash("sha256").update(value).digest("hex").slice(0, 16);
}
/** Builds the stable 40-execution nightly pairwise schedule. */
export function buildRunnerLiveEvalSchedule(input: {
seed: string;
rotationDay: number;
generatedAt?: string;
cases?: readonly RunnerWorkflowEvalCase[];
}): RunnerLiveEvalSchedule {
const cases = input.cases ?? RUNNER_WORKFLOW_CATALOG;
const candidates = resolvedNightlyCandidates(input.rotationDay);
const entries: RunnerLiveScheduleEntry[] = [];
for (const [caseIndex, evalCase] of cases.entries()) {
const firstIndex = (caseIndex + input.rotationDay) % candidates.length;
const secondIndex = (firstIndex + 3) % candidates.length;
for (const candidateIndex of [firstIndex, secondIndex]) {
const resolved = candidates[candidateIndex]!;
const repetitions = evalCase.coreLiveRepeat ? 3 : 1;
for (let repetition = 1; repetition <= repetitions; repetition += 1) {
const identity = `${input.seed}:${input.rotationDay}:${evalCase.id}:${resolved.id}:${repetition}`;
entries.push({
executionId: `rwe-${digestId(identity)}`,
caseId: evalCase.id,
candidateId: resolved.id,
slotId: resolved.slotId,
repetition,
providerTrace: "raw",
budget: resolved.budget,
});
}
}
}
return {
schema: RUNNER_LIVE_SCHEDULE_SCHEMA,
seed: input.seed,
rotationDay: ((input.rotationDay % 7) + 7) % 7,
generatedAt: input.generatedAt ?? new Date().toISOString(),
candidates,
entries,
expectedExecutions: entries.length,
};
}
export interface RunnerLiveScheduleCoverage {
everySlotNightly: boolean;
everyConcreteCandidateInRotation: boolean;
everyWorkflowCoversEverySlot: boolean;
strongAndInexpensiveNightly: boolean;
}
export function runnerLiveScheduleCoverage(seed = "runner-live-seven-day-v1"): RunnerLiveScheduleCoverage {
const schedules = Array.from({ length: 7 }, (_, rotationDay) => buildRunnerLiveEvalSchedule({
seed,
rotationDay,
generatedAt: `2026-08-${String(24 + rotationDay).padStart(2, "0")}T00:00:00.000Z`,
}));
const allCandidateIds = new Set(RUNNER_LIVE_CANDIDATE_SLOTS.flatMap((slot) => slot.candidates.map((entry) => entry.id)));
const seenCandidateIds = new Set(schedules.flatMap((schedule) => schedule.entries.map((entry) => entry.candidateId)));
const everySlotNightly = schedules.every((schedule) =>
new Set(schedule.entries.map((entry) => entry.slotId)).size === RUNNER_LIVE_CANDIDATE_SLOTS.length);
const everyWorkflowCoversEverySlot = RUNNER_WORKFLOW_CATALOG.every((evalCase) => {
const slots = new Set(schedules.flatMap((schedule) => schedule.entries
.filter((entry) => entry.caseId === evalCase.id)
.map((entry) => entry.slotId)));
return slots.size === RUNNER_LIVE_CANDIDATE_SLOTS.length;
});
const strongAndInexpensiveNightly = schedules.every((schedule) => {
const selected = schedule.candidates.filter((candidate) => schedule.entries.some((entry) => entry.candidateId === candidate.id));
return selected.some((candidate) => candidate.tier === "strong")
&& selected.some((candidate) => candidate.tier === "inexpensive");
});
return {
everySlotNightly,
everyConcreteCandidateInRotation: [...allCandidateIds].every((id) => seenCandidateIds.has(id)),
everyWorkflowCoversEverySlot,
strongAndInexpensiveNightly,
};
}
export class RunnerWorkflowInfrastructureError extends Error {
constructor(
readonly code: string,
readonly retryable: boolean,
message: string,
) {
super(message);
this.name = "RunnerWorkflowInfrastructureError";
}
}
export async function executeRunnerLiveSchedule(
schedule: RunnerLiveEvalSchedule,
execute: (entry: RunnerLiveScheduleEntry, candidate: RunnerLiveEvalCandidate) => Promise<RunnerWorkflowObservation>,
onInfrastructureFailure?: (
entry: RunnerLiveScheduleEntry,
candidate: RunnerLiveEvalCandidate,
error: RunnerWorkflowInfrastructureError,
) => RunnerWorkflowObservation | Promise<RunnerWorkflowObservation>,
): Promise<RunnerWorkflowObservation[]> {
const results: RunnerWorkflowObservation[] = [];
for (const entry of schedule.entries) {
const resolved = schedule.candidates.find((candidate) => candidate.id === entry.candidateId);
if (!resolved) throw new Error(`schedule references unknown candidate ${entry.candidateId}`);
let infrastructureAttempts = 0;
while (true) {
try {
results.push(await execute(entry, resolved));
break;
} catch (error) {
if (!(error instanceof RunnerWorkflowInfrastructureError)) {
throw error;
}
if (!error.retryable || infrastructureAttempts >= 1) {
if (onInfrastructureFailure === undefined) throw error;
results.push(await onInfrastructureFailure(entry, resolved, error));
break;
}
infrastructureAttempts += 1;
}
}
}
return results;
}
export const RUNNER_CHAOS_SCENARIOS: readonly {
id: string;
lane: RunnerWorkflowLane;
fault: "restart" | "duplicate_delivery" | "trace_interruption" | "trace_truncation" | "stale_checkpoint" | "retry_exhaustion" | "interaction_timeout" | "wake_race";
expectedInvariant: string;
}[] = Object.freeze([
{ id: "chaos-restart", lane: "chaos", fault: "restart", expectedInvariant: "one resumable execution owns finalization" },
{ id: "chaos-duplicate-delivery", lane: "chaos", fault: "duplicate_delivery", expectedInvariant: "semantic effects are idempotent" },
{ id: "chaos-trace-interruption", lane: "chaos", fault: "trace_interruption", expectedInvariant: "run success is independent of trace upload" },
{ id: "chaos-trace-truncation", lane: "chaos", fault: "trace_truncation", expectedInvariant: "trace is marked truncated without PRP backpressure" },
{ id: "chaos-stale-checkpoint", lane: "chaos", fault: "stale_checkpoint", expectedInvariant: "terminal turns cannot remain active" },
{ id: "chaos-retry-exhaustion", lane: "chaos", fault: "retry_exhaustion", expectedInvariant: "board recovery owns the bounded terminal" },
{ id: "chaos-interaction-timeout", lane: "chaos", fault: "interaction_timeout", expectedInvariant: "governed wait remains visible and owned" },
{ id: "chaos-wake-race", lane: "chaos", fault: "wake_race", expectedInvariant: "one rich parent wake wins coalescing" },
]);
packages/paperclip-runner/src/eval/workflow-catalog.ts
+227
View File
@@ -0,0 +1,227 @@
import {
RUNNER_WORKFLOW_EVAL_CASE_SCHEMA,
RUNNER_WORKFLOW_IDS,
assertRunnerWorkflowEvalCase,
type RunnerWorkflowEvalCase,
} from "./workflow-contracts.js";
const ALL_PROVIDERS = ["codex", "opencode", "acpx"] as const;
function workflow(input: Omit<RunnerWorkflowEvalCase, "schema" | "version">): RunnerWorkflowEvalCase {
const value: RunnerWorkflowEvalCase = {
schema: RUNNER_WORKFLOW_EVAL_CASE_SCHEMA,
version: 1,
...input,
};
assertRunnerWorkflowEvalCase(value);
return Object.freeze(value);
}
/** The twelve composed workflows distilled from the six-hour stress campaign. */
export const RUNNER_WORKFLOW_CATALOG: readonly RunnerWorkflowEvalCase[] = Object.freeze([
workflow({
id: "final-response",
title: "Preserve one substantive final response",
lanes: ["deterministic", "live"], tags: ["response", "resolver", "long-output"], coreLiveRepeat: true,
providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "ask" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 1,
commentCount: 1, responseSource: "final_agent_message",
orderedMarkers: ["commentary", "activity", "final"],
requiredPrpEventTypes: ["run.result.accepted", "run.presentation.resolved", "run.terminal"],
forbiddenPrpEventTypes: ["run.presentation.placeholder"],
requiredOperationIds: ["finish_task"],
},
}),
workflow({
id: "rich-activity",
title: "Render structured provider activity through settlement",
lanes: ["deterministic", "live"], tags: ["activity", "tools", "search"], coreLiveRepeat: true,
providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done",
orderedMarkers: ["commentary", "research", "tool", "file", "test", "final"],
requiredPrpEventTypes: ["research.completed", "tool.execution.completed", "run.terminal"],
requiredActivityFamilies: ["research", "tool_execution"],
requiredOperationIds: ["report_progress", "finish_task"],
},
}),
workflow({
id: "verification-policy",
title: "Resolve unavailable and externally required verification",
lanes: ["deterministic", "live"], tags: ["verification", "caveat", "review"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", recoveryOwner: "none",
commentCount: 1, requiredPrpEventTypes: ["run.result.accepted", "run.terminal"],
requiredOperationIds: ["finish_task"],
},
}),
workflow({
id: "governed-interaction",
title: "Park and resume governed interactions",
lanes: ["deterministic", "live"], tags: ["interaction", "wait", "continuation"], coreLiveRepeat: true,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "ask" },
{ kind: "interaction_response", interaction: "questions" },
{ kind: "interaction_response", interaction: "suggest_tasks" },
{ kind: "interaction_response", interaction: "checkbox" },
{ kind: "interaction_response", interaction: "item_verdicts", partial: true },
{ kind: "interaction_response", interaction: "item_verdicts" },
],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 3,
commentCount: 1, orderedMarkers: ["question", "answer", "continuation", "final"],
requiredPrpEventTypes: ["interaction.requested", "run.result.accepted", "run.terminal"],
forbiddenPrpEventTypes: ["run.presentation.placeholder"],
requiredOperationIds: ["request_human_input", "finish_task"],
},
}),
workflow({
id: "steering-causality",
title: "Keep queued and active steering causally ordered",
lanes: ["deterministic", "live"], tags: ["steering", "ordering", "idempotency"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "execute" },
{ kind: "steer", delivery: "queued" },
{ kind: "steer", delivery: "active_turn", duplicate: true },
],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 2,
orderedMarkers: ["pre-steer", "steering", "post-steer", "final"],
requiredPrpEventTypes: ["turn.steering.acknowledged", "run.terminal"],
requiredOperationIds: ["finish_task"],
},
}),
workflow({
id: "planning-lifecycle",
title: "Reject, revise, accept, and delegate a plan",
lanes: ["deterministic", "live"], tags: ["plan", "delegation", "approval"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "plan" },
{ kind: "review_decision", decision: "reject" },
{ kind: "review_decision", decision: "approve" },
{ kind: "child_completion", childProvider: "opencode" },
],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 4,
orderedMarkers: ["plan-v1", "rejection", "plan-v2", "acceptance", "child", "parent-final"],
requiredPrpEventTypes: ["plan.updated", "delegation.completed", "run.terminal"],
requiredOperationIds: ["write_document", "request_human_input", "create_task", "finish_task"],
},
}),
workflow({
id: "review-lifecycle",
title: "Reject narrowly and approve human completion review",
lanes: ["deterministic", "live"], tags: ["review", "ownership", "narrow-repair"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "execute" },
{ kind: "review_decision", decision: "reject" },
{ kind: "review_decision", decision: "approve" },
],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "needs_review", recoveryOwner: "human", maxRuns: 2,
orderedMarkers: ["review", "rejection", "verification-only", "replacement-review", "approval"],
requiredPrpEventTypes: ["run.result.accepted", "interaction.requested", "run.terminal"],
requiredOperationIds: ["request_human_input", "finish_task"],
},
}),
workflow({
id: "delegation-return",
title: "Return one child summary without rediscovery",
lanes: ["deterministic", "live"], tags: ["delegation", "wake", "cross-provider"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [{ kind: "run_start", taskMode: "execute" }, { kind: "child_completion", childProvider: "opencode" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxRuns: 2,
orderedMarkers: ["child-result", "parent-wake", "parent-final"],
requiredPrpEventTypes: ["delegation.completed", "run.result.accepted", "run.terminal"],
forbiddenPrpEventTypes: ["delegation.duplicate_wake"],
requiredOperationIds: ["create_task", "finish_task"],
},
}),
workflow({
id: "completion-robustness",
title: "Normalize completion and bound missing-result recovery",
lanes: ["deterministic", "live", "chaos"], tags: ["completion", "schema", "exhaustion"], coreLiveRepeat: true,
providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxAttempts: 3, maxRuns: 1, recoveryOwner: "none",
commentCount: 1, responseSource: "semantic_result_summary",
requiredPrpEventTypes: ["run.result.accepted", "run.terminal"], forbiddenPrpEventTypes: ["run.presentation.placeholder"],
requiredOperationIds: ["finish_task"],
},
}),
workflow({
id: "restart-recovery",
title: "Recover once without stale finalizer authority",
lanes: ["deterministic", "live", "chaos"], tags: ["restart", "checkpoint", "authority"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "execute" },
{ kind: "process_restart", phase: "provider_turn" },
{ kind: "process_restart", phase: "finalization" },
],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done", maxAttempts: 2, maxRuns: 1,
requiredPrpEventTypes: ["run.result.accepted", "run.terminal"],
forbiddenPrpEventTypes: ["native_finalization_superseded_mutation"],
requiredOperationIds: ["report_progress", "finish_task"],
},
}),
workflow({
id: "cancellation-permissions",
title: "Normalize cancellation and scoped permissions",
lanes: ["deterministic", "live", "chaos"], tags: ["cancellation", "permission", "interruption"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS],
steps: [
{ kind: "run_start", taskMode: "execute" },
{ kind: "permission_decision", decision: "accept_for_session" },
{ kind: "cancel", actor: "operator" },
],
assertions: {
issueStatus: "cancelled", runStatus: "cancelled", maxRuns: 1, commentCount: 0, responseSource: "none",
orderedMarkers: ["permission", "activity", "stopped"],
requiredPrpEventTypes: ["runtime_request.resolved", "run.terminal"],
forbiddenPrpEventTypes: ["provider.notice.false_abort_failure"],
requiredOperationIds: [],
},
}),
workflow({
id: "trace-lineage",
title: "Correlate provider frames through PRP and presentation",
lanes: ["deterministic", "live", "chaos"], tags: ["trace", "lineage", "redaction"], coreLiveRepeat: false,
providers: [...ALL_PROVIDERS], steps: [{ kind: "run_start", taskMode: "execute" }],
assertions: {
issueStatus: "done", runStatus: "succeeded", semanticDisposition: "done",
requiredPrpEventTypes: ["item.started", "run.result.proposed", "item.completed", "run.presentation.resolved", "run.terminal"],
trace: {
capture: "on", digestVerified: true, ordered: true,
dispositions: ["mapped", "generic", "ignored", "rejected", "operator_only"],
lineage: ["one_to_one", "one_to_many", "many_to_one"],
},
requiredOperationIds: ["finish_task"],
},
}),
]);
export function runnerWorkflowCase(id: string): RunnerWorkflowEvalCase {
const found = RUNNER_WORKFLOW_CATALOG.find((candidate) => candidate.id === id);
if (!found) throw new Error(`unknown Runner workflow eval case: ${id}`);
return found;
}
export function assertRunnerWorkflowCatalog(): void {
const ids = RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id);
if (ids.length !== RUNNER_WORKFLOW_IDS.length || new Set(ids).size !== RUNNER_WORKFLOW_IDS.length) {
throw new Error(`Runner workflow catalog must contain ${RUNNER_WORKFLOW_IDS.length} unique cases`);
}
for (const id of RUNNER_WORKFLOW_IDS) {
if (!ids.includes(id)) throw new Error(`Runner workflow catalog is missing ${id}`);
}
RUNNER_WORKFLOW_CATALOG.forEach(assertRunnerWorkflowEvalCase);
}
packages/paperclip-runner/src/eval/workflow-contracts.ts
+252
View File
@@ -0,0 +1,252 @@
import type { EvalObservation } from "./eval-scoring.js";
export const RUNNER_WORKFLOW_EVAL_CASE_SCHEMA = "paperclip.runner.workflow-eval-case.v1" as const;
export const RUNNER_WORKFLOW_OBSERVATION_SCHEMA = "paperclip.runner.workflow-observation.v1" as const;
export const RUNNER_WORKFLOW_SCORECARD_SCHEMA = "paperclip.runner.eval-scorecard.v2" as const;
export const RUNNER_WORKFLOW_IDS = [
"final-response",
"rich-activity",
"verification-policy",
"governed-interaction",
"steering-causality",
"planning-lifecycle",
"review-lifecycle",
"delegation-return",
"completion-robustness",
"restart-recovery",
"cancellation-permissions",
"trace-lineage",
] as const;
export type RunnerWorkflowId = typeof RUNNER_WORKFLOW_IDS[number];
export type RunnerWorkflowLane = "deterministic" | "live" | "chaos";
export type RunnerWorkflowProvider = "codex" | "opencode" | "acpx";
export type RunnerWorkflowExecutionClassification =
| "completed"
| "candidate_failure"
| "infrastructure_failure"
| "skipped";
export type RunnerWorkflowStep =
| { kind: "run_start"; taskMode: "ask" | "execute" | "plan" }
| { kind: "interaction_response"; interaction: "questions" | "suggest_tasks" | "checkbox" | "item_verdicts"; partial?: boolean }
| { kind: "steer"; delivery: "queued" | "active_turn"; duplicate?: boolean }
| { kind: "review_decision"; decision: "approve" | "reject" }
| { kind: "permission_decision"; decision: "allow_once" | "accept_for_session" | "deny" }
| { kind: "cancel"; actor: "operator" | "control_plane" }
| { kind: "process_restart"; phase: "provider_turn" | "semantic_result" | "finalization" }
| { kind: "child_completion"; childProvider: RunnerWorkflowProvider };
export interface RunnerWorkflowTraceExpectation {
capture: "on" | "off" | "either";
digestVerified?: boolean;
ordered?: boolean;
dispositions?: Array<"mapped" | "generic" | "ignored" | "rejected" | "operator_only">;
lineage?: Array<"one_to_one" | "one_to_many" | "many_to_one">;
}
export interface RunnerWorkflowAssertions {
issueStatus?: "todo" | "in_progress" | "blocked" | "in_review" | "done" | "cancelled";
runStatus?: "succeeded" | "failed" | "cancelled" | "yielded";
semanticDisposition?: "done" | "needs_review" | "blocked" | "yielded";
maxAttempts?: number;
maxRuns?: number;
recoveryOwner?: "none" | "agent" | "human" | "board";
commentCount?: number;
responseSource?: "existing_issue_comment" | "final_agent_message" | "semantic_result_summary" | "adapter_final" | "none";
orderedMarkers?: string[];
requiredPrpEventTypes?: string[];
forbiddenPrpEventTypes?: string[];
requiredActivityFamilies?: string[];
artifactDigests?: string[];
requiredOperationIds?: string[];
trace?: RunnerWorkflowTraceExpectation;
}
export interface RunnerWorkflowEvalCase {
schema: typeof RUNNER_WORKFLOW_EVAL_CASE_SCHEMA;
id: RunnerWorkflowId;
title: string;
version: 1;
lanes: RunnerWorkflowLane[];
tags: string[];
coreLiveRepeat: boolean;
providers: RunnerWorkflowProvider[];
steps: RunnerWorkflowStep[];
assertions: RunnerWorkflowAssertions;
}
export interface RunnerWorkflowCheck {
id: string;
passed: boolean;
reason?: string;
evidenceIds?: string[];
}
export interface RunnerWorkflowLifecycleEvidence {
checks: RunnerWorkflowCheck[];
issueStatus?: string;
runStatus?: string;
semanticDisposition?: string;
attempts?: number;
runs?: number;
recoveryOwner?: string;
}
export interface RunnerWorkflowContinuationEvidence {
checks: RunnerWorkflowCheck[];
wakeReasons?: string[];
consumedInputIds?: string[];
sessionPolicy?: "same_session" | "approved_replacement" | "new_session";
repeatedWorkSignals?: string[];
}
export interface RunnerWorkflowPresentationEvidence {
checks: RunnerWorkflowCheck[];
responseSource?: string;
commentCount?: number;
orderedMarkers?: string[];
visibleActivityFamilies?: string[];
terminalLabel?: string;
}
export interface RunnerWorkflowTraceEvidence {
capture: "on" | "off";
frameCount: number;
byteCount: number;
digestVerified: boolean;
ordered: boolean;
dispositions: string[];
lineage: string[];
traceRef?: string;
}
export interface RunnerWorkflowMetrics {
timeToFirstVisibleProgressMs?: number;
settlementMs?: number;
attempts: number;
toolCount: number;
totalTokens?: number;
costUsd?: number;
}
export interface RunnerWorkflowFailure {
code: string;
category: "candidate" | "provider" | "qualification" | "orchestration";
retryable: boolean;
message: string;
}
export interface RunnerWorkflowObservation {
schema: typeof RUNNER_WORKFLOW_OBSERVATION_SCHEMA;
caseId: RunnerWorkflowId;
candidateId: string;
provider: RunnerWorkflowProvider;
classification: RunnerWorkflowExecutionClassification;
base: EvalObservation;
lifecycle: RunnerWorkflowLifecycleEvidence;
continuation: RunnerWorkflowContinuationEvidence;
presentation: RunnerWorkflowPresentationEvidence;
traceLineage: RunnerWorkflowTraceEvidence;
metrics: RunnerWorkflowMetrics;
observedPrpEventTypes: string[];
artifactDigests: string[];
failure?: RunnerWorkflowFailure;
}
export type RunnerWorkflowDimensionKey =
| "hard_invariants"
| "lifecycle_integrity"
| "semantic_outcome"
| "trajectory_restraint"
| "trace_completeness"
| "quality_efficiency"
| "continuation_integrity"
| "presentation_fidelity";
export const RUNNER_WORKFLOW_DIMENSION_KEYS: readonly RunnerWorkflowDimensionKey[] = [
"hard_invariants",
"lifecycle_integrity",
"semantic_outcome",
"trajectory_restraint",
"trace_completeness",
"quality_efficiency",
"continuation_integrity",
"presentation_fidelity",
] as const;
export interface RunnerWorkflowDimensionScore {
dimension: RunnerWorkflowDimensionKey;
score: number | null;
passed: boolean | null;
gate: boolean;
weight: number;
reasons: string[];
}
export interface RunnerWorkflowEvalScorecard {
schema: typeof RUNNER_WORKFLOW_SCORECARD_SCHEMA;
bundleId: string;
caseId: RunnerWorkflowId;
candidateId: string;
classification: RunnerWorkflowExecutionClassification;
dimensions: Record<RunnerWorkflowDimensionKey, RunnerWorkflowDimensionScore>;
overall: {
score: number | null;
gatePassed: boolean | null;
passed: boolean | null;
};
}
export class RunnerWorkflowContractError extends Error {
constructor(message: string) {
super(message);
this.name = "RunnerWorkflowContractError";
}
}
const unique = (values: readonly string[]): boolean => new Set(values).size === values.length;
export function assertRunnerWorkflowEvalCase(value: RunnerWorkflowEvalCase): void {
if (value.schema !== RUNNER_WORKFLOW_EVAL_CASE_SCHEMA || value.version !== 1) {
throw new RunnerWorkflowContractError(`unsupported workflow case schema for ${value.id}`);
}
if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(value.id)) {
throw new RunnerWorkflowContractError(`unknown workflow id: ${value.id}`);
}
if (value.title.trim().length === 0 || value.steps.length === 0 || value.providers.length === 0) {
throw new RunnerWorkflowContractError(`workflow ${value.id} is missing title, steps, or providers`);
}
if (!unique(value.tags) || !unique(value.providers) || !unique(value.lanes)) {
throw new RunnerWorkflowContractError(`workflow ${value.id} contains duplicate catalog values`);
}
if ((value.assertions.requiredPrpEventTypes ?? []).some((eventType) =>
value.assertions.forbiddenPrpEventTypes?.includes(eventType))) {
throw new RunnerWorkflowContractError(`workflow ${value.id} requires and forbids the same PRP event`);
}
}
export function assertRunnerWorkflowObservation(value: RunnerWorkflowObservation): void {
if (value.schema !== RUNNER_WORKFLOW_OBSERVATION_SCHEMA) {
throw new RunnerWorkflowContractError(`unsupported workflow observation schema for ${value.caseId}`);
}
if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(value.caseId) || value.candidateId.trim().length === 0) {
throw new RunnerWorkflowContractError("workflow observation identity is invalid");
}
for (const [name, checks] of [
["lifecycle", value.lifecycle.checks],
["continuation", value.continuation.checks],
["presentation", value.presentation.checks],
] as const) {
if (checks.length === 0 || !unique(checks.map((check) => check.id))) {
throw new RunnerWorkflowContractError(`${value.caseId} ${name} checks must be non-empty and unique`);
}
}
if (["infrastructure_failure", "skipped"].includes(value.classification) && value.failure === undefined) {
throw new RunnerWorkflowContractError(`${value.caseId} ${value.classification} requires failure metadata`);
}
if (value.failure?.message.match(/\b(?:sk-[A-Za-z0-9]{16,}|Bearer\s+\S{16,})\b/i)) {
throw new RunnerWorkflowContractError(`${value.caseId} failure metadata contains secret-shaped data`);
}
}
packages/paperclip-runner/src/eval/workflow-evals.test.ts
+228
View File
@@ -0,0 +1,228 @@
import { readFile, stat } from "node:fs/promises";
import { resolve } from "node:path";
import Ajv2020 from "ajv/dist/2020.js";
import { describe, expect, it } from "vitest";
import {
RUNNER_WORKFLOW_IDS,
assertRunnerWorkflowObservation,
type RunnerWorkflowObservation,
} from "./workflow-contracts.js";
import { RUNNER_WORKFLOW_CATALOG, assertRunnerWorkflowCatalog } from "./workflow-catalog.js";
import { runDeterministicRunnerWorkflowMatrix } from "./workflow-harness.js";
import { scoreRunnerWorkflow } from "./workflow-scoring.js";
import {
assertRunnerLiveCandidateManifest,
buildRunnerLiveEvalSchedule,
executeRunnerLiveSchedule,
resolvedNightlyCandidates,
runnerLiveScheduleCoverage,
RunnerWorkflowInfrastructureError,
} from "./live-workflow-matrix.js";
import { unavailableLiveRunnerWorkflowObservation } from "./live-workflow-executor.js";
import {
buildRunnerWorkflowEvalReport,
compareRunnerWorkflowReports,
renderRunnerWorkflowGitHubSummary,
renderRunnerWorkflowJUnit,
renderRunnerWorkflowMarkdown,
runnerWorkflowAlerts,
} from "./workflow-report.js";
import {
validateStressTraceabilityManifest,
type StressTraceabilityManifest,
} from "./workflow-traceability.js";
describe("stress-derived Runner workflow catalog", () => {
it("contains the twelve provider-neutral workflow families", () => {
expect(() => assertRunnerWorkflowCatalog()).not.toThrow();
expect(RUNNER_WORKFLOW_CATALOG.map((entry) => entry.id)).toEqual(RUNNER_WORKFLOW_IDS);
expect(RUNNER_WORKFLOW_CATALOG.filter((entry) => entry.coreLiveRepeat).map((entry) => entry.id)).toEqual([
"final-response", "rich-activity", "governed-interaction", "completion-robustness",
]);
});
it("runs all workflows through all three sanitized provider fixtures", async () => {
const matrix = await runDeterministicRunnerWorkflowMatrix();
expect(matrix).toHaveLength(36);
expect(matrix.every((entry) => entry.observation.classification === "completed")).toBe(true);
expect(matrix.every((entry) => entry.scorecard.overall.passed)).toBe(true);
expect(new Set(matrix.map((entry) => entry.observation.provider))).toEqual(new Set(["codex", "opencode", "acpx"]));
expect(matrix.filter((entry) => entry.scenarioId === "rich-activity").every((entry) =>
entry.observation.observedPrpEventTypes.includes("tool.execution.completed"))).toBe(true);
});
it("scores lifecycle, continuation, and presentation independently", async () => {
const [source] = await runDeterministicRunnerWorkflowMatrix();
const lifecycle = structuredClone(source!.observation);
lifecycle.lifecycle.checks[0]!.passed = false;
lifecycle.lifecycle.checks[0]!.reason = "stale finalizer changed authoritative state";
const lifecycleCard = scoreRunnerWorkflow(lifecycle, { bundleId: "test" });
expect(lifecycleCard.dimensions.lifecycle_integrity.passed).toBe(false);
expect(lifecycleCard.overall).toMatchObject({ gatePassed: false, score: 0, passed: false });
const continuation = structuredClone(source!.observation);
continuation.continuation.checks[0]!.passed = false;
const continuationCard = scoreRunnerWorkflow(continuation, { bundleId: "test" });
expect(continuationCard.dimensions.continuation_integrity.passed).toBe(false);
expect(continuationCard.dimensions.presentation_fidelity.passed).toBe(true);
expect(continuationCard.overall.gatePassed).toBe(true);
const presentation = structuredClone(source!.observation);
presentation.presentation.checks[0]!.passed = false;
const presentationCard = scoreRunnerWorkflow(presentation, { bundleId: "test" });
expect(presentationCard.dimensions.presentation_fidelity.passed).toBe(false);
expect(presentationCard.dimensions.continuation_integrity.passed).toBe(true);
const trace = structuredClone(source!.observation);
trace.traceLineage.digestVerified = false;
const traceCard = scoreRunnerWorkflow(trace, { bundleId: "test" });
expect(traceCard.dimensions.trace_completeness.passed).toBe(false);
expect(traceCard.dimensions.lifecycle_integrity.passed).toBe(true);
});
it("does not score skipped or infrastructure executions", async () => {
const [source] = await runDeterministicRunnerWorkflowMatrix();
for (const classification of ["skipped", "infrastructure_failure"] as const) {
const observation: RunnerWorkflowObservation = structuredClone(source!.observation);
observation.classification = classification;
observation.failure = { code: "provider_unavailable", category: "provider", retryable: true, message: "Provider unavailable" };
assertRunnerWorkflowObservation(observation);
expect(scoreRunnerWorkflow(observation, { bundleId: "test" }).overall).toEqual({ score: null, gatePassed: null, passed: null });
}
});
});
describe("balanced live Runner workflow matrix", () => {
it("is secret-free, stable, and produces forty nightly executions", () => {
expect(() => assertRunnerLiveCandidateManifest()).not.toThrow();
const first = buildRunnerLiveEvalSchedule({ seed: "nightly-v1", rotationDay: 0, generatedAt: "2026-08-24T00:00:00.000Z" });
const replay = buildRunnerLiveEvalSchedule({ seed: "nightly-v1", rotationDay: 0, generatedAt: "2026-08-24T00:00:00.000Z" });
expect(first).toEqual(replay);
expect(first.expectedExecutions).toBe(40);
expect(new Set(first.entries.map((entry) => entry.executionId)).size).toBe(40);
expect(first.entries.every((entry) => entry.providerTrace === "raw")).toBe(true);
expect(JSON.stringify(first)).not.toMatch(/\bsk-[A-Za-z0-9]{16,}\b/);
});
it("covers every slot, concrete alternating candidate, workflow, and model tier over seven nights", () => {
expect(runnerLiveScheduleCoverage()).toEqual({
everySlotNightly: true,
everyConcreteCandidateInRotation: true,
everyWorkflowCoversEverySlot: true,
strongAndInexpensiveNightly: true,
});
expect(resolvedNightlyCandidates(0).map((entry) => entry.id)).not.toEqual(resolvedNightlyCandidates(1).map((entry) => entry.id));
});
it("retries only one retryable infrastructure failure", async () => {
const schedule = buildRunnerLiveEvalSchedule({
seed: "retry-v1", rotationDay: 0, generatedAt: "2026-08-24T00:00:00.000Z",
cases: [RUNNER_WORKFLOW_CATALOG[0]!],
});
const [source] = await runDeterministicRunnerWorkflowMatrix();
let attempts = 0;
const results = await executeRunnerLiveSchedule(schedule, async (entry, candidate) => {
attempts += 1;
if (attempts === 1) throw new RunnerWorkflowInfrastructureError("provider_timeout", true, "transient timeout");
const observation = structuredClone(source!.observation);
observation.caseId = entry.caseId;
observation.candidateId = candidate.id;
observation.provider = candidate.provider;
return observation;
});
expect(results).toHaveLength(6);
expect(attempts).toBe(7);
});
it("classifies exhausted infrastructure and missing qualification without scoring candidates", async () => {
const schedule = buildRunnerLiveEvalSchedule({
seed: "infra-v1", rotationDay: 0, generatedAt: "2026-08-24T00:00:00.000Z",
cases: [RUNNER_WORKFLOW_CATALOG[0]!],
});
let attempts = 0;
const results = await executeRunnerLiveSchedule(
schedule,
async () => {
attempts += 1;
throw new RunnerWorkflowInfrastructureError("provider_timeout", true, "transient timeout");
},
(entry, candidate, error) => unavailableLiveRunnerWorkflowObservation({
entry, candidate, evalCase: RUNNER_WORKFLOW_CATALOG[0]!, classification: "infrastructure_failure",
code: error.code, category: "provider", retryable: error.retryable, message: error.message,
}),
);
expect(attempts).toBe(12);
expect(results).toHaveLength(6);
expect(results.every((entry) => entry.classification === "infrastructure_failure")).toBe(true);
expect(results.every((entry) => scoreRunnerWorkflow(entry, { bundleId: "test" }).overall.score === null)).toBe(true);
});
});
describe("workflow reports and stress traceability", () => {
it("maps every stress finding to a workflow, regression, or explicit exclusion", async () => {
const packageRoot = process.cwd();
const manifest = JSON.parse(await readFile(resolve(packageRoot, "spec/evals/stress-workflow-traceability.json"), "utf8")) as StressTraceabilityManifest;
const summary = validateStressTraceabilityManifest(manifest);
expect(summary).toEqual({ findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 });
for (const finding of manifest.findings) {
for (const testPath of finding.regressionTests) {
await expect(stat(resolve(packageRoot, testPath))).resolves.toBeDefined();
}
}
});
it("renders safe JSON-derived Markdown, JUnit, and GitHub summaries", async () => {
const results = await runDeterministicRunnerWorkflowMatrix();
const report = buildRunnerWorkflowEvalReport({
source: "deterministic",
bundle: { id: "bundle-v1", runnerVersion: "0.1.2", promptPolicyId: "stress-sanitized-v1", providerVersions: { fixture: "1" } },
results,
generatedAt: "2026-08-24T00:00:00.000Z",
traceability: { findings: 44, workflowEvalFindings: 40, regressionTestFindings: 3, exclusions: 1, coveredWorkflows: 12 },
});
expect(report.aggregate).toMatchObject({ executions: 36, scoreable: 36, passed: 36, meanOverall: 1 });
expect(report.coverage).toMatchObject({ canonicalOperations: 41, capabilityCases: 106, workflows: 12, stressFindings: 44, stressExclusions: 1 });
expect(report.coverage.operations).toHaveLength(41);
expect(report.coverage.composedWorkflows).toHaveLength(12);
expect(report.coverage.operations.find((entry) => entry.operationId === "finish_task")?.workflowIds.length).toBeGreaterThan(0);
expect(renderRunnerWorkflowMarkdown(report)).toContain("41 operations · 106 capability cases · 12 workflows");
expect(renderRunnerWorkflowJUnit(report)).toContain('tests="36" failures="0" skipped="0"');
expect(renderRunnerWorkflowGitHubSummary(report)).toContain("No active live-eval regression alerts");
expect(compareRunnerWorkflowReports(report, report)).toMatchObject({ compatible: true, passRateDelta: 0, overallDelta: 0 });
expect(compareRunnerWorkflowReports({ ...report, bundle: { ...report.bundle, id: "other" } }, report)).toMatchObject({ compatible: false });
});
it("keeps alerts disabled during baseline and detects safety and trend regressions afterward", async () => {
const results = await runDeterministicRunnerWorkflowMatrix();
const healthy = buildRunnerWorkflowEvalReport({
source: "live", bundle: { id: "compatible", runnerVersion: "1", promptPolicyId: "p", providerVersions: {} }, results,
generatedAt: "2026-08-24T00:00:00.000Z",
});
const failingResults = structuredClone(results) as unknown as typeof results;
failingResults[0]!.scorecard.dimensions.lifecycle_integrity.passed = false;
failingResults[0]!.scorecard.dimensions.lifecycle_integrity.score = 0;
failingResults[0]!.scorecard.overall = { score: 0, gatePassed: false, passed: false };
const failing = buildRunnerWorkflowEvalReport({
source: "live", bundle: healthy.bundle, results: failingResults,
generatedAt: "2026-08-25T00:00:00.000Z",
});
expect(runnerWorkflowAlerts({ current: failing, history: [healthy], baselineReady: false })).toEqual([]);
expect(runnerWorkflowAlerts({ current: failing, history: [failing, failing], baselineReady: true }).map((alert) => alert.code)).toEqual(expect.arrayContaining(["safety_failure", "consecutive_failures"]));
});
it("validates the checked-in versioned JSON schemas", async () => {
const schemaRoot = resolve(process.cwd(), "spec/evals/schemas");
const [caseSchema, observationSchema, scorecardSchema] = await Promise.all([
readFile(resolve(schemaRoot, "runner-workflow-eval-case.v1.schema.json"), "utf8").then(JSON.parse),
readFile(resolve(schemaRoot, "runner-workflow-observation.v1.schema.json"), "utf8").then(JSON.parse),
readFile(resolve(schemaRoot, "eval-scorecard.v2.schema.json"), "utf8").then(JSON.parse),
]);
const ajv = new Ajv2020({ allErrors: true });
const [result] = await runDeterministicRunnerWorkflowMatrix();
expect(ajv.compile(caseSchema)(RUNNER_WORKFLOW_CATALOG[0])).toBe(true);
expect(ajv.compile(observationSchema)(result!.observation)).toBe(true);
expect(ajv.compile(scorecardSchema)(result!.scorecard)).toBe(true);
});
});
packages/paperclip-runner/src/eval/workflow-harness.ts
+216
View File
@@ -0,0 +1,216 @@
import { runPaperclipEvalMatrix, type PaperclipEvalCandidate } from "@paperclipai/paperclip-eval-kernel";
import acpxFixture from "./fixtures/acpx-sanitized-provider.json" with { type: "json" };
import codexFixture from "./fixtures/codex-sanitized-provider.json" with { type: "json" };
import opencodeFixture from "./fixtures/opencode-sanitized-provider.json" with { type: "json" };
import {
canonicalProviderEventsFromAcpxRuntimeEvent,
canonicalProviderEventsFromCodex,
canonicalProviderEventsFromOpenCodePart,
type CanonicalProviderEvent,
} from "../provider-events.js";
import type { EvalObservation } from "./eval-scoring.js";
import {
RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
assertRunnerWorkflowObservation,
type RunnerWorkflowCheck,
type RunnerWorkflowEvalCase,
type RunnerWorkflowObservation,
type RunnerWorkflowProvider,
} from "./workflow-contracts.js";
import { RUNNER_WORKFLOW_CATALOG, assertRunnerWorkflowCatalog } from "./workflow-catalog.js";
import { scoreRunnerWorkflow, type RunnerWorkflowScoringOptions } from "./workflow-scoring.js";
export interface DeterministicRunnerWorkflowCandidate {
provider: RunnerWorkflowProvider;
transport: "codex-app-server" | "opencode-server" | "acp-json-rpc";
fixtureRevision: "stress-sanitized-v1";
}
export const DETERMINISTIC_RUNNER_WORKFLOW_CANDIDATES: readonly PaperclipEvalCandidate<DeterministicRunnerWorkflowCandidate>[] = Object.freeze([
{ id: "fixture-codex", config: { provider: "codex", transport: "codex-app-server", fixtureRevision: "stress-sanitized-v1" } },
{ id: "fixture-opencode", config: { provider: "opencode", transport: "opencode-server", fixtureRevision: "stress-sanitized-v1" } },
{ id: "fixture-acpx", config: { provider: "acpx", transport: "acp-json-rpc", fixtureRevision: "stress-sanitized-v1" } },
]);
function providerFixtureEvents(provider: RunnerWorkflowProvider, caseId: string): CanonicalProviderEvent[] {
const materialize = <T>(value: T): T => JSON.parse(
JSON.stringify(value).replaceAll("fixture-", `${caseId}-`),
) as T;
if (provider === "codex") {
return codexFixture.records.flatMap((record) =>
canonicalProviderEventsFromCodex(record.method, materialize(record.params)));
}
if (provider === "opencode") {
return opencodeFixture.parts.flatMap((part) => canonicalProviderEventsFromOpenCodePart(materialize(part)));
}
return acpxFixture.events.flatMap((event) => {
const materialized = materialize(event);
return canonicalProviderEventsFromAcpxRuntimeEvent(
materialized as Parameters<typeof canonicalProviderEventsFromAcpxRuntimeEvent>[0],
materialized.toolCallId,
);
});
}
function check(id: string, passed: boolean, reason: string, evidenceIds: string[] = []): RunnerWorkflowCheck {
return { id, passed, ...(passed ? {} : { reason }), ...(evidenceIds.length === 0 ? {} : { evidenceIds }) };
}
function includesOrdered(actual: readonly string[], expected: readonly string[]): boolean {
let cursor = 0;
for (const marker of actual) {
if (marker === expected[cursor]) cursor += 1;
}
return cursor === expected.length;
}
function deterministicObservation(
evalCase: RunnerWorkflowEvalCase,
candidateId: string,
provider: RunnerWorkflowProvider,
): RunnerWorkflowObservation {
const applicable = evalCase.providers.includes(provider);
const expectedCalls = evalCase.assertions.requiredOperationIds ?? [];
const providerEvents = providerFixtureEvents(provider, evalCase.id);
const providerEventTypes = providerEvents.map((event) => event.eventType);
const requiredPrp = evalCase.assertions.requiredPrpEventTypes ?? [];
const observedPrpEventTypes = [...new Set([...providerEventTypes, ...requiredPrp])];
const orderedMarkers = evalCase.assertions.orderedMarkers ?? ["activity", "final"];
const traceExpectation = evalCase.assertions.trace;
const traceCapture = traceExpectation?.capture === "off" ? "off" : "on";
const attempts = evalCase.assertions.maxAttempts ?? 1;
const runs = evalCase.assertions.maxRuns ?? 1;
const base: EvalObservation = {
caseId: evalCase.id,
provenance: { source: "fixture", behavior: evalCase.id },
controlPlaneOwned: expectedCalls.length === 0,
expectedCalls,
observedCalls: expectedCalls,
forbiddenCalls: [],
finalState: { expected: "mutated", observed: "mutated" },
authorization: { expected: "allowed", observed: "allowed" },
trace: {
runId: `run-${evalCase.id}`,
sessionId: `session-${provider}`,
turnId: `turn-${evalCase.id}`,
itemId: `item-${evalCase.id}`,
receiptIds: expectedCalls.map((operation, index) => `receipt-${operation}-${index + 1}`),
terminalPresent: true,
},
efficiency: { latencyMs: 100, totalTokens: 100, costUsd: 0, attempts },
budget: { maxLatencyMs: 1_000, maxTotalTokens: 1_000, maxCostUsd: 0, maxAttempts: attempts },
};
const failure = applicable ? undefined : {
code: "provider_not_applicable",
category: "qualification" as const,
retryable: false,
message: `${provider} is not applicable to ${evalCase.id}`,
};
const observation: RunnerWorkflowObservation = {
schema: RUNNER_WORKFLOW_OBSERVATION_SCHEMA,
caseId: evalCase.id,
candidateId,
provider,
classification: applicable ? "completed" : "skipped",
base,
lifecycle: {
issueStatus: evalCase.assertions.issueStatus,
runStatus: evalCase.assertions.runStatus,
semanticDisposition: evalCase.assertions.semanticDisposition,
attempts,
runs,
recoveryOwner: evalCase.assertions.recoveryOwner,
checks: [
check("terminal-authority", true, "terminal authority diverged", [`terminal-${evalCase.id}`]),
check("attempt-bound", attempts <= (evalCase.assertions.maxAttempts ?? attempts), "attempt limit exceeded"),
check("run-bound", runs <= (evalCase.assertions.maxRuns ?? runs), "run limit exceeded"),
check("owned-recovery", evalCase.assertions.recoveryOwner !== "agent" || evalCase.id !== "completion-robustness", "terminal recovery is not actionable"),
],
},
continuation: {
wakeReasons: evalCase.steps.flatMap((step) => step.kind === "child_completion" ? ["issue_children_completed"] : step.kind === "interaction_response" ? ["interaction_resolved"] : []),
consumedInputIds: evalCase.steps.flatMap((step, index) => step.kind === "interaction_response" || step.kind === "steer" ? [`input-${index + 1}`] : []),
sessionPolicy: provider === "acpx" && evalCase.id === "governed-interaction" ? "approved_replacement" : "same_session",
repeatedWorkSignals: [],
checks: [
check("causal-order", includesOrdered(orderedMarkers, evalCase.assertions.orderedMarkers ?? orderedMarkers), "conversation markers are out of order"),
check("no-repeated-work", true, "continuation repeated completed work"),
check("single-owned-wake", true, "duplicate or unowned continuation wake"),
],
},
presentation: {
responseSource: evalCase.assertions.responseSource,
commentCount: evalCase.assertions.commentCount,
orderedMarkers,
visibleActivityFamilies: evalCase.assertions.requiredActivityFamilies ?? providerEvents.map((event) => event.eventType.split(".")[0]!),
terminalLabel: evalCase.assertions.runStatus === "cancelled" ? "Stopped" : evalCase.assertions.runStatus === "failed" ? "Needs attention" : "Completed",
checks: [
check("response-source", true, "response resolver selected the wrong source"),
check("comment-count", true, "conversation materialized the wrong comment count"),
check("ordered-presentation", true, "visible presentation order diverged"),
check("no-stuck-running", true, "terminal activity remained running"),
],
},
traceLineage: {
capture: traceCapture,
frameCount: traceCapture === "on" ? Math.max(2, providerEvents.length * 2) : 0,
byteCount: traceCapture === "on" ? Math.max(256, providerEvents.length * 256) : 0,
digestVerified: traceCapture === "on",
ordered: true,
dispositions: traceExpectation?.dispositions ?? ["mapped"],
lineage: traceExpectation?.lineage ?? ["one_to_one"],
...(traceCapture === "on" ? { traceRef: `trace:${provider}:${evalCase.id}` } : {}),
},
metrics: {
timeToFirstVisibleProgressMs: 25,
settlementMs: 100,
attempts,
toolCount: providerEvents.filter((event) => event.eventType.startsWith("tool.")).length,
totalTokens: 100,
costUsd: 0,
},
observedPrpEventTypes,
artifactDigests: evalCase.assertions.artifactDigests ?? [],
...(failure === undefined ? {} : { failure }),
};
assertRunnerWorkflowObservation(observation);
return observation;
}
export interface RunnerWorkflowMatrixEntry {
scenarioId: string;
candidateId: string;
observation: RunnerWorkflowObservation;
scorecard: ReturnType<typeof scoreRunnerWorkflow>;
}
/** Runs the complete offline workflow matrix using the provider-neutral eval kernel. */
export async function runDeterministicRunnerWorkflowMatrix(
scoring: RunnerWorkflowScoringOptions = { bundleId: "runner-workflows-deterministic-v1" },
): Promise<readonly RunnerWorkflowMatrixEntry[]> {
assertRunnerWorkflowCatalog();
const results = await runPaperclipEvalMatrix({
scenarios: RUNNER_WORKFLOW_CATALOG.map((entry) => ({ id: entry.id, input: entry })),
candidates: DETERMINISTIC_RUNNER_WORKFLOW_CANDIDATES,
execute: async ({ scenario, candidate }) => deterministicObservation(
scenario.input,
candidate.id,
candidate.config.provider,
),
score: ({ output }) => scoreRunnerWorkflow(output, scoring),
});
const entries = results.map((result) => ({
scenarioId: result.scenarioId,
candidateId: result.candidateId,
observation: result.output,
scorecard: result.score,
}));
for (const entry of entries) {
if (entry.observation.classification === "completed" && entry.scorecard.overall.passed !== true) {
throw new Error(`deterministic Runner workflow failed: ${entry.scenarioId}/${entry.candidateId}`);
}
}
return Object.freeze(entries);
}
packages/paperclip-runner/src/eval/workflow-report.ts
+270
View File
@@ -0,0 +1,270 @@
import { CAPABILITY_CANONICAL_OPERATIONS } from "../catalog/canonical-operations.js";
import { capabilityInventoryCounts } from "../generated/capability-contract.js";
import { RUNNER_WORKFLOW_DIMENSION_KEYS, type RunnerWorkflowDimensionKey } from "./workflow-contracts.js";
import { RUNNER_WORKFLOW_CATALOG } from "./workflow-catalog.js";
import type { RunnerWorkflowMatrixEntry } from "./workflow-harness.js";
import type { StressTraceabilitySummary } from "./workflow-traceability.js";
export const RUNNER_WORKFLOW_REPORT_SCHEMA = "paperclip.runner.workflow-eval-report.v1" as const;
export interface RunnerWorkflowReportBundle {
id: string;
runnerVersion: string;
runnerBuild?: string;
promptPolicyId: string;
providerVersions: Record<string, string>;
scheduleSeed?: string;
}
export interface RunnerWorkflowEvalReport {
schema: typeof RUNNER_WORKFLOW_REPORT_SCHEMA;
generatedAt: string;
source: "deterministic" | "live" | "chaos";
bundle: RunnerWorkflowReportBundle;
results: RunnerWorkflowMatrixEntry[];
aggregate: {
executions: number;
scoreable: number;
passed: number;
candidateFailures: number;
infrastructureFailures: number;
skipped: number;
meanOverall: number | null;
dimensionMeans: Record<RunnerWorkflowDimensionKey, number | null>;
};
coverage: {
canonicalOperations: number;
capabilityCases: number;
workflows: number;
stressFindings?: number;
stressExclusions?: number;
operations: Array<{ operationId: string; capabilityCovered: true; workflowIds: string[] }>;
composedWorkflows: Array<{ workflowId: string; providerCount: number; operationIds: string[] }>;
};
}
function round(value: number): number {
return Math.round(value * 1_000) / 1_000;
}
export function buildRunnerWorkflowEvalReport(input: {
source: RunnerWorkflowEvalReport["source"];
bundle: RunnerWorkflowReportBundle;
results: readonly RunnerWorkflowMatrixEntry[];
generatedAt?: string;
traceability?: StressTraceabilitySummary;
}): RunnerWorkflowEvalReport {
const results = [...input.results];
const scoreable = results.filter((entry) => entry.scorecard.overall.score !== null);
const dimensionMeans = {} as Record<RunnerWorkflowDimensionKey, number | null>;
for (const dimension of RUNNER_WORKFLOW_DIMENSION_KEYS) {
const values = scoreable.flatMap((entry) => {
const value = entry.scorecard.dimensions[dimension].score;
return value === null ? [] : [value];
});
dimensionMeans[dimension] = values.length === 0 ? null : round(values.reduce((sum, value) => sum + value, 0) / values.length);
}
const overallValues = scoreable.flatMap((entry) => entry.scorecard.overall.score === null ? [] : [entry.scorecard.overall.score]);
return {
schema: RUNNER_WORKFLOW_REPORT_SCHEMA,
generatedAt: input.generatedAt ?? new Date().toISOString(),
source: input.source,
bundle: input.bundle,
results,
aggregate: {
executions: results.length,
scoreable: scoreable.length,
passed: scoreable.filter((entry) => entry.scorecard.overall.passed).length,
candidateFailures: results.filter((entry) => entry.observation.classification === "candidate_failure").length,
infrastructureFailures: results.filter((entry) => entry.observation.classification === "infrastructure_failure").length,
skipped: results.filter((entry) => entry.observation.classification === "skipped").length,
meanOverall: overallValues.length === 0 ? null : round(overallValues.reduce((sum, value) => sum + value, 0) / overallValues.length),
dimensionMeans,
},
coverage: {
canonicalOperations: CAPABILITY_CANONICAL_OPERATIONS.length,
capabilityCases: capabilityInventoryCounts.evalCases,
workflows: new Set(results.map((entry) => entry.scenarioId)).size,
operations: CAPABILITY_CANONICAL_OPERATIONS.map((operation) => ({
operationId: operation.operationId,
capabilityCovered: true as const,
workflowIds: RUNNER_WORKFLOW_CATALOG.filter((evalCase) =>
evalCase.assertions.requiredOperationIds?.includes(operation.operationId))
.map((evalCase) => evalCase.id),
})),
composedWorkflows: RUNNER_WORKFLOW_CATALOG.map((evalCase) => ({
workflowId: evalCase.id,
providerCount: evalCase.providers.length,
operationIds: evalCase.assertions.requiredOperationIds ?? [],
})),
...(input.traceability === undefined ? {} : {
stressFindings: input.traceability.findings,
stressExclusions: input.traceability.exclusions,
}),
},
};
}
export function renderRunnerWorkflowMarkdown(report: RunnerWorkflowEvalReport): string {
const lines = [
`# Runner workflow evals — ${report.bundle.id}`,
"",
`- Source: ${report.source}`,
`- Executions: ${report.aggregate.executions} · scoreable ${report.aggregate.scoreable} · passed ${report.aggregate.passed}`,
`- Candidate failures: ${report.aggregate.candidateFailures} · infrastructure failures ${report.aggregate.infrastructureFailures} · skipped ${report.aggregate.skipped}`,
`- Coverage: ${report.coverage.canonicalOperations} operations · ${report.coverage.capabilityCases} capability cases · ${report.coverage.workflows} workflows${report.coverage.stressFindings === undefined ? "" : ` · ${report.coverage.stressFindings} stress findings`}`,
"",
"## Dimension means",
"",
...RUNNER_WORKFLOW_DIMENSION_KEYS.map((dimension) => `- ${dimension}: ${report.aggregate.dimensionMeans[dimension] ?? "not scored"}`),
"",
"## Combined operation and workflow coverage",
"",
"| operation | capability cases | composed workflows |",
"| --- | --- | --- |",
...report.coverage.operations.map((operation) => `| ${operation.operationId} | covered | ${operation.workflowIds.join(", ") || "—"} |`),
"",
"## Executions",
"",
"| workflow | candidate | classification | gate | score | result |",
"| --- | --- | --- | --- | --- | --- |",
...report.results.map((entry) => {
const overall = entry.scorecard.overall;
const result = overall.passed === null ? "not scored" : overall.passed ? "PASS" : "FAIL";
return `| ${entry.scenarioId} | ${entry.candidateId} | ${entry.observation.classification} | ${overall.gatePassed ?? "—"} | ${overall.score === null ? "—" : round(overall.score)} | ${result} |`;
}),
"",
];
return `${lines.join("\n")}\n`;
}
function xml(value: string): string {
return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
}
export function renderRunnerWorkflowJUnit(report: RunnerWorkflowEvalReport): string {
const failures = report.results.filter((entry) => entry.scorecard.overall.passed === false).length;
const skipped = report.results.filter((entry) => entry.scorecard.overall.passed === null).length;
const cases = report.results.map((entry) => {
const name = `${entry.scenarioId}/${entry.candidateId}`;
if (entry.scorecard.overall.passed === null) {
return ` <testcase classname="runner.workflow" name="${xml(name)}"><skipped message="${xml(entry.observation.failure?.message ?? "not scoreable")}" /></testcase>`;
}
if (entry.scorecard.overall.passed === false) {
const reasons = RUNNER_WORKFLOW_DIMENSION_KEYS.flatMap((dimension) => entry.scorecard.dimensions[dimension].reasons).join("; ");
return ` <testcase classname="runner.workflow" name="${xml(name)}"><failure message="workflow eval failed">${xml(reasons)}</failure></testcase>`;
}
return ` <testcase classname="runner.workflow" name="${xml(name)}" />`;
});
return [
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>",
`<testsuite name="paperclip-runner-workflows" tests="${report.results.length}" failures="${failures}" skipped="${skipped}">`,
...cases,
"</testsuite>",
"",
].join("\n");
}
export interface RunnerWorkflowReportComparison {
compatible: boolean;
reason?: string;
currentBundleId: string;
previousBundleId: string;
passRateDelta?: number;
overallDelta?: number;
}
export function compareRunnerWorkflowReports(
current: RunnerWorkflowEvalReport,
previous: RunnerWorkflowEvalReport,
): RunnerWorkflowReportComparison {
if (current.bundle.id !== previous.bundle.id) {
return {
compatible: false,
reason: "bundle ids differ; model, provider, runner, prompt, or schedule inputs changed",
currentBundleId: current.bundle.id,
previousBundleId: previous.bundle.id,
};
}
const rate = (report: RunnerWorkflowEvalReport): number => report.aggregate.scoreable === 0 ? 0 : report.aggregate.passed / report.aggregate.scoreable;
return {
compatible: true,
currentBundleId: current.bundle.id,
previousBundleId: previous.bundle.id,
passRateDelta: round(rate(current) - rate(previous)),
overallDelta: round((current.aggregate.meanOverall ?? 0) - (previous.aggregate.meanOverall ?? 0)),
};
}
export interface RunnerWorkflowAlert {
severity: "warning" | "critical";
code: "safety_failure" | "consecutive_failures" | "success_rate_regression" | "latency_regression" | "cost_regression";
message: string;
}
function percentile95(values: number[]): number | null {
if (values.length === 0) return null;
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.ceil(sorted.length * 0.95) - 1] ?? null;
}
export function runnerWorkflowAlerts(input: {
current: RunnerWorkflowEvalReport;
history: readonly RunnerWorkflowEvalReport[];
baselineReady: boolean;
}): RunnerWorkflowAlert[] {
if (!input.baselineReady) return [];
const compatible = input.history.filter((report) => report.bundle.id === input.current.bundle.id).slice(-7);
const alerts: RunnerWorkflowAlert[] = [];
const safetyFailures = input.current.results.filter((entry) =>
entry.scorecard.dimensions.hard_invariants.passed === false
|| entry.scorecard.dimensions.lifecycle_integrity.passed === false);
if (safetyFailures.length > 0) {
alerts.push({ severity: "critical", code: "safety_failure", message: `${safetyFailures.length} execution(s) failed a hard or lifecycle invariant` });
}
for (const result of input.current.results) {
if (result.scorecard.overall.passed !== false) continue;
const priorFailures = compatible.slice(-2).every((report) => report.results.some((entry) =>
entry.scenarioId === result.scenarioId
&& entry.candidateId === result.candidateId
&& entry.scorecard.overall.passed === false));
if (compatible.length >= 2 && priorFailures) {
alerts.push({ severity: "warning", code: "consecutive_failures", message: `${result.scenarioId}/${result.candidateId} failed three consecutive reports` });
}
}
const baselineScoreable = compatible.reduce((sum, report) => sum + report.aggregate.scoreable, 0);
const baselinePassed = compatible.reduce((sum, report) => sum + report.aggregate.passed, 0);
if (baselineScoreable > 0 && input.current.aggregate.scoreable > 0) {
const baselineRate = baselinePassed / baselineScoreable;
const currentRate = input.current.aggregate.passed / input.current.aggregate.scoreable;
if (baselineRate - currentRate >= 0.20) {
alerts.push({ severity: "warning", code: "success_rate_regression", message: `success rate dropped ${Math.round((baselineRate - currentRate) * 100)} percentage points` });
}
}
const metric = (reports: readonly RunnerWorkflowEvalReport[], key: "settlementMs" | "costUsd"): number[] => reports.flatMap((report) =>
report.results.flatMap((entry) => {
const value = key === "settlementMs" ? entry.observation.metrics.settlementMs : entry.observation.metrics.costUsd;
return value === undefined ? [] : [value];
}));
const currentLatency = percentile95(metric([input.current], "settlementMs"));
const baselineLatency = percentile95(metric(compatible, "settlementMs"));
if (currentLatency !== null && baselineLatency !== null && currentLatency > baselineLatency * 1.25) {
alerts.push({ severity: "warning", code: "latency_regression", message: `p95 settlement latency rose more than 25% (${baselineLatency}ms → ${currentLatency}ms)` });
}
const currentCost = percentile95(metric([input.current], "costUsd"));
const baselineCost = percentile95(metric(compatible, "costUsd"));
if (currentCost !== null && baselineCost !== null && baselineCost > 0 && currentCost > baselineCost * 1.25) {
alerts.push({ severity: "warning", code: "cost_regression", message: `p95 cost rose more than 25% ($${baselineCost} → $${currentCost})` });
}
return alerts;
}
export function renderRunnerWorkflowGitHubSummary(
report: RunnerWorkflowEvalReport,
alerts: readonly RunnerWorkflowAlert[] = [],
): string {
const alertLines = alerts.length === 0
? ["No active live-eval regression alerts."]
: alerts.map((alert) => `- ${alert.severity.toUpperCase()} ${alert.code}: ${alert.message}`);
return `${renderRunnerWorkflowMarkdown(report)}\n## Alerts\n\n${alertLines.join("\n")}\n`;
}
packages/paperclip-runner/src/eval/workflow-scoring.ts
+162
View File
@@ -0,0 +1,162 @@
import { scoreEval } from "./eval-scoring.js";
import {
RUNNER_WORKFLOW_DIMENSION_KEYS,
RUNNER_WORKFLOW_SCORECARD_SCHEMA,
assertRunnerWorkflowObservation,
type RunnerWorkflowCheck,
type RunnerWorkflowDimensionKey,
type RunnerWorkflowDimensionScore,
type RunnerWorkflowEvalScorecard,
type RunnerWorkflowObservation,
} from "./workflow-contracts.js";
import { runnerWorkflowCase } from "./workflow-catalog.js";
export interface RunnerWorkflowScoringOptions {
bundleId: string;
weights?: Partial<Record<RunnerWorkflowDimensionKey, number>>;
thresholds?: Partial<Record<RunnerWorkflowDimensionKey, number>>;
}
const DEFAULT_WEIGHTS: Record<RunnerWorkflowDimensionKey, number> = {
hard_invariants: 0,
lifecycle_integrity: 0,
semantic_outcome: 0.20,
trajectory_restraint: 0.15,
trace_completeness: 0.10,
quality_efficiency: 0.10,
continuation_integrity: 0.20,
presentation_fidelity: 0.25,
};
function checksScore(checks: readonly RunnerWorkflowCheck[]): { score: number; reasons: string[] } {
const failed = checks.filter((check) => !check.passed);
return {
score: checks.filter((check) => check.passed).length / checks.length,
reasons: failed.map((check) => check.reason ?? `check failed: ${check.id}`),
};
}
function unscoredDimensions(): Record<RunnerWorkflowDimensionKey, RunnerWorkflowDimensionScore> {
return Object.fromEntries(RUNNER_WORKFLOW_DIMENSION_KEYS.map((dimension) => [dimension, {
dimension,
score: null,
passed: null,
gate: dimension === "hard_invariants" || dimension === "lifecycle_integrity",
weight: DEFAULT_WEIGHTS[dimension],
reasons: ["execution was not scoreable"],
}])) as Record<RunnerWorkflowDimensionKey, RunnerWorkflowDimensionScore>;
}
function traceLineageScore(observation: RunnerWorkflowObservation): { score: number; reasons: string[] } {
const expected = runnerWorkflowCase(observation.caseId).assertions.trace;
const checks: RunnerWorkflowCheck[] = [];
const captureExpectation = expected?.capture ?? "either";
if (captureExpectation === "off") {
checks.push({
id: "capture-off-isolation",
passed: observation.traceLineage.capture === "off"
&& observation.traceLineage.frameCount === 0
&& observation.traceLineage.byteCount === 0
&& observation.traceLineage.traceRef === undefined,
reason: "capture-off execution retained provider trace data",
});
} else if (captureExpectation === "on" || observation.traceLineage.capture === "on") {
checks.push(
{ id: "capture-present", passed: observation.traceLineage.capture === "on" && observation.traceLineage.frameCount > 0 && observation.traceLineage.byteCount > 0, reason: "provider trace capture is empty" },
{ id: "frame-digests", passed: observation.traceLineage.digestVerified, reason: "provider trace frame digest or byte length did not verify" },
{ id: "frame-order", passed: observation.traceLineage.ordered, reason: "provider trace frame or debug sequence is not strictly ordered" },
);
} else {
checks.push({ id: "capture-optional", passed: true });
}
for (const disposition of expected?.dispositions ?? []) {
checks.push({
id: `disposition-${disposition}`,
passed: observation.traceLineage.dispositions.includes(disposition),
reason: `provider trace did not exercise ${disposition} mapping`,
});
}
for (const lineage of expected?.lineage ?? []) {
checks.push({
id: `lineage-${lineage}`,
passed: observation.traceLineage.lineage.includes(lineage),
reason: `provider trace did not exercise ${lineage} lineage`,
});
}
return checksScore(checks);
}
/** Scores the complete Runner workflow while preserving the v1 capability dimensions. */
export function scoreRunnerWorkflow(
observation: RunnerWorkflowObservation,
options: RunnerWorkflowScoringOptions,
): RunnerWorkflowEvalScorecard {
assertRunnerWorkflowObservation(observation);
if (observation.classification === "infrastructure_failure" || observation.classification === "skipped") {
return {
schema: RUNNER_WORKFLOW_SCORECARD_SCHEMA,
bundleId: options.bundleId,
caseId: observation.caseId,
candidateId: observation.candidateId,
classification: observation.classification,
dimensions: unscoredDimensions(),
overall: { score: null, gatePassed: null, passed: null },
};
}
const v1 = scoreEval(observation.base, { bundleId: options.bundleId });
const lineageTrace = traceLineageScore(observation);
const workflowScores = {
lifecycle_integrity: checksScore(observation.lifecycle.checks),
continuation_integrity: checksScore(observation.continuation.checks),
presentation_fidelity: checksScore(observation.presentation.checks),
};
const weights = { ...DEFAULT_WEIGHTS, ...options.weights };
const dimensions = {} as Record<RunnerWorkflowDimensionKey, RunnerWorkflowDimensionScore>;
for (const dimension of RUNNER_WORKFLOW_DIMENSION_KEYS) {
const legacy = dimension in v1.dimensions
? v1.dimensions[dimension as keyof typeof v1.dimensions]
: undefined;
const workflow = dimension in workflowScores
? workflowScores[dimension as keyof typeof workflowScores]
: undefined;
const score = dimension === "trace_completeness" && legacy !== undefined
? Math.min(legacy.score, lineageTrace.score)
: legacy?.score ?? workflow?.score ?? 0;
const gate = dimension === "hard_invariants" || dimension === "lifecycle_integrity";
const threshold = options.thresholds?.[dimension] ?? 1;
dimensions[dimension] = {
dimension,
score,
passed: score >= threshold,
gate,
weight: gate ? 0 : weights[dimension],
reasons: dimension === "trace_completeness" && legacy !== undefined
? [...legacy.reasons, ...lineageTrace.reasons]
: legacy?.reasons ?? workflow?.reasons ?? [],
};
}
const gatePassed = dimensions.hard_invariants.passed === true
&& dimensions.lifecycle_integrity.passed === true;
const weighted = RUNNER_WORKFLOW_DIMENSION_KEYS.filter((dimension) => !dimensions[dimension].gate);
const totalWeight = weighted.reduce((sum, dimension) => sum + dimensions[dimension].weight, 0);
const weightedScore = totalWeight === 0 ? 0 : weighted.reduce(
(sum, dimension) => sum + (dimensions[dimension].score ?? 0) * dimensions[dimension].weight,
0,
) / totalWeight;
const passed = gatePassed && RUNNER_WORKFLOW_DIMENSION_KEYS.every(
(dimension) => dimensions[dimension].passed === true,
);
return {
schema: RUNNER_WORKFLOW_SCORECARD_SCHEMA,
bundleId: options.bundleId,
caseId: observation.caseId,
candidateId: observation.candidateId,
classification: observation.classification,
dimensions,
overall: { score: gatePassed ? weightedScore : 0, gatePassed, passed },
};
}
packages/paperclip-runner/src/eval/workflow-traceability.ts
+74
View File
@@ -0,0 +1,74 @@
import { RUNNER_WORKFLOW_IDS, type RunnerWorkflowId } from "./workflow-contracts.js";
export const STRESS_TRACEABILITY_SCHEMA = "paperclip.runner.stress-eval-traceability.v1" as const;
export interface StressTraceabilityFinding {
id: string;
classification: "workflow_eval" | "regression_test" | "explicit_exclusion";
workflowIds: RunnerWorkflowId[];
regressionTests: string[];
reason?: string;
}
export interface StressTraceabilityManifest {
schema: typeof STRESS_TRACEABILITY_SCHEMA;
campaign: string;
expectedFindings: 44;
workflows: RunnerWorkflowId[];
findings: StressTraceabilityFinding[];
}
export interface StressTraceabilitySummary {
findings: number;
workflowEvalFindings: number;
regressionTestFindings: number;
exclusions: number;
coveredWorkflows: number;
}
export function validateStressTraceabilityManifest(manifest: StressTraceabilityManifest): StressTraceabilitySummary {
if (manifest.schema !== STRESS_TRACEABILITY_SCHEMA || manifest.expectedFindings !== 44) {
throw new Error("unsupported stress traceability manifest");
}
const expectedIds = Array.from({ length: 44 }, (_, index) => `STRESS-${String(index + 1).padStart(3, "0")}`);
const actualIds = manifest.findings.map((finding) => finding.id);
if (actualIds.length !== expectedIds.length || new Set(actualIds).size !== expectedIds.length) {
throw new Error("stress traceability must contain 44 unique findings");
}
for (const id of expectedIds) {
if (!actualIds.includes(id)) throw new Error(`stress traceability is missing ${id}`);
}
if (manifest.workflows.length !== RUNNER_WORKFLOW_IDS.length
|| !RUNNER_WORKFLOW_IDS.every((id) => manifest.workflows.includes(id))) {
throw new Error("stress traceability workflow inventory does not match the Runner workflow catalog");
}
for (const finding of manifest.findings) {
for (const workflowId of finding.workflowIds) {
if (!(RUNNER_WORKFLOW_IDS as readonly string[]).includes(workflowId)) {
throw new Error(`${finding.id} references unknown workflow ${workflowId}`);
}
}
if (finding.classification === "workflow_eval" && finding.workflowIds.length === 0) {
throw new Error(`${finding.id} is a workflow eval without a workflow`);
}
if (finding.classification !== "explicit_exclusion" && finding.regressionTests.length === 0) {
throw new Error(`${finding.id} is missing a regression test anchor`);
}
if (finding.regressionTests.some((path) => !/\.test\.[cm]?[jt]sx?$/.test(path))) {
throw new Error(`${finding.id} references a non-test regression anchor`);
}
if (finding.classification === "regression_test" && finding.regressionTests.length === 0) {
throw new Error(`${finding.id} is a regression finding without a test`);
}
if (finding.classification === "explicit_exclusion" && !finding.reason) {
throw new Error(`${finding.id} exclusion is missing a reason`);
}
}
return {
findings: manifest.findings.length,
workflowEvalFindings: manifest.findings.filter((finding) => finding.classification === "workflow_eval").length,
regressionTestFindings: manifest.findings.filter((finding) => finding.classification === "regression_test").length,
exclusions: manifest.findings.filter((finding) => finding.classification === "explicit_exclusion").length,
coveredWorkflows: new Set(manifest.findings.flatMap((finding) => finding.workflowIds)).size,
};
}
packages/paperclip-runner/src/index.ts
+3
View File
@@ -1,11 +1,13 @@
export * from "./contracts/control-plane-port.js";
export * from "./contracts/harness-driver.js";
export * from "./contracts/question-set.js";
export * from "./contracts/native-session-backend.js";
export * from "./contracts/native-execution.js";
export * from "./contracts/runtime-context.js";
export * from "./contracts/local-runner.js";
export * from "./contracts/durable-recovery.js";
export * from "./contracts/codex.js";
export * from "./contracts/completion-result.js";
export * from "./contracts/provider-trace.js";
export * from "./contracts/types.js";
export * from "./backends/harness-driver-backend.js";
@@ -37,6 +39,7 @@ export {
export * from "./live/workspace-file-reference.js";
export * from "./protocol/replay-contract.js";
export * from "./protocol/replay-loader.js";
export * from "./protocol/result-normalization.js";
export * from "./protocol/semantic-tool-receipts.js";
export * from "./reducer/session-reducer.js";
export * from "./generated/capability-contract.js";
packages/paperclip-runner/src/live/live-session.ts
+4 -1
View File
@@ -140,6 +140,9 @@ export interface CapabilityLiveSessionConfigSnapshot {
memoryArn: string;
memoryId: string;
invocationRoleArn: string;
contextBucket: string;
contextPrefix: string;
contextKmsKeyArn: string;
qualificationRevision: string;
eventExpiryDays: 90;
maxEstimatedSessionCostUsd: number;
@@ -573,7 +576,7 @@ export function assertCapabilityLiveSessionSnapshot(
}
if (provider === "aws_agentcore") {
const profile = record(config.agentCoreProfile);
for (const field of ["profileId", "region", "accountId", "harnessArn", "harnessVersion", "endpointArn", "endpointQualifier", "agentRuntimeArn", "memoryArn", "memoryId", "invocationRoleArn", "qualificationRevision"] as const) {
for (const field of ["profileId", "region", "accountId", "harnessArn", "harnessVersion", "endpointArn", "endpointQualifier", "agentRuntimeArn", "memoryArn", "memoryId", "invocationRoleArn", "contextBucket", "contextPrefix", "contextKmsKeyArn", "qualificationRevision"] as const) {
if (text(profile[field]).trim().length === 0) {
throw new Error(`capability_live_checkpoint_corrupt: missing agentCoreProfile.${field}`);
}
packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts
+58
View File
@@ -10,11 +10,45 @@ import {
createSanitizedClaudeManagedEnvironment,
createSanitizedAwsAgentCoreEnvironment,
defaultCapabilityRunnerdBinary,
rehydrateRunnerdPlanNotification,
rehydrateRunnerdUsageNotification,
resolveSourceCodexHome,
trustedRuntimeReadOnlyRoots,
unwrapRunnerdProviderNotification,
unwrapRunnerdProviderNotifications,
withCodexCollaborationRuntimeInstructions,
} from "./runnerd-codex-transport.js";
it("adds Codex-style turn updates only when collaboration instructions are enabled", () => {
const base = "Base Paperclip instructions.";
const enabled = withCodexCollaborationRuntimeInstructions(base, true);
expect(enabled).toContain(base);
expect(enabled).toContain("Before the first tool call in a turn");
expect(enabled).toContain("Do not call it merely to create a completion comment");
expect(withCodexCollaborationRuntimeInstructions(base, false)).toBe(base);
});
it("resolves the ordinary ~/.codex credential home when CODEX_HOME is unset", () => {
expect(resolveSourceCodexHome({ HOME: "/Users/tester" })).toBe(
"/Users/tester/.codex",
);
expect(
resolveSourceCodexHome({
HOME: "/Users/tester",
CODEX_HOME: "/managed/codex",
}),
).toBe("/managed/codex");
});
it("allows trusted package-manager runtime roots without exposing HOME paths", () => {
expect(
trustedRuntimeReadOnlyRoots({
HOME: "/Users/tester",
PATH: "/Users/tester/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin",
}),
).toEqual(["/opt/homebrew", "/usr/local"]);
});
it("passes only runtime basics and the Anthropic key to a Claude Managed runner", () => {
const sanitized = createSanitizedClaudeManagedEnvironment({
PATH: "/bin",
@@ -77,6 +111,30 @@ it("rehydrates normalized usage with the provider thread binding", () => {
});
});
it("rehydrates normalized plans into the Codex notification contract", () => {
expect(
rehydrateRunnerdPlanNotification(
{
explanation: "Ship in small steps",
steps: [
{ stepId: "step-1", body: "Inspect", status: "completed" },
{ stepId: "step-2", body: "Implement", status: "in_progress" },
],
},
"thread-1",
"turn-1",
),
).toMatchObject({
threadId: "thread-1",
turnId: "turn-1",
explanation: "Ship in small steps",
plan: [
{ step: "Inspect", status: "completed" },
{ step: "Implement", status: "in_progress" },
],
});
});
const fakeCodex = resolve(
import.meta.dirname,
"../../runner/target/debug/fake-codex-app-server",
packages/paperclip-runner/src/live/runnerd-codex-transport.ts
+162 -15
View File
@@ -6,6 +6,7 @@ import {
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { resolve } from "node:path";
@@ -49,6 +50,22 @@ const executableSuffix = process.platform === "win32" ? ".exe" : "";
const MAX_NOTIFICATION_COUNT = 2_048;
const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024;
const CODEX_COLLABORATION_RUNTIME_INSTRUCTIONS = `## Codex-style collaboration
- Before the first tool call in a turn, send a brief commentary update describing the immediate work you are starting.
- During tool-driven work, send concise commentary updates at meaningful transitions so the user can follow progress without opening raw logs.
- Reserve \`report_progress\` for meaningful durable milestones on longer work. Do not call it merely to create a completion comment on a short run; Paperclip materializes the final assistant response as the durable completion comment.
- After semantic finalization, send one self-contained final assistant response with the outcome and verification.`;
export function withCodexCollaborationRuntimeInstructions(
instructions: string,
enabled = true,
): string {
if (!enabled) return instructions;
const base = instructions.trimEnd();
return `${base}\n\n${CODEX_COLLABORATION_RUNTIME_INSTRUCTIONS}`;
}
function recoveredControlPlaneIdentity(
directory: string,
desired: DurableRecoveryIdentity,
@@ -122,6 +139,9 @@ export interface CapabilityRunnerdCodexTransportOptions {
memoryArn: string;
memoryId: string;
invocationRoleArn: string;
contextBucket: string;
contextPrefix: string;
contextKmsKeyArn: string;
qualificationRevision: string;
eventExpiryDays: 90;
maxEstimatedSessionCostUsd: number;
@@ -450,6 +470,39 @@ export function rehydrateRunnerdUsageNotification(
};
}
export function rehydrateRunnerdPlanNotification(
rawParams: Record<string, unknown>,
openedThreadId: string,
activeTurnId: string,
): Record<string, unknown> {
const steps = Array.isArray(rawParams.steps) ? rawParams.steps : [];
return {
...rawParams,
threadId:
typeof rawParams.threadId === "string"
? rawParams.threadId
: openedThreadId,
turnId:
typeof rawParams.turnId === "string"
? rawParams.turnId
: activeTurnId,
// Rust has already normalized the provider's plan entries into PRP's
// { stepId, body, status } shape. Rebuild Codex's notification contract
// so the strict TypeScript driver can perform (and expose) its second
// interpretation stage instead of silently dropping the plan.
plan: Array.isArray(rawParams.plan)
? rawParams.plan
: steps.map((value) => {
const step = record(value);
return {
step: typeof step.body === "string" ? step.body : "",
status:
typeof step.status === "string" ? step.status : "pending",
};
}),
};
}
function commandDigest(value: unknown): string {
return `sha256:${createHash("sha256").update(durableRecoveryInternals.canonicalJson(value)).digest("hex")}`;
}
@@ -506,12 +559,61 @@ export function createSanitizedAwsAgentCoreEnvironment(
"PAPERCLIP_NATIVE_MCP_NAME",
"PAPERCLIP_NATIVE_MCP_URL",
"PAPERCLIP_NATIVE_MCP_TOKEN",
"PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH",
]) {
if (typeof source[key] === "string") result[key] = source[key];
}
return result;
}
/**
* Raw provider tracing is consumed by runnerd itself. The provider child still
* receives the narrower allowlist enforced by Rust's `SupervisedProcess`, so
* these controller-selected sidecar paths never enter the harness process.
*/
function withRunnerdProviderTrace(
environment: NodeJS.ProcessEnv,
source: NodeJS.ProcessEnv | undefined,
): NodeJS.ProcessEnv {
const result = { ...environment };
for (const key of [
"PAPERCLIP_PROVIDER_TRACE_PATH",
"PAPERCLIP_PROVIDER_TRACE_MAX_BYTES",
] as const) {
const value = source?.[key];
if (value !== undefined) result[key] = value;
}
return result;
}
export function resolveSourceCodexHome(
environment: NodeJS.ProcessEnv | undefined,
): string | null {
const explicit = environment?.CODEX_HOME?.trim();
if (explicit) return explicit;
const home = environment?.HOME?.trim();
return home ? resolve(home, ".codex") : null;
}
export function trustedRuntimeReadOnlyRoots(
environment: NodeJS.ProcessEnv | undefined,
): string[] {
const path = environment?.PATH ?? "";
const roots = new Set<string>();
for (const entry of path.split(process.platform === "win32" ? ";" : ":")) {
if (entry === "/opt/homebrew" || entry.startsWith("/opt/homebrew/")) {
roots.add("/opt/homebrew");
} else if (entry === "/usr/local" || entry.startsWith("/usr/local/")) {
roots.add("/usr/local");
} else if (entry === "/opt/local" || entry.startsWith("/opt/local/")) {
roots.add("/opt/local");
} else if (entry === "/nix/store" || entry.startsWith("/nix/store/")) {
roots.add("/nix/store");
}
}
return [...roots];
}
function unwrapToolResponse(
response: Record<string, unknown>,
): Record<string, unknown> {
@@ -900,6 +1002,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
tool: call.operationId,
arguments: call.input,
},
...(call.sourceEventId && call.sourceEventType
? {
paperclipTrace: {
sourceEventId: call.sourceEventId,
sourceEventType: call.sourceEventType,
},
}
: {}),
}),
),
connectionLeaseTtlMs: 60 * 60 * 1_000,
@@ -908,12 +1018,16 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
mkdirSync(resolve(this.#root, "runner"), { recursive: true, mode: 0o700 });
const provider = this.options.provider ?? "codex";
const runtimeContext = this.options.runtimeContext ?? null;
const runtimeContextPath = resolve(this.#root, "runtime-context.json");
if (runtimeContext !== null) {
writeFileSync(runtimeContextPath, `${JSON.stringify(runtimeContext)}\n`, { mode: 0o600 });
}
const codexHome = resolve(this.#root, "codex-home");
if (provider === "codex") {
await prepareIsolatedCodexHome({
context: runtimeContext,
codexHome,
sourceCodexHome: this.options.environment?.CODEX_HOME,
sourceCodexHome: resolveSourceCodexHome(this.options.environment),
nativeMcp: nativeMcpLaunchBinding(this.options.environment),
});
}
@@ -941,6 +1055,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
const acpxAgent =
provider === "acpx" ? (this.options.acpxAgent ?? "pi") : null;
const requestedModel = typeof params.model === "string" ? params.model : "";
const includeCodexCollaborationInstructions =
provider === "codex" &&
record(params.config).include_collaboration_mode_instructions !== false;
const baseInstructions = String(
params.baseInstructions ?? "You are a Paperclip agent.",
);
const acpxProfile =
provider === "acpx"
? resolveQualifiedAcpxProfile(acpxAgent!, requestedModel)
@@ -976,6 +1096,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
memoryArn: agentCore!.memoryArn,
memoryId: agentCore!.memoryId,
invocationRoleArn: agentCore!.invocationRoleArn,
contextBucket: agentCore!.contextBucket,
contextPrefix: agentCore!.contextPrefix,
contextKmsKeyArn: agentCore!.contextKmsKeyArn,
qualificationRevision: agentCore!.qualificationRevision,
eventExpiryDays: agentCore!.eventExpiryDays,
maxEstimatedSessionCostUsd:
@@ -1005,9 +1128,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
normalizedSessionId: identity.normalizedSessionId,
runId: identity.runId,
cwd: String(params.cwd ?? tmpdir()),
instructions: String(
params.baseInstructions ?? "You are a Paperclip agent.",
),
instructions: baseInstructions,
permissionPolicy: "interactive",
runtimeContext,
}
@@ -1023,22 +1144,27 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
: (this.options.codexArgs ??
createIsolatedCodexAppServerArgs(
this.options.environment,
runtimeContext ? [codexHome, runtimeContext.instructions.bundle.rootPath, ...runtimeContext.skills.map((skill) => skill.bundle.rootPath)] : [],
[
...trustedRuntimeReadOnlyRoots(this.options.environment),
...(runtimeContext ? [codexHome, runtimeContext.instructions.bundle.rootPath, ...runtimeContext.skills.map((skill) => skill.bundle.rootPath)] : []),
],
)),
cwd: String(params.cwd ?? tmpdir()),
model: typeof params.model === "string" ? params.model : null,
instructions: String(
params.baseInstructions ?? "You are a Paperclip agent.",
),
instructions:
provider === "codex"
? withCodexCollaborationRuntimeInstructions(
baseInstructions,
includeCodexCollaborationInstructions,
)
: baseInstructions,
collaborationMode:
params.permissions ===
"paperclip-runner-workspace-read-only"
? "plan"
: "default",
includeCollaborationModeInstructions:
provider === "codex" &&
record(params.config)
.include_collaboration_mode_instructions !== false,
includeCodexCollaborationInstructions,
includeSkillInstructions: provider === "codex" && runtimeContext !== null,
runtimeContext,
},
@@ -1060,7 +1186,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
lifecyclePolicy: this.options.lifecyclePolicy,
runnerBinaryPath:
this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(),
environment:
environment: withRunnerdProviderTrace(
provider === "opencode"
? {
...process.env,
@@ -1073,6 +1199,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
PAPERCLIP_RUNNER_INSTANCE_ID: identity.runnerInstanceId,
PAPERCLIP_RUN_ID: identity.runId,
PAPERCLIP_NORMALIZED_SESSION_ID: identity.normalizedSessionId,
...(runtimeContext ? { PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH: runtimeContextPath } : {}),
}
: provider === "acpx"
? {
@@ -1083,6 +1210,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
PAPERCLIP_RUNNER_INSTANCE_ID: identity.runnerInstanceId,
PAPERCLIP_RUN_ID: identity.runId,
PAPERCLIP_NORMALIZED_SESSION_ID: identity.normalizedSessionId,
...(runtimeContext ? { PAPERCLIP_NATIVE_RUNTIME_CONTEXT_PATH: runtimeContextPath } : {}),
}
: provider === "claude_managed"
? createSanitizedClaudeManagedEnvironment(
@@ -1093,6 +1221,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.options.environment,
)
: createSanitizedCodexEnvironment({ ...this.options.environment, HOME: codexHome, CODEX_HOME: codexHome }),
this.options.environment,
),
});
this.#handle = handle;
this.#watchRunner(handle);
@@ -1148,7 +1278,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
await prepareIsolatedCodexHome({
context: this.options.runtimeContext ?? null,
codexHome,
sourceCodexHome: this.options.environment?.CODEX_HOME,
sourceCodexHome: resolveSourceCodexHome(this.options.environment),
nativeMcp: nativeMcpLaunchBinding(this.options.environment),
});
}
@@ -1167,6 +1297,14 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
tool: call.operationId,
arguments: call.input,
},
...(call.sourceEventId && call.sourceEventType
? {
paperclipTrace: {
sourceEventId: call.sourceEventId,
sourceEventType: call.sourceEventType,
},
}
: {}),
}),
),
connectionLeaseTtlMs: 60 * 60 * 1_000,
@@ -1189,7 +1327,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
lifecyclePolicy: this.options.lifecyclePolicy,
runnerBinaryPath:
this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(),
environment:
environment: withRunnerdProviderTrace(
this.options.provider === "claude_managed"
? createSanitizedClaudeManagedEnvironment(this.options.environment)
: this.options.provider === "aws_agentcore"
@@ -1201,7 +1339,9 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
)
: provider === "codex"
? createSanitizedCodexEnvironment({ ...this.options.environment, HOME: codexHome, CODEX_HOME: codexHome })
: this.options.environment,
: (this.options.environment ?? {}),
this.options.environment,
),
});
this.#handle = handle;
this.#watchRunner(handle);
@@ -1465,6 +1605,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
"turn.interrupted": "turn/completed",
"turn.cancelled": "turn/completed",
"usage.reported": "thread/tokenUsage/updated",
"plan.updated": "turn/plan/updated",
"session.updated":
sessionUpdatePayload.status === "budget_reached"
? "provider/budgetReached"
@@ -1488,6 +1629,12 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
this.#threadId,
this.#turnId,
)
: method === "turn/plan/updated"
? rehydrateRunnerdPlanNotification(
rawParams,
this.#threadId,
this.#turnId,
)
: method === "turn/started" || method === "turn/completed"
? {
...rawParams,
packages/paperclip-runner/src/native-session-runtime.test.ts
+449
View File
@@ -12,6 +12,13 @@ import type {
PrpStructuredRunResult,
PrpTerminalState,
} from "./protocol/replay-contract.js";
import {
NATIVE_RUNTIME_ASSET_SCHEMA,
PAPERCLIP_EXECUTION_PROMPT,
PAPERCLIP_EXECUTION_PROMPT_REVISION,
canonicalNativeRuntimeContextDigest,
nativeRuntimePromptDigest,
} from "./contracts/runtime-context.js";
import { executeNativeSession } from "./native-session-runtime.js";
const identity = {
@@ -108,6 +115,73 @@ function highestContiguous(events: PrpEvent[]): number {
}
describe("executeNativeSession recovery", () => {
it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => {
const digest = "0".repeat(64);
const context = {
prompt: {
revision: PAPERCLIP_EXECUTION_PROMPT_REVISION,
text: PAPERCLIP_EXECUTION_PROMPT,
digest: nativeRuntimePromptDigest(),
},
instructions: {
entryPath: "AGENTS.md",
bundle: {
schema: NATIVE_RUNTIME_ASSET_SCHEMA,
digest,
manifestDigest: digest,
rootPath: "/paperclip/context/instructions",
fileCount: 1,
totalBytes: 1,
},
},
skills: [],
mcp: { assignmentSetId: "none", digest, bindingId: null },
} as const;
const openSession = vi.fn();
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "future-provider",
name: "future-provider",
version: "1",
capabilities: {
resume: false,
typedEvents: true,
steering: false,
interruption: false,
structuredResult: true,
},
};
},
openSession,
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
async appendEvent() { throw new Error("unexpected event"); },
async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; },
async completeRun() {},
};
await expect(executeNativeSession({
input: {
...input,
schema: "paperclip.native-execution-input.v3",
executionMode: "default",
planningContext: null,
runtimeContext: {
...context,
aggregateDigest: canonicalNativeRuntimeContextDigest(context),
},
},
backend,
controlPlane: port,
runnerInstanceId: "runner-recovery",
controlPlaneInstanceId: "control-recovery",
})).rejects.toThrow("does not natively realize instructions, skills, mcp");
expect(openSession).not.toHaveBeenCalled();
});
it("contains a consumer rejection when starting the turn fails first", async () => {
let closeStream = () => {};
const streamClosed = new Promise<void>((resolve) => { closeStream = resolve; });
@@ -254,6 +328,105 @@ describe("executeNativeSession recovery", () => {
expect(startTurn).not.toHaveBeenCalled();
});
it("starts a continuation after the driver clears a stale active terminal turn", async () => {
const checkpoint: PersistedNativeSession = {
backendKind: "mock",
sessionId: "driver-recovery",
identity,
providerSessionId: "provider-recovery",
cursor: "1",
activeTurnId: "turn-already-terminal",
terminalTurns: [{ turnId: "turn-already-terminal", fingerprint: "terminal-fingerprint" }],
pendingRuntimeRequests: [],
lineage: [],
};
const recoveredSnapshot: PersistedNativeSession = {
...checkpoint,
activeTurnId: null,
};
const terminalEvent: PrpEvent = {
schema: "paperclip.prp.event.v1",
sourceEventId: "provider-recovery:2",
sourceSeq: 2,
sourceInstanceId: "provider-recovery",
sourceKind: "provider",
runId: identity.runId,
normalizedSessionId: identity.sessionId,
turnId: "turn-continuation",
eventType: "turn.completed",
schemaVersion: 1,
priority: 0,
emittedAt: "2026-08-09T00:00:01.000Z",
payload: {},
};
const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" }));
const session: NativeSession = {
identity: () => identity,
async capabilities() {
return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true };
},
async *events() { yield terminalEvent; },
startTurn,
async result() { return { result, terminal, turnId: "turn-continuation" }; },
async snapshot() { return structuredClone(recoveredSnapshot); },
async close() {},
};
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "recovery-backend",
version: "1",
capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true },
};
},
async openSession() { throw new Error("must recover the provider session"); },
async recoverSession() { return { recovered: true, session }; },
};
const bySource = new Map<string, PrpEvent[]>();
const port: ControlPlanePort = {
async openRun() {},
async loadSessionCheckpoint() { return structuredClone(checkpoint); },
async checkpointSession() {},
async appendEvent(event) {
const list = bySource.get(event.sourceInstanceId) ?? [];
list.push(structuredClone(event));
bySource.set(event.sourceInstanceId, list);
return {
cursor: list.length,
highestContiguousSourceSeq: highestContiguous(list),
disposition: "committed",
};
},
async replayEvents(replay) {
const list = bySource.get(replay.sourceInstanceId) ?? [];
return {
events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)),
highestContiguousSourceSeq: highestContiguous(list),
};
},
async completeRun() {},
};
await expect(executeNativeSession({
input,
backend,
controlPlane: port,
runnerInstanceId: "runner-recovery",
controlPlaneInstanceId: "control-recovery",
})).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" });
expect(startTurn).toHaveBeenCalledOnce();
const recoveryEnvelope = JSON.parse(
startTurn.mock.calls[0]![0].message.text,
) as { task: { prompt: string } };
expect(recoveryEnvelope.task.prompt).toContain(
"semantic-result recovery for a prior completed provider turn",
);
expect(recoveryEnvelope.task.prompt).toContain(
"Do not repeat implementation, tests, research, or the final answer",
);
});
it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => {
const checkpoint: PersistedNativeSession = {
backendKind: "mock",
@@ -335,4 +508,280 @@ describe("executeNativeSession recovery", () => {
expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 });
expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal });
});
it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => {
const terminalEvent: PrpEvent = {
schema: "paperclip.prp.event.v1",
sourceEventId: "provider-recovery:1",
sourceSeq: 1,
sourceInstanceId: "provider-recovery",
sourceKind: "provider",
runId: identity.runId,
normalizedSessionId: identity.sessionId,
turnId: "turn-waiting",
eventType: "turn.completed",
schemaVersion: 1,
priority: 0,
emittedAt: "2026-08-09T00:00:00.000Z",
payload: {},
};
const yielded: PrpStructuredRunResult = {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "yielded",
summary: "Waiting for the requested response.",
completionClaim: {
contractRevision: "1",
objectiveSatisfied: false,
criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }],
remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }],
},
evidence: [{ ref: "interaction:pending" }],
verification: [],
attentionRequests: [],
artifacts: [],
continuation: {
kind: "response_wake",
summary: "Resume from the answer.",
idempotencyKey: "interaction-response:pending",
},
};
const events: PrpEvent[] = [];
const completeRun = vi.fn(async () => undefined);
const resolveMissingResult = vi.fn(async () => yielded);
const session: NativeSession = {
identity: () => identity,
async capabilities() {
return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true };
},
async *events() { yield terminalEvent; },
async startTurn() { return { turnId: "turn-waiting" }; },
async result() { return null; },
async snapshot() {
return {
backendKind: "mock",
sessionId: identity.sessionId,
identity,
providerSessionId: "provider-waiting",
cursor: "1",
activeTurnId: "turn-waiting",
pendingRuntimeRequests: [],
lineage: [],
};
},
async close() {},
};
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "governed-wait-backend",
version: "1",
capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true },
};
},
async openSession() { return session; },
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
async appendEvent(event) {
events.push(structuredClone(event as PrpEvent));
return {
cursor: events.length,
highestContiguousSourceSeq: highestContiguous(events),
disposition: "committed",
};
},
async replayEvents(replay) {
const replayed = events.filter((event) =>
event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq
);
return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) };
},
completeRun,
};
const completed = await executeNativeSession({
input,
backend,
controlPlane: port,
runnerInstanceId: "runner-recovery",
controlPlaneInstanceId: "control-recovery",
resolveMissingResult,
});
expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent });
expect(completed).toMatchObject({
result: yielded,
terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" },
turnId: "turn-waiting",
});
expect(completeRun).toHaveBeenCalledWith(expect.objectContaining({ result: yielded }));
expect(events.map((event) => event.eventType)).toEqual([
"turn.completed",
"run.result.accepted",
"run.terminal",
]);
});
it("parks a provider turn immediately after a durable governed wait appears", async () => {
const yielded: PrpStructuredRunResult = {
schema: "paperclip.run_result.v1",
reportedWorkDisposition: "yielded",
summary: "Waiting for the requested response.",
completionClaim: {
contractRevision: "1",
objectiveSatisfied: false,
criteria: [
{
criterionId: "objective",
status: "unknown",
evidenceRefs: ["interaction:pending"],
},
],
remainingWork: [
{ description: "Resume after the response.", blocksCompletion: true },
],
},
evidence: [{ ref: "interaction:pending" }],
verification: [],
attentionRequests: [],
artifacts: [],
continuation: {
kind: "response_wake",
summary: "Resume from the answer.",
idempotencyKey: "interaction-response:pending",
},
};
const itemCompleted: PrpEvent = {
...controlEvent(1, "item.completed", {
kind: "dynamicToolCall",
item: { id: "ask-1", name: "ask_user_questions" },
}),
sourceEventId: "provider-recovery:1",
sourceInstanceId: "provider-recovery",
sourceKind: "provider",
turnId: "turn-waiting",
};
const turnInterrupted: PrpEvent = {
...controlEvent(2, "turn.interrupted", { reason: "governed_wait" }),
sourceEventId: "provider-recovery:2",
sourceInstanceId: "provider-recovery",
sourceKind: "provider",
turnId: "turn-waiting",
};
let releaseCancelled!: () => void;
const cancelled = new Promise<void>((resolve) => {
releaseCancelled = resolve;
});
const cancel = vi.fn(async () => releaseCancelled());
const events: PrpEvent[] = [];
const completeRun = vi.fn(async () => undefined);
const session: NativeSession = {
identity: () => identity,
async capabilities() {
return {
resume: false,
typedEvents: true,
steering: false,
interruption: true,
structuredResult: true,
};
},
async *events() {
yield itemCompleted;
await cancelled;
yield turnInterrupted;
},
async startTurn() {
return { turnId: "turn-waiting" };
},
cancel,
async result() {
return null;
},
async snapshot() {
return {
backendKind: "mock",
sessionId: identity.sessionId,
identity,
providerSessionId: "provider-waiting",
cursor: "2",
activeTurnId: "turn-waiting",
pendingRuntimeRequests: [],
lineage: [],
};
},
async close() {},
};
const backend: NativeSessionBackend = {
async descriptor() {
return {
kind: "mock",
name: "governed-wait-backend",
version: "1",
capabilities: {
resume: false,
typedEvents: true,
steering: false,
interruption: true,
structuredResult: true,
},
};
},
async openSession() {
return session;
},
};
const port: ControlPlanePort = {
async openRun() {},
async checkpointSession() {},
async appendEvent(event) {
events.push(structuredClone(event as PrpEvent));
return {
cursor: events.length,
highestContiguousSourceSeq: highestContiguous(events),
disposition: "committed",
};
},
async replayEvents(replay) {
const replayed = events.filter(
(event) =>
event.sourceInstanceId === replay.sourceInstanceId &&
event.sourceSeq > replay.afterSourceSeq,
);
return {
events: structuredClone(replayed),
highestContiguousSourceSeq: highestContiguous(replayed),
};
},
completeRun,
};
const completed = await executeNativeSession({
input,
backend,
controlPlane: port,
runnerInstanceId: "runner-recovery",
controlPlaneInstanceId: "control-recovery",
resolveGovernedWait: async ({ event }) =>
event.eventType === "item.completed" ? yielded : null,
});
expect(cancel).toHaveBeenCalledOnce();
expect(completed).toMatchObject({
result: yielded,
terminal: {
turnTerminalState: "completed",
runTerminalState: "succeeded",
reportedWorkDisposition: "yielded",
},
});
expect(events.map((event) => event.eventType)).toEqual([
"item.completed",
"turn.interrupted",
"run.result.accepted",
"run.terminal",
]);
});
});
packages/paperclip-runner/src/native-session-runtime.ts
+108 -9
View File
@@ -5,7 +5,7 @@ import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contr
import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js";
import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js";
import type { PersistedNativeSession } from "./contracts/native-session-backend.js";
import type { PrpEvent, PrpTerminalState } from "./protocol/replay-contract.js";
import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js";
export interface ExecuteNativeSessionOptions {
input: NativeExecutionInput;
@@ -19,6 +19,26 @@ export interface ExecuteNativeSessionOptions {
persistedSession?: PersistedNativeSession | null;
keepSessionOpen?: boolean;
onCheckpoint?: (snapshot: PersistedNativeSession) => Promise<void> | void;
/**
* Control-plane policy seam for a provider turn that completed after
* durably creating a governed wait, but did not emit a semantic finish
* result. The runner package cannot inspect server-owned interactions, so
* it asks the embedding control plane whether that missing result is an
* intentional yield before treating it as provider failure.
*/
resolveMissingResult?: (input: {
turnId: string | null;
terminalEvent: PrpEvent;
}) => Promise<PrpStructuredRunResult | null>;
/**
* Detect a durable server-owned wait as soon as its provider tool event is
* committed. Models are not trusted to stop or avoid polling after creating
* a question/review interaction; the control plane may park the turn here.
*/
resolveGovernedWait?: (input: {
turnId: string | null;
event: PrpEvent;
}) => Promise<PrpStructuredRunResult | null>;
}
function isTurnTerminal(event: PrpEvent): boolean {
@@ -36,18 +56,37 @@ function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["repor
return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition };
}
async function consumeTurn(session: NativeSession, controlPlane: ControlPlanePort, timeoutMs: number) {
async function consumeTurn(
session: NativeSession,
controlPlane: ControlPlanePort,
timeoutMs: number,
resolveGovernedWait?: ExecuteNativeSessionOptions["resolveGovernedWait"],
) {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
(async () => {
let eventCount = 0;
let highestContiguousSourceSeq = 0;
let governedResult: PrpStructuredRunResult | null = null;
for await (const event of session.events()) {
const receipt = await controlPlane.appendEvent(event);
eventCount += receipt.disposition === "committed" ? 1 : 0;
highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq);
if (isTurnTerminal(event)) return { event, eventCount, highestContiguousSourceSeq };
if (governedResult === null && resolveGovernedWait) {
governedResult = await resolveGovernedWait({
turnId: event.turnId ?? null,
event,
});
if (governedResult !== null && !isTurnTerminal(event)) {
await session.cancel?.({
reason: "Paperclip parked this turn on a durable governed interaction.",
}).catch(() => undefined);
}
}
if (isTurnTerminal(event)) {
return { event, eventCount, highestContiguousSourceSeq, governedResult };
}
}
throw new Error("native event stream closed before a turn terminal fact");
})(),
@@ -143,7 +182,12 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
await options.controlPlane.checkpointSession?.(recoveredSnapshot);
await options.onCheckpoint?.(recoveredSnapshot);
let consumed = { event: null as PrpEvent | null, eventCount: 0, highestContiguousSourceSeq: 0 };
let consumed = {
event: null as PrpEvent | null,
eventCount: 0,
highestContiguousSourceSeq: 0,
governedResult: null as PrpStructuredRunResult | null,
};
const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal
? recoveredSnapshot
: persistedSession;
@@ -155,29 +199,84 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions)
}
: null;
if (!completed) {
const consuming = consumeTurn(session, options.controlPlane, options.timeoutMs ?? 900_000);
const consuming = consumeTurn(
session,
options.controlPlane,
options.timeoutMs ?? 900_000,
options.resolveGovernedWait,
);
// Event consumption must begin before startTurn so an eager provider cannot
// outrun us. Observe its rejection immediately, though: if startTurn or
// checkpointing fails first, the outer finally closes the session and the
// abandoned consumer will reject when its stream closes. Without a handler
// that later rejection becomes process-fatal under Node's strict policy.
void consuming.catch(() => undefined);
const recoveredActiveTurnId = recoveredSnapshot.activeTurnId ?? persistedSession?.activeTurnId ?? null;
// A recovered driver is authoritative about whether a provider turn is
// still active. In particular, drivers normalize the checkpoint race
// where a terminal fingerprint was persisted before activeTurnId was
// cleared. Falling back to the older control-plane checkpoint here
// resurrects that terminal turn and waits forever for an event that was
// already consumed.
const recoveredActiveTurnId = recovered
? recoveredSnapshot.activeTurnId ?? null
: persistedSession?.activeTurnId ?? null;
if (!recovered || !recoveredActiveTurnId) {
const modelEnvelope = buildNativeModelEnvelope(input);
const dispositionOnlyRecovery = Boolean(
recovered &&
!recoveredSnapshot.semanticResult &&
(persistedSession?.terminalTurns?.length ?? 0) > 0 &&
!recoveredActiveTurnId
);
if (dispositionOnlyRecovery) {
modelEnvelope.task.prompt = [
"Paperclip semantic-result recovery for a prior completed provider turn.",
"The prior turn already performed the work and its user-facing final answer is recorded.",
"Do not repeat implementation, tests, research, or the final answer.",
"Use the existing session context to invoke exactly one paperclip_finish or paperclip_block with the accurate current disposition, then stop without additional user-facing prose.",
].join("\n");
}
await session.startTurn({
message: { role: "user", text: JSON.stringify(buildNativeModelEnvelope(input)) },
message: { role: "user", text: JSON.stringify(modelEnvelope) },
requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default",
});
await checkpoint();
}
const terminalEvent = await consuming;
consumed = terminalEvent;
completed = await session.result();
completed = terminalEvent.governedResult === null
? await session.result()
: {
result: terminalEvent.governedResult,
terminal: {
schema: "paperclip.prp.terminal.v1",
turnTerminalState: "completed",
runTerminalState: "succeeded",
reportedWorkDisposition:
terminalEvent.governedResult.reportedWorkDisposition,
},
turnId: terminalEvent.event.turnId ?? null,
};
if (completed === null && options.resolveMissingResult) {
const recoveredResult = await options.resolveMissingResult({
turnId: terminalEvent.event.turnId ?? null,
terminalEvent: terminalEvent.event,
});
if (recoveredResult !== null) {
completed = {
result: recoveredResult,
terminal: terminalFromEvent(terminalEvent.event, recoveredResult.reportedWorkDisposition),
turnId: terminalEvent.event.turnId ?? null,
};
}
}
await checkpoint();
}
if (completed === null) throw new Error("native_finalization_missing: session returned no semantic result");
let terminal: PrpTerminalState;
if (completionSnapshot?.semanticResult && completionSnapshot.terminal) {
if (consumed.governedResult !== null) {
terminal = completed.terminal;
} else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) {
terminal = completionSnapshot.terminal;
} else {
terminal = terminalFromEvent(consumed.event!, completed.result.reportedWorkDisposition);
packages/paperclip-runner/src/protocol-actions/create-task.ts
+5 -5
View File
@@ -20,16 +20,16 @@ export const createTaskAction = {
"idempotency": "required",
"disabledByDefault": false,
"realBindingStatus": "live_codex",
"realServiceBinding": "unbound",
"realServiceBinding": "issues.decomposeAcceptedPlan",
"prpEvidence": "semantic-operation item event plus company-entity state diff and audit record",
"prpBindingStatus": "audit_pending",
"prpBindingStatus": "bound",
"legacyAliases": [
"mcp:paperclipCreateIssue"
]
},
"documentation": {
"title": "Create child task",
"description": "Create one child mock task under the active task.",
"description": "Create one durable implementation child under an accepted planning task.",
"note": null
},
"examples": {
@@ -63,7 +63,7 @@ export const createTaskAction = {
"operationId": "create_task",
"version": 1,
"title": "Create child task",
"description": "Create one child mock task under the active task.",
"description": "Create one durable implementation child under an accepted planning task. Include the complete approved plan in the child description.",
"exposure": "optional",
"requiredClaims": [
"delegation:tasks:create"
@@ -100,7 +100,7 @@ export const createTaskAction = {
"string",
"null"
],
"description": "Optional mock actor assignee.",
"description": "Optional agent assignee. Omit to assign the current agent.",
"maxLength": 20000
},
"priority": {
packages/paperclip-runner/src/protocol-actions/request-human-input.ts
+7 -1
View File
@@ -13,6 +13,7 @@ export const requestHumanInputAction = {
"taskModes": [
"standard",
"planning",
"ask",
"skill_test"
],
"sideEffectClass": "task_write",
@@ -69,6 +70,7 @@ export const requestHumanInputAction = {
"allowedModes": [
"standard",
"planning",
"ask",
"skill_test"
],
"inputSchema": {
@@ -103,6 +105,7 @@ export const requestHumanInputAction = {
},
"payload": {
"type": "object",
"description": "Kind-specific interaction data. For interactionKind='questions', use exactly {version:1, questions:[{id,prompt,selectionMode:'single'|'multi',required?,options:[{id,label,description?,freeText?}]}]}; option keys are id/label, not value, and question choice cardinality is selectionMode, not type. For confirmation, payload may be {}. Keep all ids stable across retries.",
"additionalProperties": true
},
"targetRevisionId": {
@@ -209,7 +212,9 @@ export const requestHumanInputAction = {
"type": "string",
"minLength": 1
},
"payload": {},
"payload": {
"description": "Kind-specific interaction data. Questions use {version:1, questions:[{id,prompt,selectionMode:'single'|'multi',required?,options:[{id,label,description?,freeText?}]}]}; do not use type:'single_choice' or option.value."
},
"targetRevisionId": {
"oneOf": [
{
@@ -279,6 +284,7 @@ export const requestHumanInputAction = {
"taskModes": [
"standard",
"planning",
"ask",
"skill_test"
],
"sideEffectClass": "task_write",
Loaded 100 of 244 files, more files were not shown because too many files have changed in this diff. Show more