mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 02:44:50 +01:00
feat(runner): add skillless Codex app-server tracer
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed
1 parent
6f179780e2
commit
2161ddcc4a
25 files changed
+18738
-24
No files matched your search
@@ -10,10 +10,12 @@ static replay, and a standalone browser reference page. Phase 2 adds the Rust
|
||||
runner process, process-group supervision, a scripted fake harness over stdio
|
||||
JSONL, a TypeScript mock core, and validated live browser replay. Phase 3 adds
|
||||
the Rust outbound WebSocket client, private durable state, replay and restart
|
||||
recovery, and safe CLI/browser diagnostics. These phases do not import or start
|
||||
Paperclip's server, UI, CLI, or production database.
|
||||
recovery, and safe CLI/browser diagnostics. Phase 4 adds a direct, skillless
|
||||
Codex app-server driver, semantic completion tools, canonical event tracing,
|
||||
and a real-model example against the mock core. These phases do not import or
|
||||
start Paperclip's server, UI, CLI, or production database.
|
||||
|
||||
## Phase 0–3 quick start
|
||||
## Phase 0–4 quick start
|
||||
|
||||
From the repository root:
|
||||
|
||||
@@ -52,6 +54,7 @@ devtool:
|
||||
pnpm --filter @paperclipai/paperclip-runner replay:phase1
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase2 -- --scenario happy-path
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase3 -- --fault lost-ack
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase4
|
||||
pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port 4179
|
||||
```
|
||||
|
||||
@@ -75,9 +78,11 @@ pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port
|
||||
| `record:phase2` | Capture a validated happy-path live trace as a replay fixture. |
|
||||
| `trace:phase3` | Run the Rust runner against the mock core with a selected recovery fault. |
|
||||
| `record:phase3` | Regenerate the complete Phase 3 fault matrix and exact per-fault traces. |
|
||||
| `trace:phase4` | Run the mock core with a real, local skillless Codex app-server session. |
|
||||
| `record:phase4` | Run the safe Codex task and record its validated, normalized trace. |
|
||||
| `browser:dev` | Start the standalone live/replay/recovery browser devtool. |
|
||||
| `test:browser` | Exercise static replay and live scenarios, then capture temporary screenshots under ignored test output. |
|
||||
| `verify` | Run the complete Phase 0 through Phase 3 acceptance sequence. |
|
||||
| `verify` | Run the complete deterministic Phase 0 through Phase 4 acceptance sequence. |
|
||||
| `verify:rootless` | Extract Debian/Ubuntu browser libraries without root, then run `verify`. |
|
||||
|
||||
## Navigate
|
||||
@@ -90,6 +95,8 @@ pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port
|
||||
- [Phase 2 local protocol reference](docs/phase-02-local-protocol.md)
|
||||
- [Phase 3 break-it-on-purpose tutorial](docs/tutorials/phase-03-break-recovery.md)
|
||||
- [Phase 3 durable transport reference](docs/phase-03-durable-transport.md)
|
||||
- [Phase 4 skillless Codex tutorial](docs/tutorials/phase-04-skillless-codex.md)
|
||||
- [Phase 4 skillless Codex driver reference](docs/phase-04-skillless-codex-driver.md)
|
||||
- [PRP compatibility/versioning policy](docs/protocol-compatibility.md)
|
||||
- [Cumulative end-to-end tutorial](docs/tutorials/end-to-end.md)
|
||||
- [Journal guide](docs/journal.md)
|
||||
@@ -97,4 +104,5 @@ pnpm --filter @paperclipai/paperclip-runner browser:dev --host 127.0.0.1 --port
|
||||
- [Implementation plan](spec/paperclip-native-runner-implementation-plan.md)
|
||||
- [Normative spike specification](spec/paperclip-native-runner-spike-spec.md)
|
||||
|
||||
Phase 3 adds package-local durable network recovery. Production Paperclip integration and a real model harness remain deferred.
|
||||
Phase 4 adds the package-local real-model reference driver. Production
|
||||
Paperclip integration and Phase 5 remain deferred.
|
||||
@@ -9,6 +9,7 @@ Each tutorial starts with an explanation of what the phase is and what the phase
|
||||
- [Phase 1: static PRP replay](tutorials/phase-01-static-replay.md) — validate the shared schema/fixture corpus, reduce a fixture in the CLI, and inspect the same final snapshot in the standalone browser page.
|
||||
- [Phase 2: local runner and fake harness](tutorials/phase-02-local-runner.md) — run a supervised Rust process, exercise scripted live scenarios, resolve requests, interrupt a turn, and confirm live/replay parity.
|
||||
- [Phase 3: break recovery on purpose](tutorials/phase-03-break-recovery.md) — lose an ACK, drop the socket, restart the runner side, replay durable events, and inspect recovery diagnostics.
|
||||
- [Phase 4: run the skillless Codex driver](tutorials/phase-04-skillless-codex.md) — run a safe real-model task, inspect the exact context boundary, steer or interrupt it, and confirm one replay-safe result.
|
||||
- [Cumulative end-to-end tutorial](tutorials/end-to-end.md) — the shortest complete workflow available at the current phase.
|
||||
|
||||
## Reference
|
||||
@@ -17,8 +18,10 @@ Each tutorial starts with an explanation of what the phase is and what the phase
|
||||
- [PRP compatibility and versioning policy](protocol-compatibility.md)
|
||||
- [Phase 2 local protocol and supervision](phase-02-local-protocol.md)
|
||||
- [Phase 3 durable transport and recovery](phase-03-durable-transport.md)
|
||||
- [Phase 4 skillless Codex driver](phase-04-skillless-codex-driver.md)
|
||||
- [Engineering journal guide](journal.md)
|
||||
- [Dated shadcn/ui and AI Elements compatibility note](research/2026-08-07-ui-library-compatibility.md)
|
||||
- [Package README](../README.md)
|
||||
|
||||
Phase 3 is implemented and ready for its review and human checkpoint. Later tutorials remain deferred.
|
||||
Phase 4 is implemented and ready for Security, CTO, QA, and human review.
|
||||
Phase 5 remains uncreated.
|
||||
@@ -0,0 +1,119 @@
|
||||
# Phase 4 Skillless Codex Driver
|
||||
|
||||
## Scope
|
||||
|
||||
Phase 4 implements a direct Codex app-server v2 driver behind the package's
|
||||
existing `HarnessDriver` contract. The driver, mock core, example CLI, tests,
|
||||
and evidence stay inside `packages/paperclip-runner/`. They do not import or
|
||||
change Paperclip server, UI, database, or production control-plane behavior.
|
||||
|
||||
The app-server process is local to the execution environment and uses newline
|
||||
delimited JSON-RPC over stdio. It is not exposed as a network service.
|
||||
|
||||
## Identity mapping
|
||||
|
||||
| Runner identity | Codex source | Persistence rule |
|
||||
| --- | --- | --- |
|
||||
| run ID | mock-core input | Never replaced during recovery. |
|
||||
| normalized session ID | `session:<runId>` | Stable across transport/process recovery. |
|
||||
| driver session ID | `thread.id` | Resumed by exact ID. A different returned ID fails recovery. |
|
||||
| provider session ID | `thread.sessionId` | Kept separately from the driver thread ID. |
|
||||
| turn ID | `turn.id` | Required by steer and interrupt preconditions. |
|
||||
| item ID | `item.id`, request ID, or deterministic turn/kind key | Preserved on lifecycle and delta events. |
|
||||
| source event ID | runner instance + run + source sequence | Source sequence continues from the persisted snapshot. |
|
||||
|
||||
The persisted session snapshot records run, normalized session, driver
|
||||
session, provider session, active turn, and last source sequence. Recovery
|
||||
starts a new local app-server transport, calls `thread/resume`, then uses
|
||||
`thread/read` for reconciliation. It never silently starts a replacement
|
||||
thread.
|
||||
|
||||
## App-server operations
|
||||
|
||||
| Driver operation | App-server method | Degradation |
|
||||
| --- | --- | --- |
|
||||
| initialize | `initialize`, then `initialized` | Startup fails visibly. |
|
||||
| create | `thread/start` | Required. |
|
||||
| resume | `thread/resume` | `recovered: false` with a redacted reason. |
|
||||
| read | `thread/read` | Explicit `UnsupportedCodexOperationError`. |
|
||||
| start turn | `turn/start` | Required. |
|
||||
| steer | `turn/steer` with `expectedTurnId` | Explicit unsupported diagnostic; no stdin fallback. |
|
||||
| interrupt | `turn/interrupt` | Explicit unsupported diagnostic; session is not killed. |
|
||||
| usage | `thread/tokenUsage/updated` | Returns the last snapshot or explicit unsupported error. |
|
||||
| reconcile | `thread/read` plus `session.reconciled` | Disabled when read is unavailable. |
|
||||
|
||||
Capability flags are descriptive and executable. Unsupported operations emit
|
||||
canonical `harness.diagnostic` events with secret-redacted detail. No
|
||||
harness-specific branch is required in the mock core.
|
||||
|
||||
## Skillless context boundary
|
||||
|
||||
The model receives one text input containing `paperclip.skillless_task.v1`:
|
||||
|
||||
- objective;
|
||||
- completion-contract revision and criteria;
|
||||
- task constraints; and
|
||||
- the expected canonical result schema name.
|
||||
|
||||
The thread config explicitly disables automatic skill, app, and collaboration
|
||||
instruction blocks. The model input accepts only text, never a Codex `skill`
|
||||
input. The driver captures the returned instruction-source list and requires it
|
||||
to be empty for the skillless assertion.
|
||||
|
||||
The child process environment is allowlisted. It may receive only local runtime
|
||||
keys such as `PATH`, `HOME`, `CODEX_HOME`, locale, temporary-directory, trusted
|
||||
certificate, and credential-free proxy settings. Paperclip bearer values,
|
||||
`OPENAI_API_KEY`, arbitrary skill paths, and other inherited variables are not
|
||||
passed. Diagnostics redact bearer/basic credentials, credentialed proxy URLs,
|
||||
secret query parameters, sensitive JSON keys, and common key assignments.
|
||||
|
||||
The context snapshot records configuration and environment **key names**, not
|
||||
secret values.
|
||||
|
||||
## Semantic completion
|
||||
|
||||
The provider-facing structured-output schema covers `done` and `needs_review`.
|
||||
It uses the strict OpenAI shape: every object rejects additional properties and
|
||||
the constant schema field includes both `type: "string"` and `const`.
|
||||
|
||||
Two dynamic semantic tools are registered when supported:
|
||||
|
||||
- `paperclip_finish` accepts `done` or `needs_review`;
|
||||
- `paperclip_block` accepts `blocked` and requires a blocker owner, action,
|
||||
reason, and scope.
|
||||
|
||||
Both normalize through the canonical `paperclip.run_result.v1` validator. The
|
||||
first valid result is committed. Repeating the same result is idempotent; a
|
||||
different later result is rejected. A completed turn with no valid semantic
|
||||
result produces one explicit `needs_review` result and one failed terminal
|
||||
event. Process exit or prose alone never implies completion.
|
||||
|
||||
## Canonical event mapping
|
||||
|
||||
- thread lifecycle -> `session.started`, `session.resumed`,
|
||||
`session.reconciled`;
|
||||
- turn lifecycle -> `turn.submitted`, `turn.accepted`, `turn.started`, and one
|
||||
terminal turn event;
|
||||
- messages, reasoning, plans, commands, file changes, dynamic tools, and diffs
|
||||
-> `item.started`, `item.delta`, `item.completed`;
|
||||
- model selection -> a completed `model` item;
|
||||
- app-server decisions -> `runtime_request.created` and
|
||||
`runtime_request.resolved` with redacted detail;
|
||||
- token snapshots -> completed `usage` items;
|
||||
- semantic verification rows -> completed `verification` items;
|
||||
- completion -> one `run.result.proposed` and one `run.terminal`.
|
||||
|
||||
The existing Phase 1 reducer consumes the live stream and the replay stream.
|
||||
The Phase 4 tracer requires byte-equivalent snapshots, contiguous source
|
||||
sequences, stable item IDs, one result, and one terminal event.
|
||||
|
||||
## Runnable example
|
||||
|
||||
`trace:phase4` starts a real local `codex app-server` session through the mock
|
||||
core. Its safe task can create only `hello.txt`, disables network access, and
|
||||
checks the exact file contents. See the
|
||||
[Phase 4 tutorial](tutorials/phase-04-skillless-codex.md).
|
||||
|
||||
This phase changes no browser surface, so no new browser screenshot applies.
|
||||
The canonical events are proved through the existing reducer/replay path and
|
||||
JSON trace evidence.
|
||||
@@ -3,18 +3,21 @@
|
||||
## What this tutorial is
|
||||
|
||||
This tutorial combines each implemented Native Runner phase into one procedure.
|
||||
It currently includes Phase 0 through Phase 3.
|
||||
It currently includes Phase 0 through Phase 4.
|
||||
|
||||
## What this tutorial proves
|
||||
|
||||
This tutorial proves that the standalone package boundary, static replay path,
|
||||
local live-run path, and durable recovery path work together. It does not use
|
||||
Paperclip or a real model.
|
||||
local live-run path, durable recovery path, and direct Codex driver work
|
||||
together. It does not use the Paperclip control plane. Phase 4 uses a real
|
||||
local Codex session through the mock core.
|
||||
|
||||
The current system includes the Rust mock-core tracer, shared protocol fixtures,
|
||||
the Rust supervisor, a scripted fake harness, CLI live runs, and browser live and
|
||||
replay modes. It also includes the Rust outbound WebSocket client, durable
|
||||
outbox, package-local mock core, recovery CLI, and recovery browser view.
|
||||
The final phase adds a skillless task envelope, direct app-server driver,
|
||||
semantic completion tools, and the same reducer/replay proof used by fixtures.
|
||||
|
||||
## Current end-to-end path
|
||||
|
||||
@@ -39,6 +42,13 @@ outbox, package-local mock core, recovery CLI, and recovery browser view.
|
||||
15. Confirm that the recovered runner and session IDs stay the same.
|
||||
16. Confirm that the outbox is empty after replay and cumulative acknowledgement.
|
||||
17. Open the [Phase 3 journal entry](../../knowledge/journal/2026-08-07-phase-03.md).
|
||||
18. Follow [Phase 4: Run the Skillless Codex Driver](phase-04-skillless-codex.md).
|
||||
19. Inspect the exact model-context snapshot and confirm that it has no
|
||||
Paperclip instructions, bearer credentials, or unrelated skills.
|
||||
20. Run the safe task, then steer and interrupt separate sessions. Confirm
|
||||
stable session identities and exactly one result and terminal event.
|
||||
21. Open the [Phase 4 journal entry](../../knowledge/journal/2026-08-08-phase-04.md)
|
||||
and its linked real-session trace and verification evidence.
|
||||
|
||||
The one-command form after installation is:
|
||||
|
||||
@@ -82,6 +92,17 @@ pnpm --filter @paperclipai/paperclip-runner verify:rootless
|
||||
- lease expiry, drain, revoke, and unrecoverable storage outcomes are explicit;
|
||||
- CLI and browser diagnostics do not expose bootstrap or connection-lease
|
||||
tokens.
|
||||
- the Codex child receives an allowlisted environment without Paperclip or
|
||||
OpenAI bearer credentials;
|
||||
- automatic skill, app, and collaboration instruction blocks are disabled;
|
||||
- direct app-server create, resume, read, turn, steer, interrupt, usage, and
|
||||
reconciliation operations preserve stable identities;
|
||||
- provider events normalize to canonical lifecycle, model, tool, file,
|
||||
request, usage, verification, result, and terminal events;
|
||||
- the first validated semantic completion wins, identical duplicates are
|
||||
idempotent, and a changed duplicate is rejected;
|
||||
- unsupported capabilities degrade through explicit redacted diagnostics;
|
||||
- the real trace and its replay reduce to the same final snapshot.
|
||||
|
||||
Phase 4 remains uncreated until the Phase 3 review and human checkpoint are
|
||||
Phase 5 remains uncreated until the Phase 4 review and human checkpoint are
|
||||
complete.
|
||||
@@ -0,0 +1,170 @@
|
||||
# Phase 4: Run the Skillless Codex Driver
|
||||
|
||||
## What this phase is
|
||||
|
||||
Phase 4 connects the mock core to a real Codex app-server session.
|
||||
|
||||
Codex gets one small task envelope. Codex does not get the Paperclip skill.
|
||||
Codex does not get a Paperclip API key or API instructions.
|
||||
|
||||
## What this phase proves
|
||||
|
||||
This phase proves that a real Codex session can create a file, stream typed
|
||||
events, return one checked result, and replay the same result.
|
||||
|
||||
It also proves that steer, interrupt, resume, and unsupported-operation paths
|
||||
keep clear session identities and safe diagnostics.
|
||||
|
||||
## Before you start
|
||||
|
||||
- Run commands from the repository root.
|
||||
- Install Node.js 20+, pnpm 9+, Rust, and the `codex` CLI.
|
||||
- Sign in to Codex with the normal local Codex setup.
|
||||
- Do not add a Paperclip or OpenAI API key for this tutorial.
|
||||
|
||||
Check the installed harness:
|
||||
|
||||
```sh
|
||||
codex --version
|
||||
codex app-server --help
|
||||
```
|
||||
|
||||
## Step 1: Run the focused conformance checks
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner typecheck:typescript
|
||||
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
|
||||
src/drivers/codex/codex-app-server-driver.test.ts
|
||||
pnpm --filter @paperclipai/paperclip-runner check:forbidden-imports
|
||||
```
|
||||
|
||||
Confirm that the driver tests pass and the package boundary check says
|
||||
`Standalone boundary check passed.`
|
||||
|
||||
## Step 2: Run the safe real Codex task
|
||||
|
||||
Choose a temporary directory and trace file:
|
||||
|
||||
```sh
|
||||
phase4_workspace="$(mktemp -d)"
|
||||
phase4_trace="$phase4_workspace/trace.json"
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase4 -- \
|
||||
--working-directory "$phase4_workspace" \
|
||||
--output "$phase4_trace"
|
||||
```
|
||||
|
||||
Confirm:
|
||||
|
||||
- the result is `done`;
|
||||
- every printed assertion says `PASS`;
|
||||
- the file has the exact text:
|
||||
|
||||
```sh
|
||||
test "$(cat "$phase4_workspace/hello.txt")" = "hello from phase 4"
|
||||
```
|
||||
|
||||
## Step 3: Inspect the exact model boundary
|
||||
|
||||
```sh
|
||||
jq '.context | {
|
||||
protocolVersion,
|
||||
codexVersion,
|
||||
model,
|
||||
modelProvider,
|
||||
workingDirectory,
|
||||
sandbox,
|
||||
approvalPolicy,
|
||||
baseInstructions,
|
||||
instructionSources,
|
||||
instructionPolicy,
|
||||
environmentKeys,
|
||||
dynamicToolNames,
|
||||
modelInputKinds,
|
||||
envelope
|
||||
}' "$phase4_trace"
|
||||
```
|
||||
|
||||
Confirm:
|
||||
|
||||
- `instructionSources` is empty;
|
||||
- all three instruction-policy values are `false`;
|
||||
- `modelInputKinds` contains only `text`;
|
||||
- the semantic tools are `paperclip_finish` and `paperclip_block`;
|
||||
- no environment **value** is present;
|
||||
- the envelope contains only this safe task and its completion criteria.
|
||||
|
||||
Check that no control-plane route or bearer credential appears:
|
||||
|
||||
```sh
|
||||
if rg -ni 'authorization: bearer|paperclip_api_key|/api/issues/' "$phase4_trace"; then
|
||||
echo "FAIL: forbidden control-plane context found"
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Step 4: Inspect canonical events and replay
|
||||
|
||||
```sh
|
||||
jq '[.events[].eventType] | group_by(.) | map({type: .[0], count: length})' \
|
||||
"$phase4_trace"
|
||||
jq '.assertions' "$phase4_trace"
|
||||
```
|
||||
|
||||
Confirm:
|
||||
|
||||
- session and turn lifecycle events exist;
|
||||
- model, command/tool, usage, verification, result, and terminal events exist;
|
||||
- `exactlyOneTerminalResult` is `true`;
|
||||
- `liveReplayParity` is `true`;
|
||||
- source sequence and item identity assertions are `true`.
|
||||
|
||||
The focused test suite supplies deterministic file-change and runtime-request
|
||||
events because a real model may choose a shell command instead of a file patch
|
||||
and may not need human input for this small task.
|
||||
|
||||
## Step 5: Try steering
|
||||
|
||||
Use a new directory:
|
||||
|
||||
```sh
|
||||
phase4_steer_workspace="$(mktemp -d)"
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase4 -- \
|
||||
--working-directory "$phase4_steer_workspace" \
|
||||
--steer "Keep the answer short and verify the exact file text."
|
||||
```
|
||||
|
||||
Confirm that the same driver and provider session IDs are printed and the task
|
||||
still returns one result.
|
||||
|
||||
## Step 6: Try interruption
|
||||
|
||||
Use a new directory:
|
||||
|
||||
```sh
|
||||
phase4_interrupt_workspace="$(mktemp -d)"
|
||||
pnpm --filter @paperclipai/paperclip-runner trace:phase4 -- \
|
||||
--working-directory "$phase4_interrupt_workspace" \
|
||||
--interrupt
|
||||
```
|
||||
|
||||
An interrupt can win before the file is created. That is expected. Confirm that
|
||||
the trace still has one session identity, one semantic result, and one terminal
|
||||
event. It must not create a replacement session.
|
||||
|
||||
## Step 7: Run the package verification path
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner verify
|
||||
```
|
||||
|
||||
This command is the package acceptance path. The real Codex example remains a
|
||||
separate command because automated environments may not have a signed-in Codex
|
||||
session.
|
||||
|
||||
## Step 8: Read the evidence
|
||||
|
||||
- [Phase 4 driver reference](../phase-04-skillless-codex-driver.md)
|
||||
- [Phase 4 verification evidence](../../knowledge/evidence/2026-08-08-phase-04-verification.md)
|
||||
- [Phase 4 journal](../../knowledge/journal/2026-08-08-phase-04.md)
|
||||
|
||||
Phase 5 is not part of this tutorial and remains uncreated.
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
type: Verification Evidence
|
||||
title: Phase 4 skillless Codex driver verification
|
||||
description: Package-only evidence for the direct Codex app-server driver, skillless context boundary, semantic completion, recovery, and replay.
|
||||
tags: [native-runner, phase-4, codex, app-server, skillless, evidence]
|
||||
status: stable
|
||||
generated: { by: openai/gpt-5, at: 2026-08-08T00:00:00Z }
|
||||
---
|
||||
|
||||
# Scope
|
||||
|
||||
This evidence covers the direct Codex app-server driver and package-local mock
|
||||
core. It does not import or change Paperclip server, UI, database, CLI, or
|
||||
production control-plane behavior.
|
||||
|
||||
# Required checks
|
||||
|
||||
```sh
|
||||
pnpm --filter @paperclipai/paperclip-runner typecheck:typescript
|
||||
pnpm --filter @paperclipai/paperclip-runner exec vitest run \
|
||||
src/drivers/codex/codex-app-server-driver.test.ts
|
||||
pnpm --filter @paperclipai/paperclip-runner check:forbidden-imports
|
||||
pnpm --filter @paperclipai/paperclip-runner record:phase4
|
||||
pnpm --filter @paperclipai/paperclip-runner verify
|
||||
```
|
||||
|
||||
# Acceptance mapping
|
||||
|
||||
- Common driver conformance covers create, resume, read, turn, steer,
|
||||
interrupt, usage, reconciliation, stable identities, and exactly one result.
|
||||
- The exact real-session context records the Codex/app-server version, model,
|
||||
provider, working directory, sandbox, approval policy, base instructions,
|
||||
instruction sources and policy, environment key names, tools, input kinds,
|
||||
and complete compact envelope.
|
||||
- Automatic skill, app, and collaboration instructions are disabled. The
|
||||
instruction-source list is empty, and the model receives no Paperclip API
|
||||
instruction, bearer credential, unrelated skill, or inherited secret.
|
||||
- Semantic finish and block tools normalize to one validated canonical result.
|
||||
An identical duplicate is idempotent; a changed duplicate is rejected.
|
||||
- Steering and interruption retain the same driver and provider session.
|
||||
- Resume and reconciliation retain the persisted run, normalized session,
|
||||
driver session, provider session, turn, item, and source cursor identities.
|
||||
- Unsupported capabilities emit explicit redacted diagnostics without
|
||||
driver-specific mock-core branches.
|
||||
- All live events pass the existing Phase 1 validator/reducer. Replaying them
|
||||
produces the same snapshot.
|
||||
- The package boundary rejects imports from Paperclip core packages.
|
||||
|
||||
# Recorded outputs
|
||||
|
||||
- [Validated real Codex trace](phase-04-codex-trace.json)
|
||||
- [Driver reference](../../docs/phase-04-skillless-codex-driver.md)
|
||||
- [Hands-on tutorial](../../docs/tutorials/phase-04-skillless-codex.md)
|
||||
|
||||
The trace normalizes its temporary working directory and writable Codex state
|
||||
root. It contains no credential values. This phase changes no browser surface,
|
||||
so a browser screenshot is not applicable.
|
||||
|
||||
# Local verification result
|
||||
|
||||
- Codex CLI/app-server: `0.132.0`.
|
||||
- Real model/provider: `gpt-5.5` / `openai`.
|
||||
- Focused driver tests: 7 passed.
|
||||
- Full TypeScript suite: 73 passed.
|
||||
- Full Rust suite: 39 passed (37 unit and 2 process-supervisor tests).
|
||||
- Existing browser suite: 9 passed with the rootless Linux library path.
|
||||
- The real safe task created `hello.txt` with exactly `hello from phase 4`.
|
||||
- The real trace contains 340 canonical events and all eight assertions pass:
|
||||
exactly one terminal result, live/replay parity, stable run/session identity,
|
||||
contiguous source sequence, stable item identity, skillless context,
|
||||
unrelated-skill absence, and credential absence. The recorder separately
|
||||
verifies the exact safe-file content.
|
||||
- Real steering and interruption commands preserved the session identity and
|
||||
ended with exactly one result and terminal event.
|
||||
- TypeScript, Rust, and browser builds and typechecks passed. Golden, parity,
|
||||
token, package-boundary, documentation-link, and OKF checks passed.
|
||||
- The complete package `verify` path passed through `verify:rootless`.
|
||||
|
||||
# Failures found and fixed
|
||||
|
||||
- The first real session returned an app-server `400` because the strict JSON
|
||||
Schema used `const` without its required `type`. The finish and block schemas
|
||||
now use `type: "string"` with `const`, and tests lock the provider shape.
|
||||
- The first sandboxed launch could not write Codex's local SQLite state. The
|
||||
real-session proof uses the normal signed-in Codex environment; deterministic
|
||||
conformance remains fully package-local and does not need that state.
|
||||
- The installed Codex home contained an unrelated invalid skill. The driver
|
||||
explicitly disables automatic skill instructions and verifies the returned
|
||||
instruction-source list is empty instead of relying on the host setup.
|
||||
- The normal package verifier reached Playwright but the host lacked
|
||||
`libatk-1.0.so.0`. The supported rootless verifier supplied private browser
|
||||
libraries, then completed the same `verify` path with all nine browser tests.
|
||||
|
||||
# Review state
|
||||
|
||||
- Security context/credential review: pending.
|
||||
- CTO semantic-completion contract review: pending.
|
||||
- QA hands-on tutorial run: pending.
|
||||
- Human checkpoint: pending.
|
||||
@@ -7,3 +7,4 @@
|
||||
* [Phase 1 verification](2026-08-07-phase-01-verification.md) - Schema, golden, reducer, Rust parity, CLI, browser, token, boundary, and screenshot evidence.
|
||||
* [Phase 2 verification](2026-08-07-phase-02-verification.md) - Rust supervisor, fake-driver scenarios, cleanup, live/replay parity, browser flows, screenshots, and complete package verification.
|
||||
* [Phase 3 verification](2026-08-07-phase-03-verification.md) - Lost-ACK replay, restart recovery, command deduplication, storage, and redacted diagnostics.
|
||||
* [Phase 4 verification](2026-08-08-phase-04-verification.md) - Direct Codex conformance, exact skillless context, semantic completion, recovery, and real-session replay.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,83 @@
|
||||
---
|
||||
type: Engineering Journal Entry
|
||||
title: Phase 4 skillless Codex app-server driver
|
||||
description: Decisions, evidence, failures, and fixes for the real Codex driver and model-context boundary.
|
||||
tags: [native-runner, phase-4, codex, app-server, skillless, completion]
|
||||
status: stable
|
||||
generated: { by: openai/gpt-5, at: 2026-08-08T00:00:00Z }
|
||||
entry_kind: phase
|
||||
phase: "4"
|
||||
---
|
||||
|
||||
# Context
|
||||
|
||||
Phase 4 connects the already deterministic mock core and reducer to a real
|
||||
Codex app-server process. It proves the driver boundary without adding a
|
||||
Paperclip production integration or browser surface.
|
||||
|
||||
# Decisions
|
||||
|
||||
1. Use app-server v2 JSON-RPC over local stdio. Keep its experimental network
|
||||
transport out of the runner WAN contract.
|
||||
2. Keep runner, normalized session, driver thread, provider session, turn,
|
||||
item, and source-event identities separate and stable.
|
||||
3. Resume only the exact persisted thread. Never replace a lost session with a
|
||||
new one.
|
||||
4. Use one compact `paperclip.skillless_task.v1` text envelope. Disable
|
||||
automatic skill, app, collaboration, plugin, memory, and multi-agent
|
||||
instruction blocks in thread config.
|
||||
5. Pass an allowlisted child environment. Record key names, never values.
|
||||
6. Register separate `paperclip_finish` and `paperclip_block` dynamic tools.
|
||||
Normalize both into the same canonical result validator.
|
||||
7. Commit the first valid semantic result. Treat the same later result as an
|
||||
idempotent retry and reject any changed result.
|
||||
8. Use app-server capabilities to implement or explicitly degrade read, steer,
|
||||
interrupt, usage, structured output, dynamic tools, and reconciliation.
|
||||
9. Normalize every provider notification through the existing PRP validator,
|
||||
reducer, and replay path.
|
||||
10. Keep the real example safe: workspace-write sandbox, no network request,
|
||||
one allowed file, and exact output verification.
|
||||
|
||||
# Evidence
|
||||
|
||||
See the [Phase 4 verification evidence](../evidence/2026-08-08-phase-04-verification.md)
|
||||
and [validated real trace](../evidence/phase-04-codex-trace.json).
|
||||
|
||||
Follow the [Phase 4 tutorial](../../docs/tutorials/phase-04-skillless-codex.md)
|
||||
and read the [driver reference](../../docs/phase-04-skillless-codex-driver.md).
|
||||
|
||||
# Failures
|
||||
|
||||
- The first real structured-output request failed because the schema combined
|
||||
`const` with no explicit JSON type. Adding `type: "string"` fixed the
|
||||
app-server validation error; focused tests now assert the exact schema.
|
||||
- A sandboxed app-server could read but not update the signed-in Codex SQLite
|
||||
state. Real-session verification therefore runs with normal local Codex
|
||||
access, while all deterministic driver tests use the in-process fake
|
||||
transport.
|
||||
- An unrelated broken skill was present in the host Codex home. Thread config
|
||||
now disables automatic skill instructions, and the context assertion checks
|
||||
both the policy flags and empty instruction-source list.
|
||||
- The first redaction test stopped reading before the diagnostic event. It now
|
||||
consumes the complete deterministic event sequence before asserting that the
|
||||
secret is absent.
|
||||
|
||||
# Known gaps
|
||||
|
||||
- The reference driver is TypeScript while the production runner direction is
|
||||
Rust. It proves the contract and real provider mapping, not the final daemon
|
||||
implementation language.
|
||||
- Real-model choices are nondeterministic. The focused fake-transport suite is
|
||||
the conformance authority; the recorded real trace is a hands-on proof.
|
||||
- The safe task may use a command instead of a file-change item. Deterministic
|
||||
tests cover both file and request mappings.
|
||||
- This package proof does not supply production MCP bindings, approval UI,
|
||||
multi-tenant credentials, or control-plane status authority.
|
||||
|
||||
# Follow-up questions
|
||||
|
||||
- Which app-server versions should the production compatibility matrix retain?
|
||||
- Should the runner persist provider item reconciliation cursors separately
|
||||
from the normalized source-event cursor?
|
||||
- Which final implementation owns the Rust equivalent of this reference
|
||||
mapping after the package contract is accepted?
|
||||
@@ -7,3 +7,4 @@
|
||||
* [Phase 1 protocol, fixtures, reducer, and static replay](2026-08-07-phase-01.md) - Executable PRP schemas, deterministic replay decisions, browser findings, and checkpoint gaps.
|
||||
* [Phase 2 local runner and fake harness](2026-08-07-phase-02.md) - Rust supervision, stdio protocol, scripted driver, live browser, cleanup, and parity decisions.
|
||||
* [Phase 3 durable outbound transport and recovery](2026-08-07-phase-03.md) - WebSocket authentication, durable ACK replay, command deduplication, and recovery diagnostics.
|
||||
* [Phase 4 skillless Codex app-server driver](2026-08-08-phase-04.md) - Direct app-server mapping, context isolation, semantic completion, real-model failures, and fixes.
|
||||
@@ -7,3 +7,10 @@
|
||||
* **Phase 1 static replay**: Added executable PRP v1 schemas, cross-language fixture/golden parity, deterministic reducer, CLI/browser replay, compatibility policy, tutorial, screenshot, and verification evidence.
|
||||
* **Phase 2 local runner**: Added the deterministic Rust supervisor, stdio fake-harness protocol, scripted scenarios, bounded logs, cleanup and terminal guards, TypeScript mock core, validated live browser mode, screenshots, tutorial, and verification evidence.
|
||||
* **Phase 3 durable recovery**: Added package-local WebSocket authentication, a durable outbox, cumulative acknowledgements, restart replay, command deduplication, recovery diagnostics, tutorial, and evidence.
|
||||
|
||||
## 2026-08-08
|
||||
|
||||
* **Phase 4 skillless Codex driver**: Added the direct app-server v2 driver,
|
||||
compact task envelope, isolated model context, semantic completion tools,
|
||||
canonical event/replay mapping, safe real-model example, tutorial, journal,
|
||||
and exact verification trace.
|
||||
@@ -45,6 +45,8 @@
|
||||
"trace:phase2": "pnpm run build:typescript && pnpm run build:phase2-binaries && node dist/cli/phase2-mock-core.js",
|
||||
"record:phase2": "pnpm run trace:phase2 -- --quiet --output knowledge/evidence/phase-02-happy-path.json",
|
||||
"trace:phase3": "pnpm run build:typescript && pnpm run build:phase2-binaries && node dist/cli/phase3-recovery.js",
|
||||
"trace:phase4": "pnpm run build:typescript && node dist/cli/phase4-codex.js",
|
||||
"record:phase4": "pnpm run build:typescript && node scripts/record-phase4-evidence.mjs",
|
||||
"diagnose:phase3": "pnpm run trace:phase3",
|
||||
"record:phase3": "pnpm run build:typescript && pnpm run build:phase2-binaries && node scripts/record-phase3-evidence.mjs",
|
||||
"record:phase3:lost-ack": "pnpm run trace:phase3 -- --fault lost-ack --output knowledge/evidence/phase-03-lost-ack.json",
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const scratchRoot =
|
||||
process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
|
||||
process.env.PAPERCLIP_SCRATCH_DIR ??
|
||||
tmpdir();
|
||||
await mkdir(scratchRoot, { recursive: true });
|
||||
const workspace = await mkdtemp(join(scratchRoot, "paperclip-runner-phase4-"));
|
||||
const rawTracePath = join(workspace, "trace.raw.json");
|
||||
const evidencePath = join(
|
||||
packageRoot,
|
||||
"knowledge/evidence/phase-04-codex-trace.json",
|
||||
);
|
||||
|
||||
try {
|
||||
const run = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
join(packageRoot, "dist/cli/phase4-codex.js"),
|
||||
"--working-directory",
|
||||
workspace,
|
||||
"--output",
|
||||
rawTracePath,
|
||||
],
|
||||
{ cwd: packageRoot, env: process.env, stdio: "inherit" },
|
||||
);
|
||||
if (run.status !== 0) {
|
||||
throw new Error(`Phase 4 tracer exited with status ${run.status ?? "unknown"}`);
|
||||
}
|
||||
|
||||
const trace = JSON.parse(await readFile(rawTracePath, "utf8"));
|
||||
if (Object.values(trace.assertions ?? {}).some((passed) => passed !== true)) {
|
||||
throw new Error("Phase 4 tracer reported a failed assertion");
|
||||
}
|
||||
const hello = await readFile(join(workspace, "hello.txt"), "utf8");
|
||||
if (hello !== "hello from phase 4") {
|
||||
throw new Error("Phase 4 safe task produced unexpected hello.txt content");
|
||||
}
|
||||
|
||||
const replacements = [[workspace, "$PHASE4_WORKSPACE"]];
|
||||
const writableRoots = trace.context?.sandbox?.writableRoots;
|
||||
if (Array.isArray(writableRoots)) {
|
||||
writableRoots.forEach((root, index) => {
|
||||
if (typeof root === "string" && root.length > 0) {
|
||||
replacements.push([root, `$CODEX_WRITABLE_ROOT_${index + 1}`]);
|
||||
}
|
||||
});
|
||||
}
|
||||
const normalized = normalizeStrings(trace, replacements);
|
||||
await writeFile(evidencePath, `${JSON.stringify(normalized, null, 2)}\n`, "utf8");
|
||||
process.stdout.write(`Recorded ${evidencePath}\n`);
|
||||
} finally {
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function normalizeStrings(value, replacements) {
|
||||
if (typeof value === "string") {
|
||||
return replacements.reduce(
|
||||
(current, [from, to]) => current.replaceAll(from, to),
|
||||
value,
|
||||
);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((entry) => normalizeStrings(entry, replacements));
|
||||
}
|
||||
if (typeof value === "object" && value !== null) {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entry]) => [
|
||||
key,
|
||||
normalizeStrings(entry, replacements),
|
||||
]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
@@ -6,7 +6,9 @@ This document is the implementation-phase plan for [`paperclip-native-runner-spi
|
||||
|
||||
All implementation starts inside `packages/paperclip-runner/` with protocol and mock-control-plane layers. These standalone phases do not modify Paperclip control-plane semantics. A real Paperclip bridge is a late, separately reviewed phase after the standalone contract passes conformance tests.
|
||||
|
||||
Phase 0, Phase 1, and Phase 2 are complete. The Phase 3 tracer is implemented and awaits Security, UX, QA, and human acceptance. Keep Phase 4 and later issues deferred until the Phase 3 human checkpoint is complete.
|
||||
Phases 0 through 3 are complete and accepted. The authorized Phase 4 tracer is
|
||||
implemented and awaits Security, CTO, QA, and human acceptance. Keep Phase 5
|
||||
and later issues uncreated until the Phase 4 human checkpoint is complete.
|
||||
|
||||
## Non-negotiable delivery rules
|
||||
|
||||
@@ -217,6 +219,15 @@ to the package index and cumulative end-to-end tutorial.
|
||||
|
||||
**Tests/evidence:** common driver conformance, model-context snapshot, credential absence test, interruption without session loss, duplicate completion idempotency, resume/reconciliation test.
|
||||
|
||||
**Implementation evidence (2026-08-08):** the package now contains the direct
|
||||
app-server v2 driver, compact envelope, separate finish/block tools, safe real
|
||||
Codex example, canonical reducer/replay trace, capability degradation, focused
|
||||
conformance suite, reference documentation, Simplified English tutorial, and
|
||||
OKF evidence. The exact context snapshot proves automatic skill, app, and
|
||||
collaboration instructions are disabled and the allowlisted child environment
|
||||
contains no Paperclip or OpenAI bearer credential. Browser screenshots are not
|
||||
applicable because Phase 4 changes no browser surface.
|
||||
|
||||
**Owners/review:** CodexCoder implementation; SecurityEngineer credential/context review; QA real-harness tutorial; CTO reviews semantic completion and status-authority boundary.
|
||||
|
||||
**Depends on:** Phase 3.
|
||||
@@ -314,8 +325,8 @@ PAP-16717 Plan and phase governance
|
||||
├─ Phase 0 Boundary/journal/skeleton create now
|
||||
├─ Phase 1 Protocol/fixtures/replay create now; blocks on Phase 0
|
||||
├─ Phase 2 Local runner/fake harness deferred; blocks on Phase 1
|
||||
├─ Phase 3 Durable transport/recovery deferred; blocks on Phase 2
|
||||
├─ Phase 4 Skillless Codex driver deferred; blocks on Phase 3
|
||||
├─ Phase 3 Durable transport/recovery complete
|
||||
├─ Phase 4 Skillless Codex driver implemented; review pending
|
||||
├─ Phase 5 Browser SDK/reference console deferred; blocks on Phase 3
|
||||
├─ Phase 6 Paperclip integration adapter deferred; blocks on Phases 4 and 5 + board gate
|
||||
└─ Phase 7 Portability/release reference kit deferred; blocks on Phase 6 for final closure
|
||||
|
||||
@@ -2198,9 +2198,11 @@ Do not depend on its experimental network WebSocket as the Paperclip WAN transpo
|
||||
3. Send protocol initialization.
|
||||
4. Confirm server capabilities/version.
|
||||
5. Register client-defined semantic tools if configured.
|
||||
6. Create or resume thread.
|
||||
7. emit `session.started` or `session.resumed`.
|
||||
8. Persist thread/session parameters in the normalized session binding.
|
||||
6. Disable automatic skill, app, collaboration, plugin, memory, and
|
||||
multi-agent instruction injection in thread config.
|
||||
7. Create or resume thread.
|
||||
8. emit `session.started` or `session.resumed`.
|
||||
9. Persist thread/session parameters in the normalized session binding.
|
||||
|
||||
### 14.4 Event mapping
|
||||
|
||||
@@ -2247,11 +2249,18 @@ Race behavior:
|
||||
Preferred order:
|
||||
|
||||
1. app-server structured output schema for terminal run result, when supported;
|
||||
2. client-defined `paperclip.finish` dynamic tool;
|
||||
2. client-defined `paperclip_finish` and `paperclip_block` dynamic tools;
|
||||
3. invalid/no result becomes `needs_review`, never inferred `done`.
|
||||
|
||||
The task envelope tells the model the expected result shape but does not teach Paperclip API mechanics.
|
||||
|
||||
Every provider-facing object schema is strict and every constant field declares
|
||||
both its JSON `type` and `const`. The finish tool accepts only `done` and
|
||||
`needs_review`; the block tool accepts only `blocked` with a typed blocker
|
||||
owner/action/reason/scope. Both normalize through one canonical
|
||||
`paperclip.run_result.v1` validator. The first valid result is committed,
|
||||
identical repeats are idempotent, and changed repeats are rejected.
|
||||
|
||||
### 14.8 Reconciliation
|
||||
|
||||
After runner or control-plane reconnect:
|
||||
@@ -6575,7 +6584,7 @@ Never change an active run from native to legacy mid-session.
|
||||
|
||||
The implementation phases, dependency policy, package-local documentation layout, human checkpoints, and retained future backlog now live in [`paperclip-native-runner-implementation-plan.md`](./paperclip-native-runner-implementation-plan.md).
|
||||
|
||||
The spike specification remains the normative contract. Keep it updated when implementation evidence changes architecture, protocol, security, persistence, API, UI, performance, or exit criteria. Keep sequencing and task-creation details in the implementation plan. Phase 0, Phase 1, Phase 2, and Phase 3 are complete, including the Phase 3 implementation, security, UX, QA, documentation, tutorial, journal, and human checkpoint gates. On 2026-08-08, the board authorized Phase 4. Phase 4 must implement the skillless Codex reference driver entirely inside `packages/paperclip-runner/` against the mock core, with no changes or imports from `server/`, `ui/`, `packages/db/`, or production control-plane behavior. Keep Phase 5 and later tasks deferred until the Phase 4 implementation, security review, CTO contract review, QA tutorial evidence, package-local documentation, OKF journal entry, and human checkpoint are complete.
|
||||
The spike specification remains the normative contract. Keep it updated when implementation evidence changes architecture, protocol, security, persistence, API, UI, performance, or exit criteria. Keep sequencing and task-creation details in the implementation plan. Phase 0, Phase 1, Phase 2, and Phase 3 are complete, including the Phase 3 implementation, security, UX, QA, documentation, tutorial, journal, and human checkpoint gates. On 2026-08-08, the board authorized Phase 4. The package-local Phase 4 driver, conformance suite, safe real-model tracer, documentation, tutorial, and OKF evidence are implemented without changes or imports from `server/`, `ui/`, `packages/db/`, or production control-plane behavior. Security review, CTO contract review, QA tutorial evidence, and the human checkpoint remain pending. Keep Phase 5 and later tasks uncreated until those gates complete.
|
||||
|
||||
## 30. Dependency sequencing
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env node
|
||||
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { createPhase4TaskEnvelope } from "../contracts/phase4.js";
|
||||
import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver.js";
|
||||
import { runPhase4CodexTracer } from "../mock-core/phase4-codex-runner.js";
|
||||
|
||||
interface CliOptions {
|
||||
workingDirectory?: string;
|
||||
output?: string;
|
||||
steer?: string;
|
||||
interrupt: boolean;
|
||||
json: boolean;
|
||||
}
|
||||
|
||||
function parseArgs(args: string[]): CliOptions {
|
||||
const options: CliOptions = { interrupt: false, json: false };
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const value = args[index];
|
||||
if (value === "--") continue;
|
||||
if (value === "--json") options.json = true;
|
||||
else if (value === "--interrupt") options.interrupt = true;
|
||||
else if (value === "--working-directory") options.workingDirectory = args[++index];
|
||||
else if (value === "--output") options.output = args[++index];
|
||||
else if (value === "--steer") options.steer = args[++index];
|
||||
else throw new Error(`Unknown argument: ${value}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function formatSummary(trace: Awaited<ReturnType<typeof runPhase4CodexTracer>>): string {
|
||||
const checks = Object.entries(trace.assertions)
|
||||
.map(([name, passed]) => `${passed ? "PASS" : "FAIL"} ${name}`)
|
||||
.join("\n");
|
||||
return [
|
||||
"Phase 4 skillless Codex tracer",
|
||||
`driver session: ${trace.metadata.identity.driverSessionId}`,
|
||||
`provider session: ${trace.metadata.identity.providerSessionId}`,
|
||||
`model: ${trace.context.model} (${trace.context.modelProvider})`,
|
||||
`events: ${trace.events.length}`,
|
||||
`result: ${trace.result.reportedWorkDisposition} — ${trace.result.summary}`,
|
||||
checks,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const workingDirectory = options.workingDirectory
|
||||
? resolve(options.workingDirectory)
|
||||
: await mkdtemp(join(tmpdir(), "paperclip-phase4-safe-"));
|
||||
await mkdir(workingDirectory, { recursive: true });
|
||||
const envelope = createPhase4TaskEnvelope({
|
||||
objective: "Create hello.txt in the working directory with exactly: hello from phase 4",
|
||||
criteria: [
|
||||
{ id: "file-created", requirement: "hello.txt exists in the supplied working directory." },
|
||||
{ id: "exact-content", requirement: "hello.txt contains exactly 'hello from phase 4'." },
|
||||
],
|
||||
constraints: [
|
||||
"Write only hello.txt inside the supplied working directory.",
|
||||
"Do not use network access.",
|
||||
"Do not discover or invoke skills.",
|
||||
"Do not call a control-plane API.",
|
||||
"Return one semantic completion result.",
|
||||
...(options.steer !== undefined || options.interrupt
|
||||
? ["Before changing files, run the shell command 'sleep 3' once so the operator can send a control command."]
|
||||
: []),
|
||||
],
|
||||
});
|
||||
const driver = new CodexAppServerDriver({
|
||||
taskEnvelope: envelope,
|
||||
onDiagnostic: (message) => process.stderr.write(`[codex app-server] ${message}\n`),
|
||||
});
|
||||
const trace = await runPhase4CodexTracer({
|
||||
driver,
|
||||
taskEnvelope: envelope,
|
||||
workingDirectory,
|
||||
steer: options.steer,
|
||||
interrupt: options.interrupt,
|
||||
});
|
||||
const serialized = `${JSON.stringify(trace, null, 2)}\n`;
|
||||
if (options.output) await writeFile(resolve(options.output), serialized, "utf8");
|
||||
process.stdout.write(options.json ? serialized : `${formatSummary(trace)}\nworkspace: ${workingDirectory}\n`);
|
||||
if (Object.values(trace.assertions).some((passed) => !passed)) process.exitCode = 1;
|
||||
}
|
||||
|
||||
void main().catch((error: unknown) => {
|
||||
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,13 +1,11 @@
|
||||
import type {
|
||||
NativeRunEvent,
|
||||
NativeSessionCapabilities,
|
||||
NativeUserMessage,
|
||||
} from "./types.js";
|
||||
import type { PrpEvent } from "../protocol/phase1-contract.js";
|
||||
import type { NativeSessionCapabilities, NativeUserMessage } from "./types.js";
|
||||
|
||||
export interface HarnessDriverDescriptor {
|
||||
kind: string;
|
||||
displayName: string;
|
||||
version: string;
|
||||
protocolVersion?: string;
|
||||
capabilities: NativeSessionCapabilities;
|
||||
}
|
||||
|
||||
@@ -20,6 +18,10 @@ export interface PersistedHarnessSession {
|
||||
driverKind: string;
|
||||
driverSessionId: string;
|
||||
providerSessionId?: string | null;
|
||||
runId?: string;
|
||||
normalizedSessionId?: string;
|
||||
activeTurnId?: string | null;
|
||||
lastSourceSequence?: number;
|
||||
}
|
||||
|
||||
export interface HarnessSessionRecoveryResult {
|
||||
@@ -34,10 +36,13 @@ export interface HarnessSession {
|
||||
providerSessionId?: string | null;
|
||||
displayId?: string | null;
|
||||
};
|
||||
events(): AsyncIterable<NativeRunEvent>;
|
||||
events(): AsyncIterable<PrpEvent>;
|
||||
startTurn(input: { message: NativeUserMessage }): Promise<{ turnId: string }>;
|
||||
steer?(input: { turnId: string; message: NativeUserMessage }): Promise<void>;
|
||||
interrupt?(input: { turnId: string; reason?: string }): Promise<void>;
|
||||
read?(): Promise<Record<string, unknown>>;
|
||||
reconcile?(): Promise<Record<string, unknown>>;
|
||||
usage?(): Promise<Record<string, unknown> | null>;
|
||||
snapshot(): Promise<PersistedHarnessSession>;
|
||||
close(input: { reason: string; force?: boolean }): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
import type {
|
||||
PrpCapabilities,
|
||||
PrpEvent,
|
||||
PrpStructuredRunResult,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
import type { SessionSnapshot } from "../reducer/session-reducer.js";
|
||||
|
||||
export const PHASE4_TASK_ENVELOPE_SCHEMA = "paperclip.skillless_task.v1" as const;
|
||||
export const PHASE4_CODEX_PROTOCOL_VERSION = "v2" as const;
|
||||
export const PHASE4_COMPLETION_TOOL_NAME = "paperclip_finish" as const;
|
||||
export const PHASE4_BLOCK_TOOL_NAME = "paperclip_block" as const;
|
||||
export const PHASE4_SEMANTIC_TOOL_NAMES = [
|
||||
PHASE4_COMPLETION_TOOL_NAME,
|
||||
PHASE4_BLOCK_TOOL_NAME,
|
||||
] as const;
|
||||
export const PHASE4_SKILLLESS_BASE_INSTRUCTIONS =
|
||||
"Complete only the supplied task envelope. Do not discover or invoke skills. Do not call a control-plane API. Return exactly one semantic completion result; use paperclip_finish when the work is done or needs review, and paperclip_block only when work cannot continue." as const;
|
||||
|
||||
export interface Phase4TaskEnvelope {
|
||||
schema: typeof PHASE4_TASK_ENVELOPE_SCHEMA;
|
||||
objective: string;
|
||||
completionContract: {
|
||||
revision: string;
|
||||
criteria: Array<{ id: string; requirement: string }>;
|
||||
};
|
||||
constraints: string[];
|
||||
expectedResultSchema: "paperclip.run_result.v1";
|
||||
}
|
||||
|
||||
export interface Phase4ModelContextSnapshot {
|
||||
protocolVersion: typeof PHASE4_CODEX_PROTOCOL_VERSION;
|
||||
codexVersion: string;
|
||||
clientInfo: {
|
||||
name: "paperclip-runner";
|
||||
title: "Paperclip Runner";
|
||||
version: string;
|
||||
};
|
||||
model: string;
|
||||
modelProvider: string;
|
||||
workingDirectory: string;
|
||||
sandbox: unknown;
|
||||
approvalPolicy: unknown;
|
||||
baseInstructions: typeof PHASE4_SKILLLESS_BASE_INSTRUCTIONS;
|
||||
instructionSources: string[];
|
||||
instructionPolicy: {
|
||||
skillInstructions: false;
|
||||
appInstructions: false;
|
||||
collaborationInstructions: false;
|
||||
};
|
||||
environmentKeys: string[];
|
||||
dynamicToolNames: string[];
|
||||
modelInputKinds: ["text"];
|
||||
envelope: Phase4TaskEnvelope;
|
||||
}
|
||||
|
||||
export interface Phase4RunMetadata {
|
||||
schema: "paperclip.runner.phase4.metadata.v1";
|
||||
fixtureName: string;
|
||||
identity: {
|
||||
schema: "paperclip.prp.identity.v1";
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
runId: string;
|
||||
environmentLeaseId: string;
|
||||
runnerInstanceId: string;
|
||||
normalizedSessionId: string;
|
||||
driverSessionId?: string;
|
||||
providerSessionId?: string;
|
||||
};
|
||||
capabilities: PrpCapabilities;
|
||||
}
|
||||
|
||||
export interface Phase4RunTrace {
|
||||
schema: "paperclip.runner.phase4.trace.v1";
|
||||
metadata: Phase4RunMetadata;
|
||||
context: Phase4ModelContextSnapshot;
|
||||
events: PrpEvent[];
|
||||
result: PrpStructuredRunResult;
|
||||
liveSnapshot: SessionSnapshot;
|
||||
replaySnapshot: SessionSnapshot;
|
||||
diagnostics: string[];
|
||||
assertions: {
|
||||
exactlyOneTerminalResult: boolean;
|
||||
liveReplayParity: boolean;
|
||||
stableIdentity: boolean;
|
||||
sourceSequenceContinuous: boolean;
|
||||
stableItemIdentity: boolean;
|
||||
contextIsSkillless: boolean;
|
||||
unrelatedSkillsAbsent: boolean;
|
||||
credentialsAbsent: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export function createPhase4TaskEnvelope(input: {
|
||||
objective: string;
|
||||
contractRevision?: string;
|
||||
criteria?: Array<{ id: string; requirement: string }>;
|
||||
constraints?: string[];
|
||||
}): Phase4TaskEnvelope {
|
||||
return {
|
||||
schema: PHASE4_TASK_ENVELOPE_SCHEMA,
|
||||
objective: input.objective,
|
||||
completionContract: {
|
||||
revision: input.contractRevision ?? "phase4-demo-v1",
|
||||
criteria:
|
||||
input.criteria ?? [{ id: "objective", requirement: "Complete the objective safely." }],
|
||||
},
|
||||
constraints: input.constraints ?? [
|
||||
"Work only inside the supplied working directory.",
|
||||
"Do not discover or invoke skills.",
|
||||
"Do not call a control-plane API.",
|
||||
"Return one semantic completion result.",
|
||||
],
|
||||
expectedResultSchema: "paperclip.run_result.v1",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict provider-facing subset of the wider PRP result schema.
|
||||
*
|
||||
* OpenAI structured outputs require every object to reject additional
|
||||
* properties. The accepted value is still validated against the canonical PRP
|
||||
* schema before it is committed as a semantic result.
|
||||
*/
|
||||
const completionClaimSchema = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["contractRevision", "objectiveSatisfied", "criteria", "remainingWork"],
|
||||
properties: {
|
||||
contractRevision: { type: "string", minLength: 1 },
|
||||
objectiveSatisfied: { type: "boolean" },
|
||||
criteria: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["criterionId", "status", "evidenceRefs"],
|
||||
properties: {
|
||||
criterionId: { type: "string", minLength: 1 },
|
||||
status: { enum: ["satisfied", "not_satisfied", "unknown"] },
|
||||
evidenceRefs: { type: "array", items: { type: "string" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
remainingWork: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["description", "blocksCompletion"],
|
||||
properties: {
|
||||
description: { type: "string", minLength: 1 },
|
||||
blocksCompletion: { type: "boolean" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const evidenceSchema = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["ref"],
|
||||
properties: { ref: { type: "string", minLength: 1 } },
|
||||
},
|
||||
} as const;
|
||||
|
||||
const verificationSchema = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["commandOrCheck", "status"],
|
||||
properties: {
|
||||
commandOrCheck: { type: "string", minLength: 1 },
|
||||
status: { enum: ["passed", "failed", "not_run"] },
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const attentionRequestsSchema = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["kind", "summary"],
|
||||
properties: {
|
||||
kind: { type: "string", minLength: 1 },
|
||||
summary: { type: "string", minLength: 1 },
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const artifactsSchema = {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["kind", "ref"],
|
||||
properties: {
|
||||
kind: { type: "string", minLength: 1 },
|
||||
ref: { type: "string", minLength: 1 },
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
const commonResultProperties = {
|
||||
schema: { type: "string", const: "paperclip.run_result.v1" },
|
||||
summary: { type: "string", minLength: 1 },
|
||||
completionClaim: completionClaimSchema,
|
||||
evidence: evidenceSchema,
|
||||
verification: verificationSchema,
|
||||
attentionRequests: attentionRequestsSchema,
|
||||
artifacts: artifactsSchema,
|
||||
} as const;
|
||||
|
||||
export const PHASE4_RESULT_OUTPUT_SCHEMA = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: [
|
||||
"schema",
|
||||
"reportedWorkDisposition",
|
||||
"summary",
|
||||
"completionClaim",
|
||||
"evidence",
|
||||
"verification",
|
||||
"attentionRequests",
|
||||
"artifacts",
|
||||
],
|
||||
properties: {
|
||||
...commonResultProperties,
|
||||
reportedWorkDisposition: { enum: ["done", "needs_review"] },
|
||||
},
|
||||
} as const;
|
||||
|
||||
export const PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA = {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: [
|
||||
"schema",
|
||||
"reportedWorkDisposition",
|
||||
"summary",
|
||||
"completionClaim",
|
||||
"evidence",
|
||||
"verification",
|
||||
"attentionRequests",
|
||||
"artifacts",
|
||||
"blocker",
|
||||
],
|
||||
properties: {
|
||||
...commonResultProperties,
|
||||
reportedWorkDisposition: { type: "string", const: "blocked" },
|
||||
blocker: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["reasonCode", "owner", "unblockAction", "scope"],
|
||||
properties: {
|
||||
reasonCode: { type: "string", minLength: 1 },
|
||||
owner: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["kind", "name"],
|
||||
properties: {
|
||||
kind: { enum: ["agent", "user", "system", "external"] },
|
||||
name: { type: "string", minLength: 1 },
|
||||
},
|
||||
},
|
||||
unblockAction: { type: "string", minLength: 1 },
|
||||
scope: { enum: ["current_track", "task_wide"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
@@ -29,6 +29,11 @@ export interface NativeSessionCapabilities {
|
||||
steering: boolean;
|
||||
interruption: boolean;
|
||||
structuredResult: boolean;
|
||||
read?: boolean;
|
||||
reconciliation?: boolean;
|
||||
usage?: boolean;
|
||||
dynamicTools?: boolean;
|
||||
unsupported?: string[];
|
||||
}
|
||||
|
||||
export interface NativeUserMessage {
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||
import { createInterface } from "node:readline";
|
||||
|
||||
export interface CodexRpcNotification {
|
||||
method: string;
|
||||
params: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface CodexRpcServerRequest extends CodexRpcNotification {
|
||||
id: string | number;
|
||||
}
|
||||
|
||||
export type CodexServerRequestHandler = (
|
||||
request: CodexRpcServerRequest,
|
||||
) => Promise<Record<string, unknown>>;
|
||||
|
||||
export interface CodexAppServerTransport {
|
||||
request(method: string, params: Record<string, unknown>): Promise<Record<string, unknown>>;
|
||||
notify(method: string, params?: Record<string, unknown>): void;
|
||||
notifications(): AsyncIterable<CodexRpcNotification>;
|
||||
setServerRequestHandler(handler: CodexServerRequestHandler): void;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
class AsyncQueue<T> implements AsyncIterable<T> {
|
||||
#values: T[] = [];
|
||||
#waiters: Array<(value: IteratorResult<T>) => void> = [];
|
||||
#closed = false;
|
||||
|
||||
push(value: T): void {
|
||||
if (this.#closed) return;
|
||||
const waiter = this.#waiters.shift();
|
||||
if (waiter === undefined) this.#values.push(value);
|
||||
else waiter({ value, done: false });
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.#closed = true;
|
||||
for (const waiter of this.#waiters.splice(0)) waiter({ value: undefined, done: true });
|
||||
}
|
||||
|
||||
[Symbol.asyncIterator](): AsyncIterator<T> {
|
||||
return {
|
||||
next: async () => {
|
||||
const value = this.#values.shift();
|
||||
if (value !== undefined) return { value, done: false };
|
||||
if (this.#closed) return { value: undefined, done: true };
|
||||
return new Promise<IteratorResult<T>>((resolve) => this.#waiters.push(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const SAFE_ENVIRONMENT_KEYS = [
|
||||
"ALL_PROXY",
|
||||
"CODEX_HOME",
|
||||
"HOME",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"LANG",
|
||||
"LC_ALL",
|
||||
"NO_PROXY",
|
||||
"NODE_EXTRA_CA_CERTS",
|
||||
"PATH",
|
||||
"PATHEXT",
|
||||
"SSL_CERT_FILE",
|
||||
"SystemRoot",
|
||||
"TEMP",
|
||||
"TMP",
|
||||
"TMPDIR",
|
||||
"WINDIR",
|
||||
] as const;
|
||||
|
||||
export function createSanitizedCodexEnvironment(
|
||||
source: NodeJS.ProcessEnv = process.env,
|
||||
): NodeJS.ProcessEnv {
|
||||
const environment: NodeJS.ProcessEnv = {};
|
||||
for (const key of SAFE_ENVIRONMENT_KEYS) {
|
||||
const value = source[key];
|
||||
if (value === undefined) continue;
|
||||
if (key.includes("PROXY") && proxyContainsCredentials(value)) continue;
|
||||
environment[key] = value;
|
||||
}
|
||||
return environment;
|
||||
}
|
||||
|
||||
export function sanitizedEnvironmentKeys(source: NodeJS.ProcessEnv = process.env): string[] {
|
||||
return Object.keys(createSanitizedCodexEnvironment(source)).sort();
|
||||
}
|
||||
|
||||
export function redactCodexDiagnostic(message: string): string {
|
||||
return message
|
||||
.replaceAll(/\u001b\[[0-?]*[ -/]*[@-~]/g, "")
|
||||
.replace(/Bearer\s+[A-Za-z0-9._~+\/-]+/gi, "Bearer [REDACTED]")
|
||||
.replace(/Basic\s+[A-Za-z0-9+/=]+/gi, "Basic [REDACTED]")
|
||||
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@")
|
||||
.replace(/([?&](?:api[_-]?key|token|secret|password)=)[^&#\s]+/gi, "$1[REDACTED]")
|
||||
.replace(/(["'](?:api[_-]?key|token|secret|password|authorization)["']\s*:\s*["'])[^"']+/gi, "$1[REDACTED]")
|
||||
.replace(/(api[_-]?key|token|secret|password)\s*[=:]\s*[^\s,;]+/gi, "$1=[REDACTED]")
|
||||
.replace(/(PAPERCLIP_API_KEY|OPENAI_API_KEY)=[^\s]+/g, "$1=[REDACTED]");
|
||||
}
|
||||
|
||||
function proxyContainsCredentials(value: string): boolean {
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.username.length > 0 || url.password.length > 0;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
interface PendingRequest {
|
||||
resolve: (value: Record<string, unknown>) => void;
|
||||
reject: (error: Error) => void;
|
||||
}
|
||||
|
||||
export interface ProcessCodexTransportOptions {
|
||||
command?: string;
|
||||
args?: string[];
|
||||
environment?: NodeJS.ProcessEnv;
|
||||
onDiagnostic?: (message: string) => void;
|
||||
}
|
||||
|
||||
/** Local stdio JSON-RPC transport. App-server never becomes a WAN endpoint. */
|
||||
export class ProcessCodexAppServerTransport implements CodexAppServerTransport {
|
||||
#process: ChildProcessWithoutNullStreams;
|
||||
#notifications = new AsyncQueue<CodexRpcNotification>();
|
||||
#pending = new Map<number, PendingRequest>();
|
||||
#nextId = 1;
|
||||
#serverRequestHandler: CodexServerRequestHandler = async () => ({
|
||||
success: false,
|
||||
contentItems: [{ type: "input_text", text: "Unsupported client request." }],
|
||||
});
|
||||
#closed = false;
|
||||
#onDiagnostic?: (message: string) => void;
|
||||
|
||||
constructor(options: ProcessCodexTransportOptions = {}) {
|
||||
this.#onDiagnostic = options.onDiagnostic;
|
||||
this.#process = spawn(options.command ?? "codex", options.args ?? ["app-server"], {
|
||||
env: options.environment ?? createSanitizedCodexEnvironment(),
|
||||
stdio: "pipe",
|
||||
});
|
||||
createInterface({ input: this.#process.stdout }).on("line", (line) => this.#onLine(line));
|
||||
createInterface({ input: this.#process.stderr }).on("line", (line) => {
|
||||
this.#onDiagnostic?.(redactCodexDiagnostic(line));
|
||||
});
|
||||
this.#process.on("error", (error) => this.#failAll(error));
|
||||
this.#process.on("exit", (code, signal) => {
|
||||
if (!this.#closed) {
|
||||
this.#failAll(new Error(`codex app-server exited (code=${code ?? "null"}, signal=${signal ?? "none"})`));
|
||||
}
|
||||
this.#notifications.close();
|
||||
});
|
||||
}
|
||||
|
||||
request(method: string, params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
const id = this.#nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
this.#pending.set(id, { resolve, reject });
|
||||
try {
|
||||
this.#write({ method, id, params });
|
||||
} catch (error) {
|
||||
this.#pending.delete(id);
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
notify(method: string, params?: Record<string, unknown>): void {
|
||||
this.#write(params === undefined ? { method } : { method, params });
|
||||
}
|
||||
|
||||
notifications(): AsyncIterable<CodexRpcNotification> {
|
||||
return this.#notifications;
|
||||
}
|
||||
|
||||
setServerRequestHandler(handler: CodexServerRequestHandler): void {
|
||||
this.#serverRequestHandler = handler;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
this.#process.kill("SIGTERM");
|
||||
this.#notifications.close();
|
||||
this.#failAll(new Error("codex app-server transport closed"));
|
||||
}
|
||||
|
||||
#write(message: unknown): void {
|
||||
if (this.#closed) throw new Error("codex app-server transport is closed");
|
||||
this.#process.stdin.write(`${JSON.stringify(message)}\n`);
|
||||
}
|
||||
|
||||
#onLine(line: string): void {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(line);
|
||||
} catch {
|
||||
this.#onDiagnostic?.("codex app-server emitted malformed JSON");
|
||||
return;
|
||||
}
|
||||
if (typeof value !== "object" || value === null) return;
|
||||
const message = value as Record<string, unknown>;
|
||||
if (typeof message.id === "number" && ("result" in message || "error" in message)) {
|
||||
const pending = this.#pending.get(message.id);
|
||||
if (pending === undefined) return;
|
||||
this.#pending.delete(message.id);
|
||||
if (message.error !== undefined) {
|
||||
pending.reject(new Error(redactCodexDiagnostic(JSON.stringify(message.error))));
|
||||
} else {
|
||||
pending.resolve(asRecord(message.result));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (typeof message.method !== "string") return;
|
||||
const params = asRecord(message.params);
|
||||
if ((typeof message.id === "string" || typeof message.id === "number") && "id" in message) {
|
||||
const request: CodexRpcServerRequest = { id: message.id, method: message.method, params };
|
||||
void this.#serverRequestHandler(request).then(
|
||||
(result) => this.#write({ id: request.id, result }),
|
||||
(error: unknown) =>
|
||||
this.#write({
|
||||
id: request.id,
|
||||
error: { code: -32_000, message: redactCodexDiagnostic(String(error)) },
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
this.#notifications.push({ method: message.method, params });
|
||||
}
|
||||
|
||||
#failAll(error: Error): void {
|
||||
for (const pending of this.#pending.values()) pending.reject(error);
|
||||
this.#pending.clear();
|
||||
}
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||
PHASE4_RESULT_OUTPUT_SCHEMA,
|
||||
createPhase4TaskEnvelope,
|
||||
} from "../../contracts/phase4.js";
|
||||
import {
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata,
|
||||
} from "../../reducer/session-reducer.js";
|
||||
import {
|
||||
validatePrpEvent,
|
||||
type PrpCapabilities,
|
||||
type PrpEvent,
|
||||
type PrpStructuredRunResult,
|
||||
} from "../../protocol/phase1-contract.js";
|
||||
import {
|
||||
CodexAppServerDriver,
|
||||
UnsupportedCodexOperationError,
|
||||
isSkilllessContext,
|
||||
} from "./codex-app-server-driver.js";
|
||||
import type {
|
||||
CodexAppServerTransport,
|
||||
CodexRpcNotification,
|
||||
CodexRpcServerRequest,
|
||||
CodexServerRequestHandler,
|
||||
} from "./app-server-transport.js";
|
||||
|
||||
class TestQueue<T> implements AsyncIterable<T> {
|
||||
values: T[] = [];
|
||||
waiters: Array<(value: IteratorResult<T>) => void> = [];
|
||||
closed = false;
|
||||
|
||||
push(value: T): void {
|
||||
const waiter = this.waiters.shift();
|
||||
if (waiter) waiter({ value, done: false });
|
||||
else this.values.push(value);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.closed = true;
|
||||
for (const waiter of this.waiters.splice(0)) waiter({ value: undefined, done: true });
|
||||
}
|
||||
|
||||
[Symbol.asyncIterator](): AsyncIterator<T> {
|
||||
return {
|
||||
next: async () => {
|
||||
const value = this.values.shift();
|
||||
if (value) return { value, done: false };
|
||||
if (this.closed) return { value: undefined, done: true };
|
||||
return new Promise((resolve) => this.waiters.push(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
class FakeCodexTransport implements CodexAppServerTransport {
|
||||
readonly calls: Array<{ method: string; params: Record<string, unknown> }> = [];
|
||||
readonly sentNotifications: Array<{ method: string; params?: Record<string, unknown> }> = [];
|
||||
readonly queue = new TestQueue<CodexRpcNotification>();
|
||||
handler: CodexServerRequestHandler = async () => ({});
|
||||
rejectMethods = new Map<string, Error>();
|
||||
|
||||
constructor(
|
||||
readonly threadId = "thread-1",
|
||||
readonly providerSessionId = "provider-session-1",
|
||||
) {}
|
||||
|
||||
async request(method: string, params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
this.calls.push({ method, params: structuredClone(params) });
|
||||
const rejection = this.rejectMethods.get(method);
|
||||
if (rejection) throw rejection;
|
||||
if (method === "initialize") {
|
||||
return { userAgent: "codex-cli/0.132.0", codexHome: "/isolated/codex", platformFamily: "unix", platformOs: "linux" };
|
||||
}
|
||||
if (method === "thread/start" || method === "thread/resume") {
|
||||
return {
|
||||
thread: { id: this.threadId, sessionId: this.providerSessionId, modelProvider: "openai", cwd: "/workspace", turns: [] },
|
||||
model: "gpt-test",
|
||||
modelProvider: "openai",
|
||||
cwd: "/workspace",
|
||||
sandbox: { type: "workspaceWrite" },
|
||||
approvalPolicy: "never",
|
||||
instructionSources: [],
|
||||
};
|
||||
}
|
||||
if (method === "turn/start") return { turn: { id: "turn-1", status: "inProgress", items: [] } };
|
||||
if (method === "thread/read") {
|
||||
return { thread: { id: this.threadId, sessionId: this.providerSessionId, turns: [{ id: "turn-1", status: "inProgress", items: [] }] } };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
notify(method: string, params?: Record<string, unknown>): void {
|
||||
this.sentNotifications.push(params === undefined ? { method } : { method, params });
|
||||
}
|
||||
|
||||
notifications(): AsyncIterable<CodexRpcNotification> {
|
||||
return this.queue;
|
||||
}
|
||||
|
||||
setServerRequestHandler(handler: CodexServerRequestHandler): void {
|
||||
this.handler = handler;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.queue.close();
|
||||
}
|
||||
|
||||
push(method: string, params: Record<string, unknown>): void {
|
||||
this.queue.push({ method, params });
|
||||
}
|
||||
|
||||
invoke(request: CodexRpcServerRequest): Promise<Record<string, unknown>> {
|
||||
return this.handler(request);
|
||||
}
|
||||
}
|
||||
|
||||
const envelope = createPhase4TaskEnvelope({
|
||||
objective: "Create hello.txt with the text hello.",
|
||||
criteria: [{ id: "file", requirement: "hello.txt contains hello" }],
|
||||
});
|
||||
|
||||
const result: PrpStructuredRunResult = {
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "done",
|
||||
summary: "Created hello.txt.",
|
||||
completionClaim: {
|
||||
contractRevision: "phase4-demo-v1",
|
||||
objectiveSatisfied: true,
|
||||
criteria: [{ criterionId: "file", status: "satisfied", evidenceRefs: ["hello.txt"] }],
|
||||
remainingWork: [],
|
||||
},
|
||||
evidence: [{ ref: "hello.txt" }],
|
||||
verification: [{ commandOrCheck: "read hello.txt", status: "passed" }],
|
||||
attentionRequests: [],
|
||||
artifacts: [{ kind: "file", ref: "hello.txt" }],
|
||||
};
|
||||
|
||||
function makeDriver(transports: FakeCodexTransport[], options: Record<string, unknown> = {}) {
|
||||
let index = 0;
|
||||
return new CodexAppServerDriver({
|
||||
taskEnvelope: envelope,
|
||||
environment: {
|
||||
PATH: "/bin",
|
||||
HOME: "/isolated/home",
|
||||
CODEX_HOME: "/isolated/codex",
|
||||
LANG: "C.UTF-8",
|
||||
PAPERCLIP_API_KEY: "must-not-pass",
|
||||
RANDOM_SKILL_PATH: "/skills/unrelated",
|
||||
},
|
||||
now: () => new Date("2026-08-08T12:00:00.000Z"),
|
||||
transportFactory: () => transports[index++]!,
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
async function collectUntilTerminal(events: AsyncIterable<PrpEvent>): Promise<PrpEvent[]> {
|
||||
const collected: PrpEvent[] = [];
|
||||
for await (const event of events) {
|
||||
collected.push(event);
|
||||
if (event.eventType === "run.terminal") break;
|
||||
}
|
||||
return collected;
|
||||
}
|
||||
|
||||
describe("Codex app-server Phase 4 driver", () => {
|
||||
it("passes the common typed-event contract and produces exactly one terminal result", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const driver = makeDriver([transport]);
|
||||
const descriptor = await driver.descriptor();
|
||||
const session = await driver.openSession({ runId: "run-1", workingDirectory: "/workspace" });
|
||||
const turn = await session.startTurn({ message: { role: "user", text: "Do the safe task." } });
|
||||
|
||||
transport.push("turn/started", { threadId: "thread-1", turn: { id: turn.turnId, status: "inProgress" } });
|
||||
transport.push("item/started", { threadId: "thread-1", turnId: turn.turnId, item: { id: "cmd-1", type: "commandExecution", command: "printf hello" } });
|
||||
transport.push("item/commandExecution/outputDelta", { threadId: "thread-1", turnId: turn.turnId, itemId: "cmd-1", delta: "hello" });
|
||||
transport.push("item/completed", { threadId: "thread-1", turnId: turn.turnId, item: { id: "file-1", type: "fileChange", changes: [{ path: "hello.txt" }] } });
|
||||
expect(await transport.invoke({
|
||||
id: "request-1",
|
||||
method: "item/tool/requestUserInput",
|
||||
params: { threadId: "thread-1", turnId: turn.turnId, itemId: "question-1" },
|
||||
})).toEqual({ answers: {} });
|
||||
transport.push("thread/tokenUsage/updated", { threadId: "thread-1", turnId: turn.turnId, tokenUsage: { total: { inputTokens: 10, outputTokens: 4 }, modelContextWindow: 128000 } });
|
||||
transport.push("item/completed", { threadId: "thread-1", turnId: turn.turnId, item: { id: "answer-1", type: "agentMessage", text: JSON.stringify(result) } });
|
||||
transport.push("turn/completed", { threadId: "thread-1", turn: { id: turn.turnId, status: "completed", items: [] } });
|
||||
|
||||
const events = await collectUntilTerminal(session.events());
|
||||
expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true);
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1);
|
||||
expect(events.map((event) => event.eventType)).toEqual(expect.arrayContaining([
|
||||
"session.started", "turn.started", "item.started", "item.delta", "item.completed", "run.result.proposed", "run.terminal",
|
||||
"runtime_request.created", "runtime_request.resolved",
|
||||
]));
|
||||
|
||||
const capabilities: PrpCapabilities = {
|
||||
schema: "paperclip.prp.capabilities.v1",
|
||||
sessionReusePolicy: "reuse_per_issue",
|
||||
driver: { kind: descriptor.kind, version: descriptor.version },
|
||||
steer: true,
|
||||
interrupt: true,
|
||||
resume: true,
|
||||
runtimeRequests: true,
|
||||
structuredResult: true,
|
||||
typedEvents: true,
|
||||
};
|
||||
const metadata = {
|
||||
fixtureName: "phase4-conformance",
|
||||
identity: { schema: "paperclip.prp.identity.v1" as const, companyId: "company-1", issueId: "issue-1", runId: "run-1", environmentLeaseId: "lease-1", runnerInstanceId: "runner-phase4", normalizedSessionId: "session:run-1" },
|
||||
capabilities,
|
||||
};
|
||||
const live = events.reduce(applyPrpEvent, createSessionSnapshotFromMetadata(metadata));
|
||||
const replay = events.reduce(applyPrpEvent, createSessionSnapshotFromMetadata(metadata));
|
||||
expect(live).toEqual(replay);
|
||||
expect(live.integrity).toBe("complete");
|
||||
expect(await session.usage?.()).toMatchObject({ modelContextWindow: 128000 });
|
||||
});
|
||||
|
||||
it("captures an exact skillless model/environment snapshot with credentials absent", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-context", workingDirectory: "/workspace" });
|
||||
const iterator = session.events()[Symbol.asyncIterator]();
|
||||
const first = await iterator.next();
|
||||
const context = first.value?.payload.context as Parameters<typeof isSkilllessContext>[0];
|
||||
expect(context).toMatchObject({
|
||||
codexVersion: "codex-cli/0.132.0",
|
||||
model: "gpt-test",
|
||||
modelProvider: "openai",
|
||||
workingDirectory: "/workspace",
|
||||
approvalPolicy: "never",
|
||||
instructionSources: [],
|
||||
instructionPolicy: {
|
||||
skillInstructions: false,
|
||||
appInstructions: false,
|
||||
collaborationInstructions: false,
|
||||
},
|
||||
environmentKeys: ["CODEX_HOME", "HOME", "LANG", "PATH"],
|
||||
envelope,
|
||||
});
|
||||
expect(isSkilllessContext(context)).toBe(true);
|
||||
expect(JSON.stringify(context)).not.toContain("must-not-pass");
|
||||
expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH");
|
||||
expect(transport.calls.find((call) => call.method === "thread/start")?.params).toMatchObject({
|
||||
approvalPolicy: "never",
|
||||
config: {
|
||||
"skills.include_instructions": false,
|
||||
include_apps_instructions: false,
|
||||
include_collaboration_mode_instructions: false,
|
||||
},
|
||||
dynamicTools: [{ name: "paperclip_finish" }, { name: "paperclip_block" }],
|
||||
});
|
||||
expect(PHASE4_RESULT_OUTPUT_SCHEMA.properties.schema).toEqual({
|
||||
type: "string",
|
||||
const: "paperclip.run_result.v1",
|
||||
});
|
||||
expect(PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA.properties.reportedWorkDisposition).toEqual({
|
||||
type: "string",
|
||||
const: "blocked",
|
||||
});
|
||||
});
|
||||
|
||||
it("steers and interrupts an active turn without replacing the session", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-controls", workingDirectory: "/workspace" });
|
||||
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
|
||||
await session.steer?.({ turnId, message: { role: "user", text: "Use a shorter answer." } });
|
||||
await session.interrupt?.({ turnId, reason: "operator requested" });
|
||||
expect(transport.calls.map((call) => call.method)).toEqual([
|
||||
"initialize", "thread/start", "turn/start", "turn/steer", "turn/interrupt",
|
||||
]);
|
||||
expect(session.ids()).toEqual({ driverSessionId: "thread-1", providerSessionId: "provider-session-1", displayId: "thread-1" });
|
||||
});
|
||||
|
||||
it("makes duplicate semantic completion idempotent and rejects changed payloads", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-result", workingDirectory: "/workspace" });
|
||||
await session.startTurn({ message: { role: "user", text: "Complete." } });
|
||||
const request = { id: 1, method: "item/tool/call", params: { threadId: "thread-1", turnId: "turn-1", callId: "call-1", tool: "paperclip_finish", arguments: result } };
|
||||
expect(await transport.invoke(request)).toMatchObject({ success: true });
|
||||
expect(await transport.invoke({ ...request, id: 2 })).toMatchObject({ success: true });
|
||||
const changed = structuredClone(result);
|
||||
changed.summary = "Changed after commit.";
|
||||
expect(await transport.invoke({ ...request, id: 3, params: { ...request.params, arguments: changed } })).toMatchObject({ success: false });
|
||||
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
|
||||
const events = await collectUntilTerminal(session.events());
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("normalizes finish and block tools into one canonical result contract", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-tools", workingDirectory: "/workspace" });
|
||||
await session.startTurn({ message: { role: "user", text: "Complete or block." } });
|
||||
const blocked: PrpStructuredRunResult = {
|
||||
...structuredClone(result),
|
||||
reportedWorkDisposition: "blocked",
|
||||
summary: "Waiting on a fixture owner.",
|
||||
completionClaim: {
|
||||
...structuredClone(result.completionClaim),
|
||||
objectiveSatisfied: false,
|
||||
criteria: [{ criterionId: "file", status: "not_satisfied", evidenceRefs: [] }],
|
||||
remainingWork: [{ description: "Fixture owner must provide input.", blocksCompletion: true }],
|
||||
},
|
||||
blocker: {
|
||||
reasonCode: "fixture_input_missing",
|
||||
owner: { kind: "external", name: "fixture owner" },
|
||||
unblockAction: "Provide the fixture input.",
|
||||
scope: "task_wide",
|
||||
},
|
||||
artifacts: [],
|
||||
};
|
||||
const request = {
|
||||
id: 10,
|
||||
method: "item/tool/call",
|
||||
params: {
|
||||
threadId: "thread-1",
|
||||
turnId: "turn-1",
|
||||
callId: "call-block",
|
||||
tool: "paperclip_block",
|
||||
arguments: blocked,
|
||||
},
|
||||
};
|
||||
expect(await transport.invoke({
|
||||
...request,
|
||||
id: 9,
|
||||
params: { ...request.params, tool: "paperclip_finish" },
|
||||
})).toMatchObject({ success: false });
|
||||
expect(await transport.invoke(request)).toMatchObject({ success: true });
|
||||
expect(await transport.invoke({ ...request, id: 11 })).toMatchObject({ success: true });
|
||||
transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } });
|
||||
const events = await collectUntilTerminal(session.events());
|
||||
expect(events.filter((event) => event.eventType === "run.result.proposed")).toHaveLength(1);
|
||||
expect(events.find((event) => event.eventType === "run.result.proposed")?.payload)
|
||||
.toMatchObject({ reportedWorkDisposition: "blocked" });
|
||||
});
|
||||
|
||||
it("resumes and reconciles the exact provider thread after transport loss", async () => {
|
||||
const first = new FakeCodexTransport();
|
||||
const second = new FakeCodexTransport();
|
||||
const driver = makeDriver([first, second]);
|
||||
const original = await driver.openSession({ runId: "run-recover", workingDirectory: "/workspace" });
|
||||
await original.startTurn({ message: { role: "user", text: "Work." } });
|
||||
const snapshot = await original.snapshot();
|
||||
await original.close({ reason: "transport lost" });
|
||||
const recovery = await driver.recoverSession?.(snapshot);
|
||||
expect(recovery).toMatchObject({ recovered: true });
|
||||
expect(recovery?.session?.ids()).toEqual(original.ids());
|
||||
expect(await recovery?.session?.reconcile?.()).toMatchObject({ thread: { id: "thread-1" } });
|
||||
expect(second.calls.map((call) => call.method)).toEqual(["initialize", "thread/resume", "thread/read"]);
|
||||
expect((await recovery?.session?.snapshot())?.activeTurnId).toBe("turn-1");
|
||||
});
|
||||
|
||||
it("degrades unsupported operations with explicit redacted diagnostics", async () => {
|
||||
const transport = new FakeCodexTransport();
|
||||
transport.rejectMethods.set("turn/steer", new Error("Bearer super-secret api_key=also-secret"));
|
||||
const session = await makeDriver([transport]).openSession({ runId: "run-degrade", workingDirectory: "/workspace" });
|
||||
const { turnId } = await session.startTurn({ message: { role: "user", text: "Start." } });
|
||||
await expect(session.steer?.({ turnId, message: { role: "user", text: "Steer." } })).rejects.toBeInstanceOf(UnsupportedCodexOperationError);
|
||||
const iterator = session.events()[Symbol.asyncIterator]();
|
||||
const events: PrpEvent[] = [];
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const next = await iterator.next();
|
||||
if (next.value) events.push(next.value);
|
||||
}
|
||||
const diagnostic = events.find((event) => event.eventType === "harness.diagnostic");
|
||||
expect(diagnostic?.payload).toMatchObject({ code: "unsupported_operation", operation: "steering" });
|
||||
expect(JSON.stringify(diagnostic)).not.toContain("super-secret");
|
||||
expect(JSON.stringify(diagnostic)).not.toContain("also-secret");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,873 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import type {
|
||||
HarnessDriver,
|
||||
HarnessDriverDescriptor,
|
||||
HarnessSession,
|
||||
HarnessSessionRecoveryResult,
|
||||
OpenHarnessSessionInput,
|
||||
PersistedHarnessSession,
|
||||
} from "../../contracts/harness-driver.js";
|
||||
import {
|
||||
PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||
PHASE4_BLOCK_TOOL_NAME,
|
||||
PHASE4_CODEX_PROTOCOL_VERSION,
|
||||
PHASE4_COMPLETION_TOOL_NAME,
|
||||
PHASE4_RESULT_OUTPUT_SCHEMA,
|
||||
PHASE4_SEMANTIC_TOOL_NAMES,
|
||||
PHASE4_SKILLLESS_BASE_INSTRUCTIONS,
|
||||
type Phase4ModelContextSnapshot,
|
||||
type Phase4TaskEnvelope,
|
||||
} from "../../contracts/phase4.js";
|
||||
import {
|
||||
validatePrpStructuredRunResult,
|
||||
type PrpEvent,
|
||||
type PrpStructuredRunResult,
|
||||
} from "../../protocol/phase1-contract.js";
|
||||
import type { NativeUserMessage } from "../../contracts/types.js";
|
||||
import {
|
||||
ProcessCodexAppServerTransport,
|
||||
createSanitizedCodexEnvironment,
|
||||
redactCodexDiagnostic,
|
||||
sanitizedEnvironmentKeys,
|
||||
type CodexAppServerTransport,
|
||||
type CodexRpcNotification,
|
||||
type CodexRpcServerRequest,
|
||||
} from "./app-server-transport.js";
|
||||
|
||||
const DRIVER_KIND = "codex_app_server";
|
||||
const DRIVER_VERSION = "phase4-v1";
|
||||
|
||||
class AsyncQueue<T> implements AsyncIterable<T> {
|
||||
#values: T[] = [];
|
||||
#waiters: Array<(value: IteratorResult<T>) => void> = [];
|
||||
#closed = false;
|
||||
|
||||
push(value: T): void {
|
||||
if (this.#closed) return;
|
||||
const waiter = this.#waiters.shift();
|
||||
if (waiter === undefined) this.#values.push(value);
|
||||
else waiter({ value, done: false });
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
for (const waiter of this.#waiters.splice(0)) waiter({ value: undefined, done: true });
|
||||
}
|
||||
|
||||
[Symbol.asyncIterator](): AsyncIterator<T> {
|
||||
return {
|
||||
next: async () => {
|
||||
const value = this.#values.shift();
|
||||
if (value !== undefined) return { value, done: false };
|
||||
if (this.#closed) return { value: undefined, done: true };
|
||||
return new Promise((resolve) => this.#waiters.push(resolve));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export class UnsupportedCodexOperationError extends Error {
|
||||
readonly operation: string;
|
||||
|
||||
constructor(operation: string, detail: string) {
|
||||
super(`${operation} is unsupported: ${redactCodexDiagnostic(detail)}`);
|
||||
this.name = "UnsupportedCodexOperationError";
|
||||
this.operation = operation;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CodexAppServerDriverOptions {
|
||||
taskEnvelope: Phase4TaskEnvelope;
|
||||
transportFactory?: () => CodexAppServerTransport;
|
||||
environment?: NodeJS.ProcessEnv;
|
||||
now?: () => Date;
|
||||
runnerInstanceId?: string;
|
||||
onDiagnostic?: (message: string) => void;
|
||||
capabilities?: Partial<{
|
||||
resume: boolean;
|
||||
read: boolean;
|
||||
steering: boolean;
|
||||
interruption: boolean;
|
||||
usage: boolean;
|
||||
reconciliation: boolean;
|
||||
dynamicTools: boolean;
|
||||
}>;
|
||||
}
|
||||
|
||||
type CodexCapabilities = Required<
|
||||
NonNullable<CodexAppServerDriverOptions["capabilities"]>
|
||||
>;
|
||||
|
||||
interface OpenedCodexThread {
|
||||
threadId: string;
|
||||
providerSessionId: string;
|
||||
context: Phase4ModelContextSnapshot;
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function text(value: unknown, fallback = ""): string {
|
||||
return typeof value === "string" ? value : fallback;
|
||||
}
|
||||
|
||||
function itemFromParams(params: Record<string, unknown>): Record<string, unknown> {
|
||||
return record(params.item);
|
||||
}
|
||||
|
||||
function itemText(item: Record<string, unknown>): string {
|
||||
for (const key of ["text", "aggregatedOutput", "patch", "delta"]) {
|
||||
if (typeof item[key] === "string") return item[key];
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function userInput(message: NativeUserMessage): Record<string, unknown> {
|
||||
return { type: "text", text: message.text, text_elements: [] };
|
||||
}
|
||||
|
||||
function terminalState(status: string): "completed" | "failed" | "interrupted" {
|
||||
if (status === "failed") return "failed";
|
||||
if (status === "interrupted") return "interrupted";
|
||||
return "completed";
|
||||
}
|
||||
|
||||
function dispositionRunState(
|
||||
result: PrpStructuredRunResult,
|
||||
): "succeeded" | "failed" {
|
||||
return result.reportedWorkDisposition === "done" ? "succeeded" : "failed";
|
||||
}
|
||||
|
||||
function tryParseResult(value: unknown): PrpStructuredRunResult | null {
|
||||
let candidate = value;
|
||||
if (typeof value === "string") {
|
||||
try {
|
||||
candidate = JSON.parse(value);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
const validation = validatePrpStructuredRunResult(candidate);
|
||||
return validation.ok ? validation.result : null;
|
||||
}
|
||||
|
||||
function finishToolSpec(): Record<string, unknown> {
|
||||
return {
|
||||
name: PHASE4_COMPLETION_TOOL_NAME,
|
||||
description: "Return the one semantic completion result for this task.",
|
||||
inputSchema: PHASE4_RESULT_OUTPUT_SCHEMA,
|
||||
};
|
||||
}
|
||||
|
||||
function blockToolSpec(): Record<string, unknown> {
|
||||
return {
|
||||
name: PHASE4_BLOCK_TOOL_NAME,
|
||||
description: "Return the one semantic result when the task cannot continue.",
|
||||
inputSchema: PHASE4_BLOCK_RESULT_OUTPUT_SCHEMA,
|
||||
};
|
||||
}
|
||||
|
||||
const SKILLLESS_THREAD_CONFIG = {
|
||||
"skills.include_instructions": false,
|
||||
include_apps_instructions: false,
|
||||
include_collaboration_mode_instructions: false,
|
||||
"features.apps": false,
|
||||
"features.plugins": false,
|
||||
"features.multi_agent": false,
|
||||
"features.memories": false,
|
||||
} as const;
|
||||
|
||||
export class CodexAppServerDriver implements HarnessDriver {
|
||||
readonly #options: CodexAppServerDriverOptions;
|
||||
readonly #caps: CodexCapabilities;
|
||||
|
||||
constructor(options: CodexAppServerDriverOptions) {
|
||||
this.#options = options;
|
||||
this.#caps = {
|
||||
resume: true,
|
||||
read: true,
|
||||
steering: true,
|
||||
interruption: true,
|
||||
usage: true,
|
||||
reconciliation: true,
|
||||
dynamicTools: true,
|
||||
...options.capabilities,
|
||||
};
|
||||
if (!this.#caps.read) this.#caps.reconciliation = false;
|
||||
}
|
||||
|
||||
async descriptor(): Promise<HarnessDriverDescriptor> {
|
||||
const unsupported = Object.entries(this.#caps)
|
||||
.filter(([, supported]) => !supported)
|
||||
.map(([operation]) => operation);
|
||||
return {
|
||||
kind: DRIVER_KIND,
|
||||
displayName: "Codex app-server",
|
||||
version: DRIVER_VERSION,
|
||||
protocolVersion: PHASE4_CODEX_PROTOCOL_VERSION,
|
||||
capabilities: {
|
||||
resume: this.#caps.resume,
|
||||
typedEvents: true,
|
||||
steering: this.#caps.steering,
|
||||
interruption: this.#caps.interruption,
|
||||
structuredResult: true,
|
||||
read: this.#caps.read,
|
||||
reconciliation: this.#caps.reconciliation,
|
||||
usage: this.#caps.usage,
|
||||
dynamicTools: this.#caps.dynamicTools,
|
||||
unsupported,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async openSession(input: OpenHarnessSessionInput): Promise<HarnessSession> {
|
||||
const transport = this.#transport();
|
||||
const initialize = await this.#initialize(transport);
|
||||
const response = await transport.request("thread/start", {
|
||||
cwd: input.workingDirectory,
|
||||
approvalPolicy: "never",
|
||||
sandbox: "workspace-write",
|
||||
config: SKILLLESS_THREAD_CONFIG,
|
||||
baseInstructions: PHASE4_SKILLLESS_BASE_INSTRUCTIONS,
|
||||
dynamicTools: this.#caps.dynamicTools ? [finishToolSpec(), blockToolSpec()] : [],
|
||||
experimentalRawEvents: false,
|
||||
persistExtendedHistory: false,
|
||||
});
|
||||
const opened = this.#openedThread(response, initialize, input.workingDirectory);
|
||||
return this.#session({
|
||||
transport,
|
||||
runId: input.runId,
|
||||
normalizedSessionId: `session:${input.runId}`,
|
||||
opened,
|
||||
resumed: false,
|
||||
sourceSequence: 0,
|
||||
});
|
||||
}
|
||||
|
||||
async recoverSession(
|
||||
snapshot: PersistedHarnessSession,
|
||||
): Promise<HarnessSessionRecoveryResult> {
|
||||
if (!this.#caps.resume) {
|
||||
return { recovered: false, reason: "resume capability is unavailable" };
|
||||
}
|
||||
if (!snapshot.runId || !snapshot.normalizedSessionId || !snapshot.driverSessionId) {
|
||||
return { recovered: false, reason: "persisted session identity is incomplete" };
|
||||
}
|
||||
const transport = this.#transport();
|
||||
try {
|
||||
const initialize = await this.#initialize(transport);
|
||||
const response = await transport.request("thread/resume", {
|
||||
threadId: snapshot.driverSessionId,
|
||||
config: SKILLLESS_THREAD_CONFIG,
|
||||
baseInstructions: PHASE4_SKILLLESS_BASE_INSTRUCTIONS,
|
||||
persistExtendedHistory: false,
|
||||
});
|
||||
const opened = this.#openedThread(response, initialize, text(record(response.thread).cwd));
|
||||
if (opened.threadId !== snapshot.driverSessionId) {
|
||||
await transport.close();
|
||||
return { recovered: false, reason: "provider resumed a different session" };
|
||||
}
|
||||
return {
|
||||
recovered: true,
|
||||
session: this.#session({
|
||||
transport,
|
||||
runId: snapshot.runId,
|
||||
normalizedSessionId: snapshot.normalizedSessionId,
|
||||
opened,
|
||||
resumed: true,
|
||||
activeTurnId: snapshot.activeTurnId ?? null,
|
||||
sourceSequence: snapshot.lastSourceSequence ?? 0,
|
||||
}),
|
||||
};
|
||||
} catch (error) {
|
||||
await transport.close();
|
||||
return { recovered: false, reason: redactCodexDiagnostic(String(error)) };
|
||||
}
|
||||
}
|
||||
|
||||
#transport(): CodexAppServerTransport {
|
||||
return (
|
||||
this.#options.transportFactory?.() ??
|
||||
new ProcessCodexAppServerTransport({
|
||||
environment: createSanitizedCodexEnvironment(this.#options.environment),
|
||||
onDiagnostic: this.#options.onDiagnostic,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
async #initialize(transport: CodexAppServerTransport): Promise<Record<string, unknown>> {
|
||||
const initialized = await transport.request("initialize", {
|
||||
clientInfo: { name: "paperclip-runner", title: "Paperclip Runner", version: DRIVER_VERSION },
|
||||
capabilities: { experimentalApi: true, requestAttestation: false },
|
||||
});
|
||||
transport.notify("initialized");
|
||||
return initialized;
|
||||
}
|
||||
|
||||
#openedThread(
|
||||
response: Record<string, unknown>,
|
||||
initialize: Record<string, unknown>,
|
||||
workingDirectory: string,
|
||||
): OpenedCodexThread {
|
||||
const thread = record(response.thread);
|
||||
const threadId = text(thread.id);
|
||||
if (threadId.length === 0) throw new Error("Codex thread response omitted thread.id");
|
||||
return {
|
||||
threadId,
|
||||
providerSessionId: text(thread.sessionId, threadId),
|
||||
context: {
|
||||
protocolVersion: PHASE4_CODEX_PROTOCOL_VERSION,
|
||||
codexVersion: text(initialize.userAgent, "unknown"),
|
||||
clientInfo: {
|
||||
name: "paperclip-runner",
|
||||
title: "Paperclip Runner",
|
||||
version: DRIVER_VERSION,
|
||||
},
|
||||
model: text(response.model, "unknown"),
|
||||
modelProvider: text(response.modelProvider, text(thread.modelProvider, "unknown")),
|
||||
workingDirectory: text(response.cwd, workingDirectory),
|
||||
sandbox: response.sandbox ?? null,
|
||||
approvalPolicy: response.approvalPolicy ?? "never",
|
||||
baseInstructions: PHASE4_SKILLLESS_BASE_INSTRUCTIONS,
|
||||
instructionSources: Array.isArray(response.instructionSources)
|
||||
? response.instructionSources.filter((value): value is string => typeof value === "string")
|
||||
: [],
|
||||
instructionPolicy: {
|
||||
skillInstructions: false,
|
||||
appInstructions: false,
|
||||
collaborationInstructions: false,
|
||||
},
|
||||
environmentKeys: sanitizedEnvironmentKeys(this.#options.environment),
|
||||
dynamicToolNames: this.#caps.dynamicTools ? [...PHASE4_SEMANTIC_TOOL_NAMES] : [],
|
||||
modelInputKinds: ["text"],
|
||||
envelope: structuredClone(this.#options.taskEnvelope),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
#session(input: {
|
||||
transport: CodexAppServerTransport;
|
||||
runId: string;
|
||||
normalizedSessionId: string;
|
||||
opened: OpenedCodexThread;
|
||||
resumed: boolean;
|
||||
activeTurnId?: string | null;
|
||||
sourceSequence: number;
|
||||
}): CodexHarnessSession {
|
||||
return new CodexHarnessSession({
|
||||
...input,
|
||||
taskEnvelope: this.#options.taskEnvelope,
|
||||
now: this.#options.now ?? (() => new Date()),
|
||||
runnerInstanceId: this.#options.runnerInstanceId ?? "runner-phase4",
|
||||
capabilities: this.#caps,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class CodexHarnessSession implements HarnessSession {
|
||||
readonly #transport: CodexAppServerTransport;
|
||||
readonly #runId: string;
|
||||
readonly #normalizedSessionId: string;
|
||||
readonly #opened: OpenedCodexThread;
|
||||
readonly #taskEnvelope: Phase4TaskEnvelope;
|
||||
readonly #now: () => Date;
|
||||
readonly #runnerInstanceId: string;
|
||||
readonly #capabilities: CodexCapabilities;
|
||||
readonly #events = new AsyncQueue<PrpEvent>();
|
||||
#sourceSequence: number;
|
||||
#activeTurnId: string | null;
|
||||
#usage: Record<string, unknown> | null = null;
|
||||
#result: PrpStructuredRunResult | null = null;
|
||||
#resultFingerprint: string | null = null;
|
||||
#terminal = false;
|
||||
|
||||
constructor(input: {
|
||||
transport: CodexAppServerTransport;
|
||||
runId: string;
|
||||
normalizedSessionId: string;
|
||||
opened: OpenedCodexThread;
|
||||
taskEnvelope: Phase4TaskEnvelope;
|
||||
resumed: boolean;
|
||||
activeTurnId?: string | null;
|
||||
sourceSequence: number;
|
||||
now: () => Date;
|
||||
runnerInstanceId: string;
|
||||
capabilities: CodexCapabilities;
|
||||
}) {
|
||||
this.#transport = input.transport;
|
||||
this.#runId = input.runId;
|
||||
this.#normalizedSessionId = input.normalizedSessionId;
|
||||
this.#opened = input.opened;
|
||||
this.#taskEnvelope = input.taskEnvelope;
|
||||
this.#activeTurnId = input.activeTurnId ?? null;
|
||||
this.#sourceSequence = input.sourceSequence;
|
||||
this.#now = input.now;
|
||||
this.#runnerInstanceId = input.runnerInstanceId;
|
||||
this.#capabilities = input.capabilities;
|
||||
this.#transport.setServerRequestHandler((request) => this.#handleServerRequest(request));
|
||||
this.#emit(input.resumed ? "session.resumed" : "session.started", {
|
||||
driverSessionId: input.opened.threadId,
|
||||
providerSessionId: input.opened.providerSessionId,
|
||||
context: input.opened.context,
|
||||
});
|
||||
this.#emit("item.completed", {
|
||||
kind: "model",
|
||||
text: `${input.opened.context.model} (${input.opened.context.modelProvider})`,
|
||||
model: {
|
||||
name: input.opened.context.model,
|
||||
provider: input.opened.context.modelProvider,
|
||||
codexVersion: input.opened.context.codexVersion,
|
||||
},
|
||||
}, { itemId: `${input.opened.threadId}:model` });
|
||||
void this.#pumpNotifications();
|
||||
}
|
||||
|
||||
ids(): ReturnType<HarnessSession["ids"]> {
|
||||
return {
|
||||
driverSessionId: this.#opened.threadId,
|
||||
providerSessionId: this.#opened.providerSessionId,
|
||||
displayId: this.#opened.threadId,
|
||||
};
|
||||
}
|
||||
|
||||
contextSnapshot(): Phase4ModelContextSnapshot {
|
||||
return structuredClone(this.#opened.context);
|
||||
}
|
||||
|
||||
events(): AsyncIterable<PrpEvent> {
|
||||
return this.#events;
|
||||
}
|
||||
|
||||
async startTurn(input: { message: NativeUserMessage }): Promise<{ turnId: string }> {
|
||||
const taskText = JSON.stringify({ task: this.#taskEnvelope, message: input.message.text });
|
||||
this.#emit("turn.submitted", { envelopeSchema: this.#taskEnvelope.schema });
|
||||
const response = await this.#transport.request("turn/start", {
|
||||
threadId: this.#opened.threadId,
|
||||
input: [userInput({ role: "user", text: taskText })],
|
||||
outputSchema: PHASE4_RESULT_OUTPUT_SCHEMA,
|
||||
});
|
||||
const turn = record(response.turn);
|
||||
const turnId = text(turn.id);
|
||||
if (turnId.length === 0) throw new Error("Codex turn response omitted turn.id");
|
||||
this.#activeTurnId = turnId;
|
||||
this.#emit("turn.accepted", { turnId }, { turnId });
|
||||
return { turnId };
|
||||
}
|
||||
|
||||
async steer(input: { turnId: string; message: NativeUserMessage }): Promise<void> {
|
||||
this.#requireCapability("steering");
|
||||
this.#requireActiveTurn(input.turnId);
|
||||
try {
|
||||
await this.#transport.request("turn/steer", {
|
||||
threadId: this.#opened.threadId,
|
||||
input: [userInput(input.message)],
|
||||
expectedTurnId: input.turnId,
|
||||
});
|
||||
} catch (error) {
|
||||
throw this.#unsupported("steering", error);
|
||||
}
|
||||
}
|
||||
|
||||
async interrupt(input: { turnId: string; reason?: string }): Promise<void> {
|
||||
this.#requireCapability("interruption");
|
||||
this.#requireActiveTurn(input.turnId);
|
||||
try {
|
||||
await this.#transport.request("turn/interrupt", {
|
||||
threadId: this.#opened.threadId,
|
||||
turnId: input.turnId,
|
||||
});
|
||||
} catch (error) {
|
||||
throw this.#unsupported("interruption", error);
|
||||
}
|
||||
}
|
||||
|
||||
async read(): Promise<Record<string, unknown>> {
|
||||
this.#requireCapability("read");
|
||||
try {
|
||||
return await this.#transport.request("thread/read", {
|
||||
threadId: this.#opened.threadId,
|
||||
includeTurns: true,
|
||||
});
|
||||
} catch (error) {
|
||||
throw this.#unsupported("read", error);
|
||||
}
|
||||
}
|
||||
|
||||
async reconcile(): Promise<Record<string, unknown>> {
|
||||
this.#requireCapability("reconciliation");
|
||||
const snapshot = await this.read();
|
||||
const thread = record(snapshot.thread);
|
||||
const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : [];
|
||||
const active = turns.find((turn) => text(turn.status) === "inProgress");
|
||||
this.#activeTurnId = active === undefined ? null : text(active.id);
|
||||
this.#emit("session.reconciled", {
|
||||
providerSessionId: this.#opened.providerSessionId,
|
||||
turnCount: turns.length,
|
||||
activeTurnId: this.#activeTurnId,
|
||||
});
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
async usage(): Promise<Record<string, unknown> | null> {
|
||||
this.#requireCapability("usage");
|
||||
return this.#usage === null ? null : structuredClone(this.#usage);
|
||||
}
|
||||
|
||||
async snapshot(): Promise<PersistedHarnessSession> {
|
||||
return {
|
||||
driverKind: DRIVER_KIND,
|
||||
driverSessionId: this.#opened.threadId,
|
||||
providerSessionId: this.#opened.providerSessionId,
|
||||
runId: this.#runId,
|
||||
normalizedSessionId: this.#normalizedSessionId,
|
||||
activeTurnId: this.#activeTurnId,
|
||||
lastSourceSequence: this.#sourceSequence,
|
||||
};
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#events.close();
|
||||
await this.#transport.close();
|
||||
}
|
||||
|
||||
async #pumpNotifications(): Promise<void> {
|
||||
try {
|
||||
for await (const notification of this.#transport.notifications()) {
|
||||
this.#mapNotification(notification);
|
||||
}
|
||||
} catch (error) {
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: "notification_transport_failed",
|
||||
message: redactCodexDiagnostic(String(error)),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#mapNotification(notification: CodexRpcNotification): void {
|
||||
const params = notification.params;
|
||||
const turn = record(params.turn);
|
||||
const item = itemFromParams(params);
|
||||
const turnId = text(params.turnId, text(turn.id, this.#activeTurnId ?? ""));
|
||||
const itemId = text(item.id, text(params.itemId));
|
||||
if (notification.method === "turn/started") {
|
||||
this.#activeTurnId = turnId;
|
||||
this.#emit("turn.started", { status: text(turn.status, "inProgress") }, { turnId });
|
||||
return;
|
||||
}
|
||||
if (notification.method === "turn/completed") {
|
||||
const status = text(turn.status, "completed");
|
||||
const eventType = status === "failed" ? "turn.failed" : status === "interrupted" ? "turn.interrupted" : "turn.completed";
|
||||
this.#captureResultFromTurn(turn);
|
||||
this.#emit(eventType, { status, error: turn.error ?? null }, { turnId });
|
||||
this.#activeTurnId = null;
|
||||
this.#finalize(status);
|
||||
return;
|
||||
}
|
||||
if (notification.method === "item/started") {
|
||||
this.#emit("item.started", { kind: text(item.type, "unknown"), text: itemText(item), item }, { turnId, itemId });
|
||||
return;
|
||||
}
|
||||
if (notification.method === "item/completed") {
|
||||
this.#captureResultFromItem(item);
|
||||
this.#emit("item.completed", { kind: text(item.type, "unknown"), text: itemText(item), item }, { turnId, itemId });
|
||||
return;
|
||||
}
|
||||
const deltaKinds: Record<string, string> = {
|
||||
"item/agentMessage/delta": "agentMessage",
|
||||
"item/plan/delta": "plan",
|
||||
"item/reasoning/summaryTextDelta": "reasoning",
|
||||
"item/reasoning/textDelta": "reasoning",
|
||||
"item/commandExecution/outputDelta": "commandExecution",
|
||||
"item/fileChange/outputDelta": "fileChange",
|
||||
"item/fileChange/patchUpdated": "fileChange",
|
||||
"turn/diff/updated": "diff",
|
||||
"turn/plan/updated": "plan",
|
||||
};
|
||||
const deltaKind = deltaKinds[notification.method];
|
||||
if (deltaKind !== undefined) {
|
||||
this.#emit("item.delta", {
|
||||
kind: deltaKind,
|
||||
text: text(params.delta, text(params.patch, text(params.output))),
|
||||
update: params,
|
||||
}, { turnId, itemId: itemId || `${turnId}:${deltaKind}` });
|
||||
return;
|
||||
}
|
||||
if (notification.method === "thread/tokenUsage/updated") {
|
||||
this.#usage = record(params.tokenUsage);
|
||||
this.#emit("item.completed", { kind: "usage", usage: this.#usage }, {
|
||||
turnId,
|
||||
itemId: `${turnId}:usage:${this.#sourceSequence + 1}`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (notification.method === "error" || notification.method === "warning" || notification.method === "configWarning") {
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: notification.method.replaceAll("/", "_"),
|
||||
message: redactCodexDiagnostic(text(params.message, JSON.stringify(params))),
|
||||
}, turnId ? { turnId } : {});
|
||||
}
|
||||
}
|
||||
|
||||
async #handleServerRequest(request: CodexRpcServerRequest): Promise<Record<string, unknown>> {
|
||||
if (request.method === "item/tool/call") {
|
||||
const tool = text(request.params.tool);
|
||||
if (!isSemanticTool(tool)) {
|
||||
this.#diagnoseUnsupported(`dynamic tool ${tool}`);
|
||||
return { success: false, contentItems: [{ type: "inputText", text: "Unsupported tool." }] };
|
||||
}
|
||||
const validation = validatePrpStructuredRunResult(request.params.arguments);
|
||||
if (!validation.ok) {
|
||||
return {
|
||||
success: false,
|
||||
contentItems: [{ type: "inputText", text: `Invalid semantic result: ${validation.issues.map((issue) => issue.message).join("; ")}` }],
|
||||
};
|
||||
}
|
||||
if (!toolAcceptsDisposition(tool, validation.result.reportedWorkDisposition)) {
|
||||
return {
|
||||
success: false,
|
||||
contentItems: [{
|
||||
type: "inputText",
|
||||
text:
|
||||
tool === PHASE4_BLOCK_TOOL_NAME || tool === "paperclip.block"
|
||||
? "paperclip_block requires reportedWorkDisposition=blocked."
|
||||
: "paperclip_finish accepts only done or needs_review.",
|
||||
}],
|
||||
};
|
||||
}
|
||||
const fingerprint = JSON.stringify(validation.result);
|
||||
if (this.#resultFingerprint !== null && this.#resultFingerprint !== fingerprint) {
|
||||
return { success: false, contentItems: [{ type: "inputText", text: "A different semantic result was already committed." }] };
|
||||
}
|
||||
if (this.#result === null) {
|
||||
this.#commitResult(
|
||||
validation.result,
|
||||
text(request.params.callId, randomUUID()),
|
||||
text(request.params.turnId, this.#activeTurnId ?? ""),
|
||||
);
|
||||
}
|
||||
return { success: true, contentItems: [{ type: "inputText", text: "Semantic completion accepted." }] };
|
||||
}
|
||||
|
||||
const requestKind = request.method.includes("requestApproval") ? "approval" : "user_input";
|
||||
const requestId = String(request.id);
|
||||
this.#emit("runtime_request.created", {
|
||||
request: {
|
||||
requestId,
|
||||
requestKind,
|
||||
type: request.method,
|
||||
status: "pending",
|
||||
prompt: "Codex requires an external runtime decision.",
|
||||
details: redactCodexValue(request.params),
|
||||
},
|
||||
}, {
|
||||
turnId: text(request.params.turnId, this.#activeTurnId ?? ""),
|
||||
itemId: text(request.params.itemId, requestId),
|
||||
});
|
||||
const response = safeRequestResponse(request.method);
|
||||
this.#emit("runtime_request.resolved", {
|
||||
requestId,
|
||||
resolution: "declined_by_skillless_driver",
|
||||
}, {
|
||||
turnId: text(request.params.turnId, this.#activeTurnId ?? ""),
|
||||
itemId: text(request.params.itemId, requestId),
|
||||
});
|
||||
return response;
|
||||
}
|
||||
|
||||
#captureResultFromItem(item: Record<string, unknown>): void {
|
||||
if (text(item.type) !== "agentMessage" || this.#result !== null) return;
|
||||
const result = tryParseResult(item.text);
|
||||
if (result !== null) this.#commitResult(result, text(item.id));
|
||||
}
|
||||
|
||||
#captureResultFromTurn(turn: Record<string, unknown>): void {
|
||||
if (this.#result !== null || !Array.isArray(turn.items)) return;
|
||||
for (const value of turn.items.map(record).reverse()) {
|
||||
if (text(value.type) !== "agentMessage") continue;
|
||||
const result = tryParseResult(value.text);
|
||||
if (result !== null) {
|
||||
this.#commitResult(result, text(value.id));
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#commitResult(result: PrpStructuredRunResult, itemId: string, turnId?: string): void {
|
||||
if (this.#result !== null) return;
|
||||
this.#result = structuredClone(result);
|
||||
this.#resultFingerprint = JSON.stringify(result);
|
||||
result.verification.forEach((verification, index) => {
|
||||
this.#emit("item.completed", {
|
||||
kind: "verification",
|
||||
text: `${verification.status}: ${verification.commandOrCheck}`,
|
||||
verification,
|
||||
}, {
|
||||
turnId: turnId || this.#activeTurnId || undefined,
|
||||
itemId: `${itemId || "semantic-result"}:verification:${index + 1}`,
|
||||
});
|
||||
});
|
||||
this.#emit("run.result.proposed", result, {
|
||||
turnId: turnId || this.#activeTurnId || undefined,
|
||||
itemId: itemId || undefined,
|
||||
});
|
||||
}
|
||||
|
||||
#finalize(turnStatus: string): void {
|
||||
if (this.#terminal) return;
|
||||
if (this.#result === null) {
|
||||
this.#commitResult({
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "needs_review",
|
||||
summary: "Codex ended without a valid semantic completion.",
|
||||
completionClaim: {
|
||||
contractRevision: this.#taskEnvelope.completionContract.revision,
|
||||
objectiveSatisfied: false,
|
||||
criteria: this.#taskEnvelope.completionContract.criteria.map((criterion) => ({
|
||||
criterionId: criterion.id,
|
||||
status: "unknown",
|
||||
evidenceRefs: [],
|
||||
})),
|
||||
remainingWork: [{ description: "Review the incomplete Codex turn.", blocksCompletion: true }],
|
||||
},
|
||||
evidence: [],
|
||||
verification: [],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
}, "missing-semantic-result");
|
||||
}
|
||||
const result = this.#result;
|
||||
if (result === null) throw new Error("semantic result finalization invariant failed");
|
||||
this.#terminal = true;
|
||||
this.#emit("run.terminal", {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: terminalState(turnStatus),
|
||||
runTerminalState: dispositionRunState(result),
|
||||
reportedWorkDisposition: result.reportedWorkDisposition,
|
||||
});
|
||||
}
|
||||
|
||||
#requireActiveTurn(turnId: string): void {
|
||||
if (this.#activeTurnId !== turnId) {
|
||||
throw this.#unsupported("active turn operation", "turn is no longer active");
|
||||
}
|
||||
}
|
||||
|
||||
#requireCapability(operation: keyof CodexCapabilities): void {
|
||||
if (!this.#capabilities[operation]) throw this.#unsupported(operation, "capability not advertised");
|
||||
}
|
||||
|
||||
#unsupported(operation: string, detail: unknown): UnsupportedCodexOperationError {
|
||||
const error = new UnsupportedCodexOperationError(operation, String(detail));
|
||||
this.#diagnoseUnsupported(operation, error.message);
|
||||
return error;
|
||||
}
|
||||
|
||||
#diagnoseUnsupported(operation: string, detail = "operation is not available"): void {
|
||||
this.#emit("harness.diagnostic", {
|
||||
code: "unsupported_operation",
|
||||
operation,
|
||||
message: redactCodexDiagnostic(detail),
|
||||
});
|
||||
}
|
||||
|
||||
#emit(
|
||||
eventType: PrpEvent["eventType"],
|
||||
payload: Record<string, unknown>,
|
||||
refs: { turnId?: string; itemId?: string } = {},
|
||||
): void {
|
||||
const sourceSeq = ++this.#sourceSequence;
|
||||
this.#events.push({
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `${this.#runnerInstanceId}:${this.#runId}:${sourceSeq}`,
|
||||
sourceSeq,
|
||||
sourceInstanceId: this.#runnerInstanceId,
|
||||
sourceKind: "runner",
|
||||
runId: this.#runId,
|
||||
normalizedSessionId: this.#normalizedSessionId,
|
||||
...(refs.turnId ? { turnId: refs.turnId } : {}),
|
||||
...(refs.itemId ? { itemId: refs.itemId } : {}),
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1,
|
||||
emittedAt: this.#now().toISOString(),
|
||||
payload,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function isSkilllessContext(context: Phase4ModelContextSnapshot): boolean {
|
||||
const serialized = JSON.stringify(context).toLowerCase();
|
||||
return (
|
||||
context.instructionSources.length === 0 &&
|
||||
context.baseInstructions === PHASE4_SKILLLESS_BASE_INSTRUCTIONS &&
|
||||
context.modelInputKinds.length === 1 &&
|
||||
context.modelInputKinds[0] === "text" &&
|
||||
context.dynamicToolNames.length === PHASE4_SEMANTIC_TOOL_NAMES.length &&
|
||||
context.dynamicToolNames.every((name) =>
|
||||
PHASE4_SEMANTIC_TOOL_NAMES.includes(
|
||||
name as (typeof PHASE4_SEMANTIC_TOOL_NAMES)[number],
|
||||
),
|
||||
) &&
|
||||
context.instructionPolicy.skillInstructions === false &&
|
||||
context.instructionPolicy.appInstructions === false &&
|
||||
context.instructionPolicy.collaborationInstructions === false &&
|
||||
!serialized.includes("paperclip_api_key") &&
|
||||
!serialized.includes("authorization: bearer") &&
|
||||
!serialized.includes("/api/issues/") &&
|
||||
!serialized.includes('"type":"skill"')
|
||||
);
|
||||
}
|
||||
|
||||
function isSemanticTool(tool: string): boolean {
|
||||
return (
|
||||
tool === PHASE4_COMPLETION_TOOL_NAME ||
|
||||
tool === PHASE4_BLOCK_TOOL_NAME ||
|
||||
tool === "paperclip.finish" ||
|
||||
tool === "paperclip.block"
|
||||
);
|
||||
}
|
||||
|
||||
function toolAcceptsDisposition(
|
||||
tool: string,
|
||||
disposition: PrpStructuredRunResult["reportedWorkDisposition"],
|
||||
): boolean {
|
||||
if (tool === PHASE4_BLOCK_TOOL_NAME || tool === "paperclip.block") {
|
||||
return disposition === "blocked";
|
||||
}
|
||||
return disposition === "done" || disposition === "needs_review";
|
||||
}
|
||||
|
||||
function redactCodexValue(value: unknown): unknown {
|
||||
if (typeof value === "string") return redactCodexDiagnostic(value);
|
||||
if (Array.isArray(value)) return value.map(redactCodexValue);
|
||||
const object = record(value);
|
||||
return Object.fromEntries(
|
||||
Object.entries(object).map(([key, entry]) => [
|
||||
key,
|
||||
/(?:api[_-]?key|token|secret|password|authorization)/i.test(key)
|
||||
? "[REDACTED]"
|
||||
: redactCodexValue(entry),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function safeRequestResponse(method: string): Record<string, unknown> {
|
||||
if (method === "item/permissions/requestApproval") {
|
||||
return { permissions: {} };
|
||||
}
|
||||
if (method === "mcpServer/elicitation/request") {
|
||||
return { action: "decline", content: null };
|
||||
}
|
||||
if (method === "item/tool/requestUserInput" || method === "tool/requestUserInput") {
|
||||
return { answers: {} };
|
||||
}
|
||||
if (method.includes("requestApproval")) {
|
||||
return { decision: "decline" };
|
||||
}
|
||||
return {};
|
||||
}
|
||||
@@ -3,8 +3,12 @@ export * from "./contracts/harness-driver.js";
|
||||
export * from "./contracts/native-session-backend.js";
|
||||
export * from "./contracts/phase2.js";
|
||||
export * from "./contracts/phase3.js";
|
||||
export * from "./contracts/phase4.js";
|
||||
export * from "./contracts/types.js";
|
||||
export * from "./drivers/codex/app-server-transport.js";
|
||||
export * from "./drivers/codex/codex-app-server-driver.js";
|
||||
export * from "./mock-core/mock-control-plane-adapter.js";
|
||||
export * from "./mock-core/phase4-codex-runner.js";
|
||||
export * from "./mock-core/phase2-local-runner.js";
|
||||
export * from "./mock-core/phase3-recovery.js";
|
||||
export * from "./protocol/phase0-fixture.js";
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
import type { HarnessDriver, HarnessSession } from "../contracts/harness-driver.js";
|
||||
import {
|
||||
type Phase4RunMetadata,
|
||||
type Phase4RunTrace,
|
||||
type Phase4TaskEnvelope,
|
||||
} from "../contracts/phase4.js";
|
||||
import type { NativeUserMessage } from "../contracts/types.js";
|
||||
import {
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata,
|
||||
} from "../reducer/session-reducer.js";
|
||||
import type {
|
||||
PrpCapabilities,
|
||||
PrpEvent,
|
||||
PrpStructuredRunResult,
|
||||
} from "../protocol/phase1-contract.js";
|
||||
import {
|
||||
UnsupportedCodexOperationError,
|
||||
isSkilllessContext,
|
||||
} from "../drivers/codex/codex-app-server-driver.js";
|
||||
|
||||
export interface Phase4CodexTracerInput {
|
||||
driver: HarnessDriver;
|
||||
taskEnvelope: Phase4TaskEnvelope;
|
||||
workingDirectory: string;
|
||||
message?: NativeUserMessage;
|
||||
runId?: string;
|
||||
companyId?: string;
|
||||
issueId?: string;
|
||||
environmentLeaseId?: string;
|
||||
runnerInstanceId?: string;
|
||||
steer?: string;
|
||||
interrupt?: boolean;
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
function prpCapabilities(
|
||||
descriptor: Awaited<ReturnType<HarnessDriver["descriptor"]>>,
|
||||
): PrpCapabilities {
|
||||
return {
|
||||
schema: "paperclip.prp.capabilities.v1",
|
||||
sessionReusePolicy: "reuse_per_issue",
|
||||
driver: { kind: descriptor.kind, version: descriptor.version },
|
||||
steer: descriptor.capabilities.steering,
|
||||
interrupt: descriptor.capabilities.interruption,
|
||||
resume: descriptor.capabilities.resume,
|
||||
runtimeRequests: true,
|
||||
structuredResult: descriptor.capabilities.structuredResult,
|
||||
typedEvents: descriptor.capabilities.typedEvents,
|
||||
...(descriptor.capabilities.unsupported?.length
|
||||
? { unsupported: descriptor.capabilities.unsupported }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function contextFrom(events: PrpEvent[]) {
|
||||
const event = events.find(
|
||||
(candidate) => candidate.eventType === "session.started" || candidate.eventType === "session.resumed",
|
||||
);
|
||||
const context = event?.payload.context;
|
||||
if (typeof context !== "object" || context === null || Array.isArray(context)) {
|
||||
throw new Error("Codex session did not emit its model-context snapshot");
|
||||
}
|
||||
return context as Phase4RunTrace["context"];
|
||||
}
|
||||
|
||||
function resultFrom(events: PrpEvent[]): PrpStructuredRunResult {
|
||||
const event = events.find((candidate) => candidate.eventType === "run.result.proposed");
|
||||
if (event === undefined) throw new Error("Codex session emitted no semantic result");
|
||||
return event.payload as PrpStructuredRunResult;
|
||||
}
|
||||
|
||||
async function consumeToTerminal(
|
||||
session: HarnessSession,
|
||||
timeoutMs: number,
|
||||
onEvent?: (event: PrpEvent) => void,
|
||||
): Promise<PrpEvent[]> {
|
||||
const consume = async () => {
|
||||
const events: PrpEvent[] = [];
|
||||
for await (const event of session.events()) {
|
||||
events.push(event);
|
||||
onEvent?.(event);
|
||||
if (event.eventType === "run.terminal") return events;
|
||||
}
|
||||
throw new Error("Codex event stream closed before run.terminal");
|
||||
};
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
consume(),
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`Codex tracer timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Small in-memory Phase 4 mock core. It accepts only the public driver contract,
|
||||
* reduces events live, and replays the same canonical stream after completion.
|
||||
*/
|
||||
export async function runPhase4CodexTracer(input: Phase4CodexTracerInput): Promise<Phase4RunTrace> {
|
||||
const runId = input.runId ?? "phase4-safe-run";
|
||||
const normalizedSessionId = `session:${runId}`;
|
||||
const descriptor = await input.driver.descriptor();
|
||||
const capabilities = prpCapabilities(descriptor);
|
||||
const metadata: Phase4RunMetadata = {
|
||||
schema: "paperclip.runner.phase4.metadata.v1",
|
||||
fixtureName: "phase-04-safe-codex-task",
|
||||
identity: {
|
||||
schema: "paperclip.prp.identity.v1",
|
||||
companyId: input.companyId ?? "mock-company",
|
||||
issueId: input.issueId ?? "mock-issue",
|
||||
runId,
|
||||
environmentLeaseId: input.environmentLeaseId ?? "mock-lease",
|
||||
runnerInstanceId: input.runnerInstanceId ?? "runner-phase4",
|
||||
normalizedSessionId,
|
||||
},
|
||||
capabilities,
|
||||
};
|
||||
const session = await input.driver.openSession({ runId, workingDirectory: input.workingDirectory });
|
||||
let signalTurnStarted: (() => void) | undefined;
|
||||
const turnStarted = new Promise<void>((resolve) => {
|
||||
signalTurnStarted = resolve;
|
||||
});
|
||||
const consuming = consumeToTerminal(session, input.timeoutMs ?? 180_000, (event) => {
|
||||
if (event.eventType === "turn.started") signalTurnStarted?.();
|
||||
});
|
||||
|
||||
try {
|
||||
const turn = await session.startTurn({
|
||||
message: input.message ?? { role: "user", text: "Complete the supplied task envelope." },
|
||||
});
|
||||
if (input.steer !== undefined || input.interrupt) {
|
||||
await Promise.race([
|
||||
turnStarted,
|
||||
consuming.then(() => {
|
||||
throw new Error("Codex turn reached terminal state before accepting the control command");
|
||||
}),
|
||||
]);
|
||||
}
|
||||
if (input.steer !== undefined) {
|
||||
if (session.steer === undefined) throw new Error("driver did not expose steering");
|
||||
try {
|
||||
await session.steer({ turnId: turn.turnId, message: { role: "user", text: input.steer } });
|
||||
} catch (error) {
|
||||
// A short real-model turn can finish between turn/start and turn/steer.
|
||||
// The driver already emitted the explicit capability diagnostic; keep
|
||||
// consuming the same session so the mock core records its terminal.
|
||||
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
|
||||
}
|
||||
}
|
||||
if (input.interrupt) {
|
||||
if (session.interrupt === undefined) throw new Error("driver did not expose interruption");
|
||||
try {
|
||||
await session.interrupt({ turnId: turn.turnId, reason: "Phase 4 tutorial interrupt" });
|
||||
} catch (error) {
|
||||
if (!(error instanceof UnsupportedCodexOperationError)) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const events = await consuming;
|
||||
const ids = session.ids();
|
||||
const identity = {
|
||||
...metadata.identity,
|
||||
driverSessionId: ids.driverSessionId,
|
||||
...(ids.providerSessionId ? { providerSessionId: ids.providerSessionId } : {}),
|
||||
};
|
||||
metadata.identity = identity;
|
||||
const liveSnapshot = events.reduce(
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata({
|
||||
fixtureName: metadata.fixtureName,
|
||||
identity,
|
||||
capabilities,
|
||||
}),
|
||||
);
|
||||
const replaySnapshot = events.reduce(
|
||||
applyPrpEvent,
|
||||
createSessionSnapshotFromMetadata({
|
||||
fixtureName: metadata.fixtureName,
|
||||
identity,
|
||||
capabilities,
|
||||
}),
|
||||
);
|
||||
const context = contextFrom(events);
|
||||
const result = resultFrom(events);
|
||||
const terminalCount = events.filter((event) => event.eventType === "run.terminal").length;
|
||||
const resultCount = events.filter((event) => event.eventType === "run.result.proposed").length;
|
||||
const serialized = JSON.stringify(context).toLowerCase();
|
||||
const itemEvents = events.filter((event) => event.eventType.startsWith("item."));
|
||||
return {
|
||||
schema: "paperclip.runner.phase4.trace.v1",
|
||||
metadata,
|
||||
context,
|
||||
events,
|
||||
result,
|
||||
liveSnapshot,
|
||||
replaySnapshot,
|
||||
diagnostics: events
|
||||
.filter((event) => event.eventType === "runner.diagnostic" || event.eventType === "harness.diagnostic")
|
||||
.map((event) => String(event.payload.message ?? event.payload.code ?? "diagnostic")),
|
||||
assertions: {
|
||||
exactlyOneTerminalResult: terminalCount === 1 && resultCount === 1,
|
||||
liveReplayParity: JSON.stringify(liveSnapshot) === JSON.stringify(replaySnapshot),
|
||||
stableIdentity:
|
||||
events.every(
|
||||
(event) =>
|
||||
event.runId === runId && event.normalizedSessionId === normalizedSessionId,
|
||||
) && ids.driverSessionId.length > 0,
|
||||
sourceSequenceContinuous: events.every(
|
||||
(event, index) => event.sourceSeq === index + 1,
|
||||
),
|
||||
stableItemIdentity: itemEvents.every(
|
||||
(event) => typeof event.itemId === "string" && event.itemId.length > 0,
|
||||
),
|
||||
contextIsSkillless: isSkilllessContext(context),
|
||||
unrelatedSkillsAbsent:
|
||||
context.instructionSources.length === 0 &&
|
||||
context.instructionPolicy.skillInstructions === false &&
|
||||
context.modelInputKinds.length === 1 &&
|
||||
context.modelInputKinds[0] === "text",
|
||||
credentialsAbsent:
|
||||
!serialized.includes("paperclip_api_key") &&
|
||||
!serialized.includes("authorization: bearer") &&
|
||||
!serialized.includes("/api/issues/"),
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
await session.close({ reason: "Phase 4 tracer complete" });
|
||||
}
|
||||
}
|
||||
@@ -90,6 +90,11 @@ if (registeredEventValidator === undefined) {
|
||||
throw new Error("PRP event validator was not registered");
|
||||
}
|
||||
const eventValidator = registeredEventValidator as ValidateFunction<PrpEvent>;
|
||||
const registeredResultValidator = ajv.getSchema(resultSchema.$id);
|
||||
if (registeredResultValidator === undefined) {
|
||||
throw new Error("PRP structured result validator was not registered");
|
||||
}
|
||||
const resultValidator = registeredResultValidator as ValidateFunction<PrpStructuredRunResult>;
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
@@ -297,6 +302,24 @@ export function validatePrpEvent(value: unknown): EventValidationResult {
|
||||
return { ok: true, event: value, issues: [] };
|
||||
}
|
||||
|
||||
export type StructuredResultValidationResult =
|
||||
| { ok: true; result: PrpStructuredRunResult; issues: [] }
|
||||
| { ok: false; result: null; issues: ProtocolValidationIssue[] };
|
||||
|
||||
/** Validate a semantic completion independently from a complete replay fixture. */
|
||||
export function validatePrpStructuredRunResult(
|
||||
value: unknown,
|
||||
): StructuredResultValidationResult {
|
||||
if (!resultValidator(value)) {
|
||||
return {
|
||||
ok: false,
|
||||
result: null,
|
||||
issues: (resultValidator.errors ?? []).map(ajvIssue),
|
||||
};
|
||||
}
|
||||
return { ok: true, result: value, issues: [] };
|
||||
}
|
||||
|
||||
export function negotiateProtocolVersion(
|
||||
runner: ProtocolVersionRange,
|
||||
controller: ProtocolVersionRange,
|
||||
|
||||
Reference in new issue
Block a user