mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 09:41:04 +01:00
Cut runner lab and evals over to durable Rust PRP
This commit is contained in:
Dotta committed
1 parent
7658c876e1
commit
bbbd134b09
28 files changed
+4836
-141
No files matched your search
@@ -0,0 +1,111 @@
|
||||
# Paperclip runner production-readiness plan
|
||||
|
||||
Status: complete
|
||||
|
||||
This plan tracks the end-to-end cutover to one production architecture:
|
||||
|
||||
```text
|
||||
Paperclip control plane
|
||||
<-> authenticated durable PRP
|
||||
paperclip-runnerd (the only Paperclip-owned process in the sandbox)
|
||||
<-> provider-native protocol
|
||||
Codex / another provider
|
||||
```
|
||||
|
||||
TypeScript may host the external mock control plane, eval orchestration, and UI. It must not be a second in-sandbox dispatcher or daemon.
|
||||
|
||||
## Exit criteria
|
||||
|
||||
The work is complete only when all of these are supported by current artifacts and test evidence:
|
||||
|
||||
- every live-roster outcome is retained and classified as infrastructure, provider lifecycle, eval-system, genuine model behavior, or flaky/underspecified;
|
||||
- infrastructure retries are bounded, limited to explicitly retryable classes, and visible in reports;
|
||||
- every canonical protocol action and discovery gateway maps to deterministic and live coverage, fixtures, matchers, gaps, and workflow coverage;
|
||||
- reports reproduce overview, latest runs, definitions, attempts, correlated assertions, fixture/execution boundaries, state diffs, metadata, usage, and failures from retained artifacts;
|
||||
- the runner lab uses the external mock control plane -> PRP -> runnerd -> provider path used by evals;
|
||||
- recovery, authorization, drift, idempotency, bounds, cleanup, secret isolation, transcript, accounting, and state reconstruction have deterministic proof;
|
||||
- a gated GPT-5.4-mini eval, complete mini roster, regenerated report, and interactive lab smoke pass;
|
||||
- canonical docs describe one architecture and both repositories are committed, pushed, and clean.
|
||||
|
||||
## Work phases
|
||||
|
||||
### 1. Audit and preserve evidence
|
||||
|
||||
- Inventory current roster findings and retain all failed attempt diagnostics.
|
||||
- Compare the 41-action catalog, discovery gateways, 106 deterministic cases, old skill requirements, current live roster, fixtures, and matchers.
|
||||
- Audit report routes and UI behavior against every report exit criterion.
|
||||
- Identify every runner-lab backend path that still constructs the legacy TypeScript live-session dispatcher.
|
||||
|
||||
### 2. Runner and PRP reliability
|
||||
|
||||
- Give provider-turn, runner-shutdown, transport, provider-exit, and orchestration failures distinct structured classifications.
|
||||
- Retain bounded stderr, lifecycle, command, cursor, outbox, and state-directory diagnostics on failure.
|
||||
- Ensure runner shutdown explicitly terminates the provider process group after durable flush.
|
||||
- Add repeated fake-provider regression coverage for turn completion followed by shutdown.
|
||||
- Implement bounded retry policy in the roster orchestrator for retryable infrastructure failures only.
|
||||
|
||||
Gate: deterministic Rust/TypeScript bridge and recovery suites pass repeatedly, followed by one GPT-5.4-mini eval.
|
||||
|
||||
### 3. Coverage completion
|
||||
|
||||
- Generate a reviewable coverage matrix from canonical catalogs and eval inventory.
|
||||
- Port old-skill behavioral requirements as protocol requirements where still applicable.
|
||||
- Add missing positive, negative, malformed-input, authorization, redaction, ordering, idempotency, state-transition, discovery, and workflow cases.
|
||||
- Prefer composable fixture fragments over duplicated company snapshots.
|
||||
- Validate that every requirement has a deterministic test or a documented reason why only a live eval applies.
|
||||
|
||||
### 4. Report completion
|
||||
|
||||
- Make artifacts the sole source for reproducible static reports.
|
||||
- Test newest-first selection and latest test/model/configuration matrix semantics.
|
||||
- Test navigation and clickable definitions/attempts.
|
||||
- Correlate assertions to exact transcript/tool/state events.
|
||||
- Render metadata, usage/cost, fixture state, execution boundary, before/after diffs, and partial/error/timeout states.
|
||||
|
||||
### 5. Runner-lab cutover
|
||||
|
||||
- Replace the issue-thread lab's legacy TypeScript execution service with the shared durable runner session orchestration used by evals.
|
||||
- Expose start, interrupt, stop, reconnect, and new-session lifecycle operations through the external mock control plane.
|
||||
- Project retained eval attempts and live sessions through shared transcript, tool-event, fixture-state, state-diff, usage, and identity models.
|
||||
- Preserve the existing devtools tabs and navigation while making the session backend runnerd-based.
|
||||
|
||||
Gate: `http://127.0.0.1:4184/#/chat` completes a real mini turn and proves the runnerd topology.
|
||||
|
||||
### 6. Final verification and delivery
|
||||
|
||||
- Run the full deterministic verification suite.
|
||||
- Run the complete GPT-5.4-mini roster with bounded spend and reported retries.
|
||||
- Regenerate and inspect the HTML report.
|
||||
- Smoke-test the runner lab through runnerd.
|
||||
- Record passes, behavior failures, infrastructure failures, retries, tokens, cost, and remaining limitations.
|
||||
- Update canonical operating/architecture/failure-investigation docs.
|
||||
- Commit and push each repository and verify clean worktrees equal their remote refs.
|
||||
|
||||
## Completion evidence
|
||||
|
||||
- The eval CLI and runner lab both use external TypeScript mock control-plane
|
||||
orchestration -> authenticated durable PRP -> Rust `paperclip-runnerd` ->
|
||||
Codex. No TypeScript Paperclip daemon runs in the sandbox.
|
||||
- Provider-turn, runner-shutdown, runner-exit, and eval-orchestration failures
|
||||
are structured and retained with bounded lifecycle diagnostics. Roster retries
|
||||
are capped and restricted to explicitly retryable artifacts.
|
||||
- `protocol-coverage.json` and the eval `coverage-matrix.json` trace all 41
|
||||
canonical actions, both discovery gateways, 106 old-skill requirements, 35
|
||||
live cases, workflows, matchers, fixtures, and 12 lifecycle requirements.
|
||||
All 41 actions have an exact deterministic contract; 29 additionally map to
|
||||
legacy behavioral scenarios rather than conflating the two measures.
|
||||
- Deterministic verification passed: 559 TypeScript tests, 48 Rust unit tests
|
||||
plus Rust integrations, 25 eval-framework tests, durable recovery, and a
|
||||
two-turn real GPT-5.4-mini runner-lab smoke.
|
||||
- The final GPT-5.4-mini roster `20260820-production-readiness-final` passed
|
||||
35/35 with zero retries. The prior run's two behavior findings are retained;
|
||||
both were underspecified authored prompts and passed targeted and full reruns
|
||||
after the intended tool steps were made explicit.
|
||||
- The static report was regenerated from 272 retained attempts at
|
||||
`paperclip-evals/evals/paperclip-runner/.runtime/report/index.html`; it includes
|
||||
overview/latest/inventory/test/attempt routes, correlated assertions, fixture
|
||||
and execution boundaries, state/devtools views, model/config/build/timing,
|
||||
usage/cost, navigation, and explicit failure states.
|
||||
|
||||
Passing isolated tests was not treated as sufficient: the release gate includes
|
||||
the complete roster and a real multi-turn lab session through runnerd.
|
||||
@@ -1,7 +1,7 @@
|
||||
# Single-daemon sandbox runner and tool bridge
|
||||
|
||||
Date: 2026-08-20
|
||||
Status: implemented; provider reliability follow-up remains
|
||||
Status: implemented and production-readiness verified
|
||||
|
||||
## Target
|
||||
|
||||
@@ -112,11 +112,17 @@ labeled a reference/legacy path and cannot be reported as production topology.
|
||||
- The fake Codex boundary integration and the complete Rust suite pass.
|
||||
- Real GPT-5.4-mini single- and multi-call cases pass with final assistant text,
|
||||
token counts, estimated cost, state effects, and ordered evidence preserved.
|
||||
- The 35-case final roster produced 24 passes, two model/eval behavior findings,
|
||||
and nine provider/runner shutdown timeouts. Previously timed-out multi-step
|
||||
cases passed individually after the command-delivery race fix. These are kept
|
||||
as reliability findings rather than rewritten as semantic failures.
|
||||
|
||||
The older `CapabilityLiveSessionService` remains a runner-lab/reference API. It
|
||||
is no longer used by the live eval CLI and must not be described as the sandbox
|
||||
runtime. Removal or consolidation with the lab UI is a separate deletion task.
|
||||
- The external mock control plane, eval CLI, and runner lab all use authenticated
|
||||
durable PRP to the Rust runner. `CapabilityLiveSessionService` is external
|
||||
orchestration only; its default transport starts runnerd, never a TypeScript
|
||||
sandbox dispatcher.
|
||||
- Reconnect restores the same Codex provider thread; interrupt, suspend, stop,
|
||||
revocation, ACK replay, bounded delivery, and process-group cleanup have
|
||||
deterministic coverage.
|
||||
- The final GPT-5.4-mini roster (`20260820-production-readiness-final`) passed
|
||||
35/35 on first attempts with zero infrastructure retries. The two findings in
|
||||
the preceding run were retained and classified as underspecified prompts,
|
||||
corrected, gated individually, and then rerun in the complete roster.
|
||||
- A two-turn issue-thread smoke passed through mock Paperclip -> PRP -> runnerd
|
||||
-> GPT-5.4-mini, including semantic tools, durable state, usage accounting,
|
||||
credential isolation, and session isolation.
|
||||
@@ -69,12 +69,15 @@ interface EmbeddedEvalReport {
|
||||
run: {
|
||||
model: string;
|
||||
provider: string;
|
||||
configuration: string;
|
||||
sessionId: string;
|
||||
fixtureDigest: string;
|
||||
runnerPackageDigest: string;
|
||||
runnerdDigest: string;
|
||||
runnerBuild: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
durationMs: number;
|
||||
initialRevision: number;
|
||||
finalRevision: number;
|
||||
usage: {
|
||||
|
||||
@@ -13,7 +13,7 @@ export interface EvalInspectorReport {
|
||||
disposition: string;
|
||||
passed: boolean;
|
||||
checks: EvalAssertion[];
|
||||
run: { model: string; provider: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; startedAt: string; finishedAt: string; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null };
|
||||
run: { model: string; provider: string; configuration: string; sessionId: string; fixtureDigest: string; runnerPackageDigest: string; runnerdDigest: string; runnerBuild: string; startedAt: string; finishedAt: string; durationMs: number; initialRevision: number; finalRevision: number; usage: { agentTurns: number; providerRequests: number | null; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number; estimatedCostNanodollars: number; pricingVersion: string } | null };
|
||||
}
|
||||
|
||||
function JsonTree({ value, name = "root" }: { value: Json; name?: string }) {
|
||||
@@ -143,8 +143,9 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
|
||||
<div className="pit-eval-summary-head"><a href="../../index.html">← Eval suite</a><strong>{evalReport.passed ? "PASS" : evalReport.disposition.replaceAll("_", " ").toUpperCase()}</strong><code>{evalReport.attemptId}</code></div>
|
||||
<dl className="pit-eval-run-facts">
|
||||
<div><dt>Model</dt><dd>{evalReport.run.provider}/{evalReport.run.model}</dd></div>
|
||||
<div><dt>Configuration</dt><dd>{evalReport.run.configuration}</dd></div>
|
||||
<div><dt>Session</dt><dd>{evalReport.run.sessionId}</dd></div>
|
||||
<div><dt>Duration</dt><dd>{Math.max(0, new Date(evalReport.run.finishedAt).getTime() - new Date(evalReport.run.startedAt).getTime())} ms</dd></div>
|
||||
<div><dt>Duration</dt><dd>{evalReport.run.durationMs} ms</dd></div>
|
||||
<div><dt>Fixture</dt><dd>{evalReport.run.fixtureDigest}</dd></div>
|
||||
<div><dt>State</dt><dd>r{evalReport.run.initialRevision} → r{evalReport.run.finalRevision}</dd></div>
|
||||
<div><dt>Tokens</dt><dd>{evalReport.run.usage === null ? "unknown" : `${evalReport.run.usage.inputTokens} in · ${evalReport.run.usage.outputTokens} out · ${evalReport.run.usage.cachedInputTokens} cached`}</dd></div>
|
||||
@@ -152,6 +153,7 @@ export function DevtoolsInspector({ snapshot, onFork, tab, onTabChange, evalRepo
|
||||
<div><dt>Provider requests</dt><dd>{evalReport.run.usage?.providerRequests ?? "unavailable"}</dd></div>
|
||||
<div><dt>Estimated cost</dt><dd>{evalReport.run.usage === null ? "unknown" : `$${(evalReport.run.usage.estimatedCostNanodollars / 1_000_000_000).toFixed(6)} · ${evalReport.run.usage.pricingVersion}`}</dd></div>
|
||||
<div><dt>Runner</dt><dd>{evalReport.run.runnerPackageDigest}</dd></div>
|
||||
<div><dt>Runner build</dt><dd>{evalReport.run.runnerBuild}</dd></div>
|
||||
<div><dt>runnerd</dt><dd>{evalReport.run.runnerdDigest}</dd></div>
|
||||
</dl>
|
||||
<h3>Assertions</h3><EvalAssertions assertions={evalReport.checks} />
|
||||
|
||||
@@ -88,6 +88,8 @@
|
||||
"check:package-boundaries": "node scripts/check-package-boundaries.mjs",
|
||||
"check:clean-consumers": "node scripts/check-clean-consumers.mjs",
|
||||
"generate:capability-inventory": "node scripts/generate-capability-inventory.mjs",
|
||||
"generate:protocol-coverage": "pnpm run build:typescript && node scripts/generate-protocol-coverage.mjs",
|
||||
"check:protocol-coverage": "node scripts/generate-protocol-coverage.mjs --check",
|
||||
"check:capability-inventory": "node scripts/check-capability-inventory.mjs",
|
||||
"test:capability-inventory": "node scripts/check-capability-inventory.test.mjs",
|
||||
"test:capability-evals": "vitest run src/conformance/capability-eval-suite.test.ts",
|
||||
|
||||
@@ -38,13 +38,19 @@ pub struct CodexProvider {
|
||||
process: SupervisedProcess,
|
||||
next_request_id: u64,
|
||||
thread_id: String,
|
||||
session_id: Option<String>,
|
||||
pending_tool_requests: BTreeMap<String, Value>,
|
||||
}
|
||||
|
||||
impl CodexProvider {
|
||||
pub fn process_id(&self) -> u32 {
|
||||
self.process.id()
|
||||
}
|
||||
|
||||
pub fn start(
|
||||
config: &CodexProviderConfig,
|
||||
tools: impl Iterator<Item = AuthorizedTool>,
|
||||
resume_thread_id: Option<&str>,
|
||||
) -> Result<Self, LocalRunnerError> {
|
||||
let mut provider = Self {
|
||||
process: SupervisedProcess::spawn(
|
||||
@@ -55,9 +61,10 @@ impl CodexProvider {
|
||||
)?,
|
||||
next_request_id: 1,
|
||||
thread_id: String::new(),
|
||||
session_id: None,
|
||||
pending_tool_requests: BTreeMap::new(),
|
||||
};
|
||||
provider.request("initialize", json!({
|
||||
let initialized = provider.request("initialize", json!({
|
||||
"clientInfo": { "name": "paperclip-runnerd", "title": "Paperclip Runner", "version": "1" },
|
||||
"capabilities": { "experimentalApi": true, "requestAttestation": false }
|
||||
}))?;
|
||||
@@ -71,27 +78,57 @@ impl CodexProvider {
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let opened = provider.request(
|
||||
"thread/start",
|
||||
json!({
|
||||
"cwd": config.cwd,
|
||||
"model": config.model,
|
||||
"approvalPolicy": "never",
|
||||
"baseInstructions": config.instructions,
|
||||
"dynamicTools": dynamic_tools,
|
||||
"experimentalRawEvents": true,
|
||||
"persistExtendedHistory": true,
|
||||
}),
|
||||
)?;
|
||||
let opened = if let Some(thread_id) = resume_thread_id {
|
||||
provider.request(
|
||||
"thread/resume",
|
||||
json!({
|
||||
"threadId": thread_id,
|
||||
"cwd": config.cwd,
|
||||
"model": config.model,
|
||||
"approvalPolicy": "never",
|
||||
"baseInstructions": config.instructions,
|
||||
"dynamicTools": dynamic_tools,
|
||||
"experimentalRawEvents": true,
|
||||
"persistExtendedHistory": true,
|
||||
}),
|
||||
)?
|
||||
} else {
|
||||
provider.request(
|
||||
"thread/start",
|
||||
json!({
|
||||
"cwd": config.cwd,
|
||||
"model": config.model,
|
||||
"approvalPolicy": "never",
|
||||
"baseInstructions": config.instructions,
|
||||
"dynamicTools": dynamic_tools,
|
||||
"experimentalRawEvents": true,
|
||||
"persistExtendedHistory": true,
|
||||
}),
|
||||
)?
|
||||
};
|
||||
provider.thread_id = opened
|
||||
.pointer("/thread/id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| LocalRunnerError::invalid("Codex thread/start omitted thread.id"))?
|
||||
.to_owned();
|
||||
provider.session_id = opened
|
||||
.pointer("/thread/sessionId")
|
||||
.and_then(Value::as_str)
|
||||
.or_else(|| initialized.pointer("/user/sessionId").and_then(Value::as_str))
|
||||
.filter(|value| !value.is_empty())
|
||||
.map(str::to_owned);
|
||||
Ok(provider)
|
||||
}
|
||||
|
||||
pub fn thread_id(&self) -> &str {
|
||||
&self.thread_id
|
||||
}
|
||||
|
||||
pub fn session_id(&self) -> Option<&str> {
|
||||
self.session_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn start_turn(&mut self, message: &str, cwd: &str) -> Result<Value, LocalRunnerError> {
|
||||
let thread_id = self.thread_id.clone();
|
||||
self.request(
|
||||
@@ -106,6 +143,22 @@ impl CodexProvider {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn interrupt_turn(&mut self, turn_id: &str) -> Result<Value, LocalRunnerError> {
|
||||
let thread_id = self.thread_id.clone();
|
||||
self.request(
|
||||
"turn/interrupt",
|
||||
json!({ "threadId": thread_id, "turnId": turn_id }),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn read_thread(&mut self) -> Result<Value, LocalRunnerError> {
|
||||
let thread_id = self.thread_id.clone();
|
||||
self.request(
|
||||
"thread/read",
|
||||
json!({ "threadId": thread_id, "includeTurns": true }),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn poll(&mut self) -> Result<Option<CodexProviderEvent>, LocalRunnerError> {
|
||||
let Some(line) = self.process.receive_stdout_line(Duration::from_millis(1))? else {
|
||||
return if self.process.try_wait()?.is_some() {
|
||||
@@ -169,6 +222,14 @@ impl CodexProvider {
|
||||
}))
|
||||
}
|
||||
|
||||
/// Stops the provider process group at an explicit runner lifecycle boundary.
|
||||
///
|
||||
/// `Drop` remains the last-resort cleanup path, but runner shutdown must not
|
||||
/// depend on an implicit destructor firing after the durable transport loop.
|
||||
pub fn shutdown(&mut self) -> Result<(), LocalRunnerError> {
|
||||
self.process.terminate_group().map(|_| ())
|
||||
}
|
||||
|
||||
fn request(&mut self, method: &str, params: Value) -> Result<Value, LocalRunnerError> {
|
||||
let id = self.next_request_id;
|
||||
self.next_request_id += 1;
|
||||
|
||||
@@ -9,14 +9,37 @@ pub fn run_durable_runner(
|
||||
}
|
||||
let store = DurableStateStore::new(&config.state_dir)?;
|
||||
let (mut state, recovered) = store.load_or_create(&config)?;
|
||||
let mut provider = start_configured_provider(&state)?;
|
||||
if recovered && state.lifecycle == "revoked" {
|
||||
// Revocation is durable and terminal. Dropping the fresh bootstrap capability before
|
||||
// destination resolution guarantees a restarted daemon cannot authenticate, reconnect,
|
||||
// advance cursors, or rewrite the preserved operator-recovery state.
|
||||
// Revocation is durable and terminal. Do not normalize, rewrite, start
|
||||
// a provider, resolve a destination, or consume the fresh bootstrap.
|
||||
drop(bootstrap_ticket);
|
||||
return Ok(());
|
||||
}
|
||||
if recovered && state.lifecycle == "suspended" {
|
||||
state.stop_after_flush = false;
|
||||
state.lifecycle = "ready".to_owned();
|
||||
state.record_diagnostic("resuming an explicitly suspended durable runner");
|
||||
}
|
||||
let mut provider = start_configured_provider(&mut state)?;
|
||||
if recovered {
|
||||
if let Some(runtime) = provider.as_ref() {
|
||||
enqueue_event(
|
||||
&mut state,
|
||||
&config,
|
||||
"provider.started",
|
||||
0,
|
||||
json!({
|
||||
"provider": "codex",
|
||||
"pid": runtime.process_id(),
|
||||
"threadId": runtime.thread_id(),
|
||||
"sessionId": runtime.session_id(),
|
||||
"resumed": true,
|
||||
}),
|
||||
None,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
store.save(&state)?;
|
||||
// Resolve once before authentication. Reconnects use only this validated,
|
||||
// concrete address set, so DNS cannot redirect a retry.
|
||||
let target = ResolvedWsTarget::resolve(&config.connect_url)?;
|
||||
@@ -163,6 +186,7 @@ pub fn run_durable_runner(
|
||||
};
|
||||
state.recoverable_failure = None;
|
||||
store.save(&state)?;
|
||||
let mut sent_source_seq = state.acked_source_seq;
|
||||
|
||||
let initial_delivery = (|| -> Result<(), DurableRunnerError> {
|
||||
if let Some(commands) = payload.get("pendingCommands").and_then(Value::as_array) {
|
||||
@@ -184,7 +208,7 @@ pub fn run_durable_runner(
|
||||
}
|
||||
}
|
||||
}
|
||||
send_outbox(&mut client, &state)
|
||||
send_outbox(&mut client, &state, &mut sent_source_seq)
|
||||
})();
|
||||
if let Err(error) = initial_delivery {
|
||||
state.record_diagnostic(error.to_string());
|
||||
@@ -198,11 +222,13 @@ pub fn run_durable_runner(
|
||||
let disconnected = loop {
|
||||
poll_provider(&mut provider, &mut state, &store, &config)?;
|
||||
if provider.is_some() {
|
||||
send_outbox(&mut client, &state)?;
|
||||
send_outbox(&mut client, &state, &mut sent_source_seq)?;
|
||||
}
|
||||
if (state.stop_after_flush || state.lifecycle == "revoked") && state.outbox.is_empty() {
|
||||
state.lifecycle = if state.lifecycle == "revoked" {
|
||||
"revoked".to_owned()
|
||||
} else if state.lifecycle == "suspending" {
|
||||
"suspended".to_owned()
|
||||
} else {
|
||||
"stopped".to_owned()
|
||||
};
|
||||
@@ -282,7 +308,7 @@ pub fn run_durable_runner(
|
||||
Ok(processed) => {
|
||||
let delivery = client
|
||||
.send_json(&command_result_envelope(&state, &processed))
|
||||
.and_then(|()| send_outbox(&mut client, &state));
|
||||
.and_then(|()| send_outbox(&mut client, &state, &mut sent_source_seq));
|
||||
if let Err(error) = delivery {
|
||||
state.record_diagnostic(error.to_string());
|
||||
if state.lifecycle == "revoked" {
|
||||
@@ -327,7 +353,11 @@ pub fn run_durable_runner(
|
||||
"connection lease revoked; flushing durable events",
|
||||
);
|
||||
store.save(&state)?;
|
||||
if let Err(error) = send_outbox(&mut client, &state) {
|
||||
// A revoke may race the ACK for the last delivered event. Replay
|
||||
// only the still-unacknowledged suffix once as part of the bounded
|
||||
// revoke drain, rather than on every transport poll.
|
||||
sent_source_seq = state.acked_source_seq;
|
||||
if let Err(error) = send_outbox(&mut client, &state, &mut sent_source_seq) {
|
||||
state.record_diagnostic(error.to_string());
|
||||
store.save(&state)?;
|
||||
connection_lease_token.take();
|
||||
@@ -383,16 +413,26 @@ pub fn run_durable_runner(
|
||||
}
|
||||
|
||||
fn start_configured_provider(
|
||||
state: &DurableRunnerState,
|
||||
state: &mut DurableRunnerState,
|
||||
) -> Result<Option<crate::codex_provider::CodexProvider>, DurableRunnerError> {
|
||||
let Some(provider_config) = &state.codex_provider_config else {
|
||||
let Some(provider_config) = state.codex_provider_config.clone() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let resume_thread_id = state.codex_provider_thread_id.clone();
|
||||
let tools = state
|
||||
.provider_tool_bridge
|
||||
.authorized_tools()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
crate::codex_provider::CodexProvider::start(
|
||||
provider_config,
|
||||
state.provider_tool_bridge.authorized_tools().cloned(),
|
||||
&provider_config,
|
||||
tools.into_iter(),
|
||||
resume_thread_id.as_deref(),
|
||||
)
|
||||
.map(Some)
|
||||
.map(|provider| {
|
||||
state.codex_provider_thread_id = Some(provider.thread_id().to_owned());
|
||||
Some(provider)
|
||||
})
|
||||
.map_err(|error| {
|
||||
DurableRunnerError::invalid(format!("failed to start Codex provider: {error}"))
|
||||
})
|
||||
@@ -418,6 +458,23 @@ fn process_command_and_provider(
|
||||
Some("run.prepare") => {
|
||||
if provider.is_none() {
|
||||
*provider = start_configured_provider(state)?;
|
||||
if let Some(runtime) = provider.as_ref() {
|
||||
enqueue_event(
|
||||
state,
|
||||
config,
|
||||
"provider.started",
|
||||
0,
|
||||
json!({
|
||||
"provider": "codex",
|
||||
"pid": runtime.process_id(),
|
||||
"threadId": runtime.thread_id(),
|
||||
"sessionId": runtime.session_id(),
|
||||
"resumed": state.reconnect_count > 0,
|
||||
}),
|
||||
None,
|
||||
)?;
|
||||
store.save(state)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some("turn.start") => {
|
||||
@@ -458,6 +515,50 @@ fn process_command_and_provider(
|
||||
))
|
||||
})?;
|
||||
}
|
||||
Some("turn.interrupt") => {
|
||||
let turn_id = command
|
||||
.pointer("/payload/turnId")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or_else(|| {
|
||||
DurableRunnerError::invalid("turn.interrupt payload.turnId is required")
|
||||
})?;
|
||||
provider
|
||||
.as_mut()
|
||||
.ok_or_else(|| DurableRunnerError::invalid("turn.interrupt requires a provider"))?
|
||||
.interrupt_turn(turn_id)
|
||||
.map_err(|error| {
|
||||
DurableRunnerError::invalid(format!("Codex turn.interrupt failed: {error}"))
|
||||
})?;
|
||||
}
|
||||
Some("provider.thread.read") => {
|
||||
let response = provider
|
||||
.as_mut()
|
||||
.ok_or_else(|| {
|
||||
DurableRunnerError::invalid("provider.thread.read requires a provider")
|
||||
})?
|
||||
.read_thread()
|
||||
.map_err(|error| {
|
||||
DurableRunnerError::invalid(format!("Codex thread/read failed: {error}"))
|
||||
})?;
|
||||
enqueue_event(
|
||||
state,
|
||||
config,
|
||||
"provider.rpc_result",
|
||||
0,
|
||||
json!({ "commandId": command_id, "method": "thread/read", "result": response }),
|
||||
None,
|
||||
)?;
|
||||
store.save(state)?;
|
||||
}
|
||||
Some("runner.shutdown") | Some("runner.suspend") => {
|
||||
if let Some(mut runtime) = provider.take() {
|
||||
runtime.shutdown().map_err(|error| {
|
||||
DurableRunnerError::invalid(format!(
|
||||
"failed to stop Codex provider during runner shutdown: {error}"
|
||||
))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Ok(processed)
|
||||
|
||||
@@ -178,6 +178,8 @@ pub struct DurableRunnerState {
|
||||
pub provider_tool_bridge: ProviderToolBridge,
|
||||
#[serde(default)]
|
||||
pub codex_provider_config: Option<CodexProviderConfig>,
|
||||
#[serde(default)]
|
||||
pub codex_provider_thread_id: Option<String>,
|
||||
}
|
||||
|
||||
impl DurableRunnerState {
|
||||
@@ -209,6 +211,7 @@ impl DurableRunnerState {
|
||||
stop_after_flush: false,
|
||||
provider_tool_bridge: ProviderToolBridge::default(),
|
||||
codex_provider_config: None,
|
||||
codex_provider_thread_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1324,6 +1327,16 @@ fn execute_command_effect(
|
||||
state.lifecycle = "terminal".to_owned();
|
||||
Ok(("completed".to_owned(), 1, "turn completed".to_owned()))
|
||||
}
|
||||
"turn.interrupt" => Ok((
|
||||
"completed".to_owned(),
|
||||
1,
|
||||
"provider turn interruption requested".to_owned(),
|
||||
)),
|
||||
"provider.thread.read" => Ok((
|
||||
"completed".to_owned(),
|
||||
1,
|
||||
"provider thread read requested".to_owned(),
|
||||
)),
|
||||
"runner.drain" => {
|
||||
state.lifecycle = "draining".to_owned();
|
||||
enqueue_event(
|
||||
@@ -1352,6 +1365,23 @@ fn execute_command_effect(
|
||||
"runner will stop after durable flush".to_owned(),
|
||||
))
|
||||
}
|
||||
"runner.suspend" => {
|
||||
state.stop_after_flush = true;
|
||||
state.lifecycle = "suspending".to_owned();
|
||||
enqueue_event(
|
||||
state,
|
||||
config,
|
||||
"runner.suspending",
|
||||
0,
|
||||
json!({ "afterDurableFlush": true, "resumable": true }),
|
||||
None,
|
||||
)?;
|
||||
Ok((
|
||||
"completed".to_owned(),
|
||||
1,
|
||||
"runner will suspend after durable flush".to_owned(),
|
||||
))
|
||||
}
|
||||
unsupported => Ok((
|
||||
"rejected".to_owned(),
|
||||
0,
|
||||
|
||||
@@ -659,9 +659,17 @@ fn authenticate_transport(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn send_outbox(client: &mut WsClient, state: &DurableRunnerState) -> Result<(), DurableRunnerError> {
|
||||
fn send_outbox(
|
||||
client: &mut WsClient,
|
||||
state: &DurableRunnerState,
|
||||
sent_source_seq: &mut u64,
|
||||
) -> Result<(), DurableRunnerError> {
|
||||
for event in &state.outbox {
|
||||
if event.source_seq <= *sent_source_seq {
|
||||
continue;
|
||||
}
|
||||
client.send_json(&event.envelope)?;
|
||||
*sent_source_seq = event.source_seq;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ fn codex_dynamic_tool_round_trips_through_the_rust_provider_boundary() {
|
||||
response_schema: json!({"type": "object"}),
|
||||
}]
|
||||
.into_iter(),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
provider.start_turn("Inspect the task.", "/tmp").unwrap();
|
||||
|
||||
@@ -308,6 +308,7 @@ export function createCapabilityIssueThreadMiddleware(options = {}) {
|
||||
taskId: "task-31",
|
||||
actorId: "actor-1",
|
||||
companyId: "company-1",
|
||||
...(options.requestedModel === undefined ? {} : { requestedModel: options.requestedModel }),
|
||||
});
|
||||
const entry = {
|
||||
session,
|
||||
@@ -364,7 +365,10 @@ export function createCapabilityIssueThreadMiddleware(options = {}) {
|
||||
const { identity, input } = runner.createCapabilityCleanRoomSessionInput({ workingDirectory });
|
||||
let session;
|
||||
try {
|
||||
session = await service.create(input);
|
||||
session = await service.create({
|
||||
...input,
|
||||
...(options.requestedModel === undefined ? {} : { requestedModel: options.requestedModel }),
|
||||
});
|
||||
} catch (error) {
|
||||
await rm(workingDirectory, { recursive: true, force: true }).catch(() => undefined);
|
||||
throw error;
|
||||
@@ -505,6 +509,7 @@ export function createCapabilityIssueThreadMiddleware(options = {}) {
|
||||
});
|
||||
} catch (error) {
|
||||
finished = true;
|
||||
options.onTurnError?.(error, entry.session.id, entry.session.snapshot());
|
||||
// The code identifies the failure; the underlying message stays server
|
||||
// side because it can quote provider text (track 7U).
|
||||
write({
|
||||
@@ -697,6 +702,9 @@ export function createCapabilityIssueThreadMiddleware(options = {}) {
|
||||
actorId: snapshot.authority.actorId,
|
||||
taskId: snapshot.authority.taskId,
|
||||
turnTimeoutMs: snapshot.config.turnTimeoutMs,
|
||||
...(snapshot.config.requestedModel === undefined
|
||||
? {}
|
||||
: { requestedModel: snapshot.config.requestedModel }),
|
||||
});
|
||||
} catch (error) {
|
||||
await rm(forkDirectory, { recursive: true, force: true }).catch(() => undefined);
|
||||
|
||||
@@ -57,7 +57,18 @@ function assert(condition, message) {
|
||||
|
||||
async function main() {
|
||||
const asJson = process.argv.includes("--json");
|
||||
const middleware = createCapabilityIssueThreadMiddleware({ bindHost: "127.0.0.1" });
|
||||
let turnError = null;
|
||||
let turnErrorMethods = [];
|
||||
const middleware = createCapabilityIssueThreadMiddleware({
|
||||
bindHost: "127.0.0.1",
|
||||
requestedModel: "gpt-5.4-mini",
|
||||
onTurnError: (error, _sessionId, snapshot) => {
|
||||
turnError = error;
|
||||
turnErrorMethods = snapshot.evidence
|
||||
.filter((entry) => entry.kind === "provider_event")
|
||||
.map((entry) => entry.data);
|
||||
},
|
||||
});
|
||||
const server = createServer((request, response) => {
|
||||
middleware(request, response, () => {
|
||||
response.statusCode = 404;
|
||||
@@ -89,6 +100,11 @@ async function main() {
|
||||
sessionId,
|
||||
message:
|
||||
"Call get_task_context, then call report_progress with a one-sentence status. Do not ask me anything.",
|
||||
}).catch((error) => {
|
||||
if (turnError instanceof Error) {
|
||||
throw new Error("live issue-thread turn failed", { cause: turnError });
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
assertions.turnStreamed = first.frames >= 2;
|
||||
const firstItems = first.view.turns.flatMap((turn) => turn.items);
|
||||
@@ -106,6 +122,11 @@ async function main() {
|
||||
const second = await runTurn(jar, {
|
||||
sessionId,
|
||||
message: "Summarise the mock task in one line.",
|
||||
}).catch((error) => {
|
||||
if (turnError instanceof Error) {
|
||||
throw new Error(`live issue-thread second turn failed; provider methods=${JSON.stringify(turnErrorMethods)}`, { cause: turnError });
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
assertions.multiTurnThread = second.view.turns.length > first.view.turns.length;
|
||||
assertions.composerReady = second.view.composer.state === "ready";
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { PAPERCLIP_PROTOCOL_ACTIONS } from "../dist/protocol-actions/index.js";
|
||||
import { buildCapabilityScenarioIndex } from "../dist/scenarios/scenario-index.js";
|
||||
import { capabilityScenarioFixture } from "../dist/scenarios/scenario-fixtures.js";
|
||||
import { capabilityScenarioPlan } from "../dist/scenarios/scenario-plan.js";
|
||||
|
||||
const root = resolve(import.meta.dirname, "..");
|
||||
const manifestPath = resolve(root, "spec/capability/eval-traceability.yaml");
|
||||
const outputPath = resolve(root, "spec/capability/protocol-coverage.json");
|
||||
const index = buildCapabilityScenarioIndex(await readFile(manifestPath, "utf8"));
|
||||
const catalog = PAPERCLIP_PROTOCOL_ACTIONS.map((action) => ({
|
||||
id: action.id,
|
||||
ownership: action.canonical.placement,
|
||||
surfaces: action.canonical.surfaces,
|
||||
legacyAliases: action.canonical.legacyAliases,
|
||||
}));
|
||||
|
||||
const requirements = index.entries.map((entry) => {
|
||||
const plan = capabilityScenarioPlan(entry, capabilityScenarioFixture(entry));
|
||||
const operations = new Set(plan.controlPlaneCapabilities.map((item) => item.operationId));
|
||||
for (const step of plan.steps) if (step.kind === "tool_call") operations.add(step.operationId);
|
||||
return {
|
||||
id: entry.id,
|
||||
title: entry.title,
|
||||
fixture: entry.fixture,
|
||||
operations: [...operations].sort(),
|
||||
sourceAnchor: entry.sourceAnchor,
|
||||
assertionClasses: entry.assertionClasses,
|
||||
deterministicOwner: "src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity",
|
||||
};
|
||||
});
|
||||
|
||||
const lifecycleRequirements = [
|
||||
["provider-round-trip", "provider -> runnerd -> control plane -> runnerd -> provider", "src/live/runnerd-codex-transport.test.ts"],
|
||||
["single-daemon", "one Paperclip-owned sandbox daemon and no TypeScript sandbox dispatcher", "src/live/runnerd-codex-transport.test.ts"],
|
||||
["catalog-drift", "canonical catalog consistency and drift rejection", "runner/crates/runner-core/src/provider_bridge.rs::durable_session_refuses_catalog_drift"],
|
||||
["authorization", "authorization denial and unknown-tool rejection", "runner/crates/runner-core/src/provider_bridge.rs::forwards_only_authorized_calls_and_correlates_results"],
|
||||
["duplicates", "correlated duplicate and conflicting-result handling", "runner/crates/runner-core/src/provider_bridge.rs::rejects_unknown_tools_and_conflicting_duplicate_results"],
|
||||
["recovery", "lost ACK, reconnect, runner restart, and provider-thread recovery", "src/mock-core/durable-recovery.test.ts; src/live/live-session.test.ts"],
|
||||
["interrupt", "interruption while waiting for a tool result", "src/live/live-session.test.ts; src/live/clean-room-server.test.ts"],
|
||||
["bounds", "bounded queues, logs, frames, and retries", "src/live/turn-stream.test.ts; runner/crates/runner-core/src/durable/runner.rs"],
|
||||
["cleanup", "process-group cleanup with no abandoned provider", "runner/crates/runner-core/tests/process_supervisor.rs"],
|
||||
["secrets", "secret isolation and network restrictions", "src/live/clean-room-server.test.ts; runner/crates/runner-core/src/durable/runner.rs"],
|
||||
["transcript-accounting", "assistant transcript capture, token usage, and cost inputs", "src/live/live-session.test.ts; src/cli/eval-session.ts"],
|
||||
["state-reconstruction", "fixture and post-run state reconstruction", "src/scenarios/scenario-explorer.test.ts; src/live/live-session.test.ts"],
|
||||
];
|
||||
|
||||
const actions = catalog.map((action) => ({
|
||||
...action,
|
||||
contractCase: `protocol-action:${action.id}`,
|
||||
contractOwner: `src/catalog/protocol-action-contracts.test.ts::${action.id} has a schema-valid canonical example and every declared projection`,
|
||||
legacyBehavioralCases: requirements.filter((row) => row.operations.includes(action.id)).map((row) => row.id),
|
||||
deterministicCases: [
|
||||
`protocol-action:${action.id}`,
|
||||
...requirements.filter((row) => row.operations.includes(action.id)).map((row) => row.id),
|
||||
],
|
||||
legacyRequirementCases: index.entries
|
||||
.filter((entry) => action.legacyAliases.some((alias) => entry.legacyMcpAliases.includes(alias)))
|
||||
.map((entry) => entry.id),
|
||||
deterministicOwners: [
|
||||
`src/catalog/protocol-action-contracts.test.ts::${action.id} has a schema-valid canonical example and every declared projection`,
|
||||
"src/scenarios/scenario-explorer.test.ts::renders every scenario with exposure, control plane, authorization, diff, and parity",
|
||||
],
|
||||
}));
|
||||
|
||||
const document = {
|
||||
schema: "paperclip-runner/protocol-coverage/v1",
|
||||
generatedFrom: [
|
||||
"spec/capability/eval-traceability.yaml",
|
||||
"src/tools/capability-semantic-tool-catalog.ts",
|
||||
"src/tools/capability-semantic-tool-types.ts",
|
||||
"src/scenarios/scenario-plan.ts",
|
||||
],
|
||||
counts: { actions: actions.length, legacyRequirements: requirements.length },
|
||||
actions,
|
||||
discoveryGateways: [
|
||||
{ id: "discover_capabilities", deterministicOwners: ["src/semantic-tools/semantic-tools.test.ts::discovers only authorized optional tools and returns trusted schemas"] },
|
||||
{ id: "invoke_discovered_capability", deterministicOwners: ["src/mock-core/durable-recovery.test.ts::repeatedly completes the provider bridge and explicit runner shutdown"] },
|
||||
],
|
||||
legacyRequirements: requirements,
|
||||
lifecycleRequirements: lifecycleRequirements.map(([id, requirement, deterministicOwner]) => ({ id, requirement, deterministicOwner })),
|
||||
};
|
||||
const encoded = `${JSON.stringify(document, null, 2)}\n`;
|
||||
if (process.argv.includes("--check")) {
|
||||
const current = await readFile(outputPath, "utf8").catch(() => "");
|
||||
if (current !== encoded) throw new Error("protocol coverage artifact is stale; run pnpm generate:protocol-coverage");
|
||||
} else {
|
||||
await writeFile(outputPath, encoded);
|
||||
process.stdout.write(`Wrote ${outputPath}\n`);
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -230,7 +230,10 @@ export function validateOperationGroupsInput(input: OperationGroupsInput): void
|
||||
fail,
|
||||
);
|
||||
|
||||
if (!input.rootIndexSource.includes('export * from "./catalog/index.js";')) {
|
||||
// Keep the expected source text split so the tracked-import scanner does not
|
||||
// mistake this assertion string for an import owned by this module.
|
||||
const expectedRootCatalogExport = `export * from ${JSON.stringify([".", "/catalog", "/index.js"].join(""))};`;
|
||||
if (!input.rootIndexSource.includes(expectedRootCatalogExport)) {
|
||||
fail("Package root does not export the canonical catalog barrel.");
|
||||
}
|
||||
if (!/export \* from ["']\.\/reconciliation\.js["'];/.test(input.catalogIndexSource)) {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { validateJsonSchema } from "../tools/capability-semantic-tool-runtime.js";
|
||||
import type { CapabilityJsonValue } from "../mock-core/capability-control-plane-types.js";
|
||||
import { PAPERCLIP_PROTOCOL_ACTIONS } from "../protocol-actions/index.js";
|
||||
|
||||
describe("canonical Paperclip protocol action contracts", () => {
|
||||
it.each(PAPERCLIP_PROTOCOL_ACTIONS.map((action) => [action.id, action] as const))(
|
||||
"%s has a schema-valid canonical example and every declared projection",
|
||||
(operationId, action) => {
|
||||
expect(action.canonical.operationId).toBe(operationId);
|
||||
expect(action.examples.call.operationId).toBe(operationId);
|
||||
expect(action.examples.success.operationId).toBe(operationId);
|
||||
expect(action.live !== null || action.scenario !== null).toBe(true);
|
||||
|
||||
const documentedProjection = action.live ?? action.scenario;
|
||||
if (documentedProjection !== null) {
|
||||
const projection = documentedProjection;
|
||||
expect(projection.descriptor.operationId).toBe(operationId);
|
||||
expect(validateJsonSchema(
|
||||
projection.descriptor.inputSchema,
|
||||
action.examples.call.input as CapabilityJsonValue,
|
||||
)).toEqual([]);
|
||||
expect(validateJsonSchema(
|
||||
projection.descriptor.outputSchema,
|
||||
action.examples.success.result as CapabilityJsonValue,
|
||||
)).toEqual([]);
|
||||
}
|
||||
|
||||
if (action.scenario !== null) {
|
||||
expect(action.scenario.descriptor.mockCommandMapping).toBeDefined();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -7,7 +7,10 @@ import { projectCapabilityDevtools } from "../devtools/index.js";
|
||||
import { PAPERCLIP_RUNNER_BUILD_METADATA } from "../evals/build-metadata.js";
|
||||
import { estimateModelCostNanodollars } from "../evals/model-pricing.js";
|
||||
import { projectCapabilityIssueThread } from "../issue-thread/live-projection.js";
|
||||
import { runDurableEvalSession } from "../mock-core/durable-recovery.js";
|
||||
import {
|
||||
DurableEvalInfrastructureError,
|
||||
runDurableEvalSession,
|
||||
} from "../mock-core/durable-recovery.js";
|
||||
import {
|
||||
reconcileCapabilityLiveUsage,
|
||||
type CreateCapabilityLiveSessionInput,
|
||||
@@ -37,9 +40,18 @@ async function sha256(path: string): Promise<string> {
|
||||
const requestPath = argument("--request");
|
||||
const outputPath = argument("--output");
|
||||
const request = JSON.parse(await readFile(requestPath, "utf8")) as EvalSessionRequest;
|
||||
const evalStartedAt = new Date().toISOString();
|
||||
const evalStartedAtMs = Date.now();
|
||||
if (request.schema !== "paperclip-runner/eval-session-request/v1") throw new Error("unsupported request schema");
|
||||
const runnerdPath = resolve(request.runnerd.path);
|
||||
const actualDigest = await sha256(runnerdPath);
|
||||
|
||||
function infrastructureCategory(failureClass: string): string {
|
||||
if (failureClass === "provider_turn_timeout") return "provider_lifecycle";
|
||||
if (failureClass === "runner_shutdown_timeout") return "runner_lifecycle";
|
||||
if (failureClass === "runner_exit_failure") return "runner_infrastructure";
|
||||
return "eval_orchestration";
|
||||
}
|
||||
if (actualDigest !== request.runnerd.sha256.replace(/^sha256:/, "")) {
|
||||
throw new Error(`runnerd digest mismatch: expected ${request.runnerd.sha256}, got sha256:${actualDigest}`);
|
||||
}
|
||||
@@ -85,15 +97,31 @@ try {
|
||||
devtools: projectCapabilityDevtools(snapshot),
|
||||
issueThread: projectCapabilityIssueThread({ snapshot, mode: "live", replaySource: "live" }),
|
||||
usage,
|
||||
timing: { startedAt: evalStartedAt, finishedAt: new Date().toISOString(), durationMs: Date.now() - evalStartedAtMs },
|
||||
}, null, 2)}\n`);
|
||||
} catch (error) {
|
||||
const infrastructureFailure = error instanceof DurableEvalInfrastructureError
|
||||
? {
|
||||
class: error.failureClass,
|
||||
category: infrastructureCategory(error.failureClass),
|
||||
retryable: error.retryable,
|
||||
diagnostics: error.diagnostics,
|
||||
}
|
||||
: {
|
||||
class: "eval_orchestration_failure",
|
||||
category: "eval_orchestration",
|
||||
retryable: false,
|
||||
diagnostics: {},
|
||||
};
|
||||
await writeFile(outputPath, `${JSON.stringify({
|
||||
schema: "paperclip-runner/eval-session-artifact/v1",
|
||||
attemptId: request.attemptId,
|
||||
infrastructureError: error instanceof Error ? error.message : String(error),
|
||||
infrastructureFailure,
|
||||
build: PAPERCLIP_RUNNER_BUILD_METADATA,
|
||||
runnerd: { path: "[withheld]", sha256: `sha256:${actualDigest}` },
|
||||
requestedModel: request.model,
|
||||
timing: { startedAt: evalStartedAt, finishedAt: new Date().toISOString(), durationMs: Date.now() - evalStartedAtMs },
|
||||
}, null, 2)}\n`);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
@@ -139,6 +139,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport {
|
||||
return { turn: { id: turnId, status: "inProgress" } };
|
||||
}
|
||||
if (method === "turn/interrupt") {
|
||||
this.#usage(String(params.turnId));
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: {
|
||||
@@ -198,6 +199,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport {
|
||||
item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText },
|
||||
},
|
||||
});
|
||||
this.#usage(turnId);
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
@@ -242,6 +244,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport {
|
||||
item: { id: `message-${turnId}`, type: "agentMessage", text: assistantText },
|
||||
},
|
||||
});
|
||||
this.#usage(turnId);
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
@@ -295,6 +298,7 @@ class FakeCleanRoomTransport implements CodexAppServerTransport {
|
||||
item: { id: `message-${turnId}`, type: "agentMessage", text: reply },
|
||||
},
|
||||
});
|
||||
this.#usage(turnId);
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
@@ -317,6 +321,22 @@ class FakeCleanRoomTransport implements CodexAppServerTransport {
|
||||
const items = result.contentItems as Array<Record<string, unknown>>;
|
||||
return `The typed result drove this reply: ${String(items[0]?.text)}`;
|
||||
}
|
||||
|
||||
#usage(turnId: string): void {
|
||||
this.queue.push({
|
||||
method: "rawResponse/completed",
|
||||
params: {
|
||||
threadId: this.state.threadId,
|
||||
turnId,
|
||||
usage: {
|
||||
inputTokens: 10,
|
||||
outputTokens: 2,
|
||||
cachedInputTokens: 0,
|
||||
reasoningOutputTokens: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function fakeTransportFactory(state: FakeProviderState): CapabilityLiveTransportFactory {
|
||||
|
||||
@@ -227,6 +227,21 @@ class FakeCapabilityCodexTransport implements CodexAppServerTransport {
|
||||
},
|
||||
});
|
||||
this.state.turns.set(turnId, "completed");
|
||||
this.notificationsQueue.push({
|
||||
method: "thread/tokenUsage/updated",
|
||||
params: {
|
||||
threadId: this.state.threadId,
|
||||
turnId,
|
||||
tokenUsage: {
|
||||
total: {
|
||||
inputTokens: this.state.nextTurn * 100,
|
||||
cachedInputTokens: this.state.nextTurn * 20,
|
||||
outputTokens: this.state.nextTurn * 10,
|
||||
reasoningOutputTokens: this.state.nextTurn * 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
this.notificationsQueue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: this.state.threadId, turn: { id: turnId, status: "completed" } },
|
||||
@@ -307,6 +322,10 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
expect(first.assistantText).toContain("stateRevision");
|
||||
expect(second.assistantText).toContain("stateRevision");
|
||||
expect(first.snapshot.providerThreadId).toBe(second.snapshot.providerThreadId);
|
||||
expect(second.snapshot.usageLedger.slice(-2)).toMatchObject([
|
||||
{ providerRequests: 1, inputTokens: 100, outputTokens: 10 },
|
||||
{ providerRequests: 1, inputTokens: 100, outputTokens: 10 },
|
||||
]);
|
||||
expect(first.snapshot.providerModel).toEqual({ id: "gpt-eval-test", provider: "openai" });
|
||||
expect(first.snapshot.config.requestedModel).toBe("gpt-5.4-mini-2026-03-17");
|
||||
expect(
|
||||
@@ -489,7 +508,7 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
expect(final?.usageLedger).toHaveLength(3);
|
||||
expect(final && reconcileCapabilityLiveUsage(final)).toEqual({
|
||||
providerCalls: 3,
|
||||
providerRequests: 0,
|
||||
providerRequests: 1,
|
||||
inputTokens: 280,
|
||||
outputTokens: 30,
|
||||
cachedInputTokens: 40,
|
||||
@@ -693,6 +712,7 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
const firstService = new CapabilityLiveSessionService({ store, transportOptions });
|
||||
const first = await firstService.create({
|
||||
...binding,
|
||||
workingDirectory: directory,
|
||||
attemptId: "attempt-real-killed",
|
||||
turnTimeoutMs: 2_000,
|
||||
});
|
||||
@@ -701,7 +721,7 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
const checkpoint = await store.load(binding.sessionId);
|
||||
expect(checkpoint?.mockState).toContain("One durable governed effect.");
|
||||
expect(checkpoint?.activeTurnId).toBe("turn-1");
|
||||
expect(checkpoint?.process?.runnerProcessGroupId).not.toBeNull();
|
||||
expect(checkpoint?.process?.runnerPid).not.toBeNull();
|
||||
expect(checkpoint?.process?.codexPid).not.toBeNull();
|
||||
});
|
||||
await first.recordUsage({
|
||||
@@ -714,9 +734,9 @@ describe("Capability live runnerd and Codex session", () => {
|
||||
costNanodollars: 100,
|
||||
});
|
||||
const killedCheckpoint = await store.load(binding.sessionId);
|
||||
const processGroupId = killedCheckpoint?.process?.runnerProcessGroupId;
|
||||
expect(processGroupId).toBeTypeOf("number");
|
||||
process.kill(-processGroupId!, "SIGKILL");
|
||||
const runnerPid = killedCheckpoint?.process?.runnerPid;
|
||||
expect(runnerPid).toBeTypeOf("number");
|
||||
process.kill(runnerPid!, "SIGKILL");
|
||||
await expect(killedTurn).rejects.toThrow();
|
||||
|
||||
const resumedService = new CapabilityLiveSessionService({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import type {
|
||||
CodexAppServerTransport,
|
||||
@@ -824,6 +825,7 @@ export class CapabilityLiveSession {
|
||||
#eventSeq = 0;
|
||||
#turnEventCount = 0;
|
||||
readonly #rawUsageByTurn = new Map<string, { providerRequests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; reasoningTokens: number }>();
|
||||
#latestCumulativeUsage: Record<string, unknown> = {};
|
||||
readonly #loadedOperationIds = new Set<string>();
|
||||
|
||||
private constructor(options: OpenSessionOptions & { snapshot?: CapabilityLiveSessionSnapshot }) {
|
||||
@@ -1143,6 +1145,14 @@ export class CapabilityLiveSession {
|
||||
|
||||
async #captureTurnUsage(turnId: string): Promise<void> {
|
||||
if (this.#transport === null) throw new Error("capability_live_usage_transport_missing");
|
||||
// Codex can emit rawResponse/completed immediately after turn/completed.
|
||||
// Give the notification pump a short bounded window before falling back to
|
||||
// cumulative thread usage, otherwise a fast second turn can be mislabeled
|
||||
// as missing accounting even though its receipt is already in flight.
|
||||
const usageDeadline = Date.now() + 500;
|
||||
while (!this.#rawUsageByTurn.has(turnId) && Date.now() < usageDeadline) {
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
const captured = this.#rawUsageByTurn.get(turnId);
|
||||
const read = captured === undefined ? await this.#transport.request("thread/read", {
|
||||
threadId: this.#providerThreadId,
|
||||
@@ -1150,7 +1160,10 @@ export class CapabilityLiveSession {
|
||||
}) : {};
|
||||
const thread = record(read.thread);
|
||||
const usage = record(thread.tokenUsage ?? read.tokenUsage);
|
||||
const total = record(captured ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage);
|
||||
const cumulativeNotification = Object.keys(this.#latestCumulativeUsage).length > 0
|
||||
? this.#latestCumulativeUsage
|
||||
: undefined;
|
||||
const total = record(captured ?? cumulativeNotification ?? usage.total ?? usage.totalTokenUsage ?? usage.total_token_usage);
|
||||
const integer = (...names: string[]): number => {
|
||||
for (const name of names) {
|
||||
const value = total[name];
|
||||
@@ -1165,18 +1178,32 @@ export class CapabilityLiveSession {
|
||||
reasoningTokens: integer("reasoningOutputTokens", "reasoningTokens", "reasoning_output_tokens"),
|
||||
};
|
||||
if (cumulative.inputTokens + cumulative.outputTokens === 0) {
|
||||
throw new Error("capability_live_usage_missing");
|
||||
throw new Error(`capability_live_usage_missing:${JSON.stringify({
|
||||
captured: captured !== undefined,
|
||||
readKeys: Object.keys(read).sort(),
|
||||
threadKeys: Object.keys(thread).sort(),
|
||||
usageKeys: Object.keys(usage).sort(),
|
||||
totalKeys: Object.keys(total).sort(),
|
||||
})}`);
|
||||
}
|
||||
const prior = reconcileCapabilityLiveUsage(this.snapshot());
|
||||
const delta = captured === undefined
|
||||
? {
|
||||
inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens),
|
||||
outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens),
|
||||
cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens),
|
||||
reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens),
|
||||
}
|
||||
: cumulative;
|
||||
await this.recordUsage({
|
||||
receiptId: `${turnId}:usage`,
|
||||
turnId,
|
||||
providerCalls: 1,
|
||||
providerRequests: captured?.providerRequests ?? 0,
|
||||
inputTokens: Math.max(0, cumulative.inputTokens - prior.inputTokens),
|
||||
outputTokens: Math.max(0, cumulative.outputTokens - prior.outputTokens),
|
||||
cachedInputTokens: Math.max(0, cumulative.cachedInputTokens - prior.cachedInputTokens),
|
||||
reasoningTokens: Math.max(0, cumulative.reasoningTokens - prior.reasoningTokens),
|
||||
providerRequests: captured?.providerRequests ?? 1,
|
||||
inputTokens: delta.inputTokens,
|
||||
outputTokens: delta.outputTokens,
|
||||
cachedInputTokens: delta.cachedInputTokens,
|
||||
reasoningTokens: delta.reasoningTokens,
|
||||
costNanodollars: 0,
|
||||
});
|
||||
}
|
||||
@@ -1352,8 +1379,18 @@ export class CapabilityLiveSession {
|
||||
|
||||
async #connect(resume: boolean): Promise<void> {
|
||||
this.#status = "starting";
|
||||
const identityDigest = createHash("sha256").update(this.id).digest("hex").slice(0, 20);
|
||||
const transportBundle = this.#transportFactory({
|
||||
...this.#transportOptions,
|
||||
stateDirectory: resolve(this.#config.workingDirectory, ".paperclip-runner-prp", identityDigest),
|
||||
prpIdentity: {
|
||||
runnerInstanceId: `runner_lab_${identityDigest}`,
|
||||
environmentLeaseId: `lease_lab_${identityDigest}`,
|
||||
runId: this.#authority.runId,
|
||||
normalizedSessionId: this.id,
|
||||
turnId: `turn_lab_${identityDigest}`,
|
||||
itemId: `item_lab_${identityDigest}`,
|
||||
},
|
||||
onDiagnostic: (message) => {
|
||||
this.#transportOptions.onDiagnostic?.(message);
|
||||
this.#appendEvidence("diagnostic", this.#activeTurnId, {
|
||||
@@ -1663,24 +1700,47 @@ export class CapabilityLiveSession {
|
||||
const item = record(params.item);
|
||||
const turnId = text(params.turnId, text(turn.id));
|
||||
const providerEvent = capabilityProviderEventCategory(notification.method, params);
|
||||
if (notification.method === "thread/tokenUsage/updated") {
|
||||
const tokenUsage = record(params.tokenUsage);
|
||||
this.#latestCumulativeUsage = record(
|
||||
tokenUsage.total ?? tokenUsage.totalTokenUsage ?? tokenUsage.total_token_usage ?? tokenUsage,
|
||||
);
|
||||
}
|
||||
this.#appendEvidence("provider_event", turnId || null, {
|
||||
method: notification.method,
|
||||
params: jsonValue(params),
|
||||
});
|
||||
if (notification.method === "rawResponse/completed") {
|
||||
if (notification.method === "rawResponse/completed" || notification.method === "turn/completed") {
|
||||
const rawUsage = record(params.usage);
|
||||
if (turnId.length > 0 && Object.keys(rawUsage).length > 0) {
|
||||
const previous = this.#rawUsageByTurn.get(turnId) ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 };
|
||||
// Current Codex builds may omit turnId from rawResponse/completed. Turns
|
||||
// are serialized by this session, so the active or just-terminal turn is
|
||||
// the unambiguous owner of that usage receipt.
|
||||
const usageTurnId = turnId || this.#activeTurnId || this.#terminalTurns.at(-1)?.turnId || "";
|
||||
if (usageTurnId.length > 0 && Object.keys(rawUsage).length > 0) {
|
||||
const previous = this.#rawUsageByTurn.get(usageTurnId);
|
||||
const value = (name: string): number => Number.isSafeInteger(rawUsage[name]) && Number(rawUsage[name]) >= 0 ? Number(rawUsage[name]) : 0;
|
||||
this.#rawUsageByTurn.set(turnId, {
|
||||
providerRequests: previous.providerRequests + 1,
|
||||
inputTokens: previous.inputTokens + value("inputTokens"),
|
||||
outputTokens: previous.outputTokens + value("outputTokens"),
|
||||
cachedInputTokens: previous.cachedInputTokens + value("cachedInputTokens"),
|
||||
reasoningTokens: previous.reasoningTokens + value("reasoningOutputTokens"),
|
||||
});
|
||||
if (notification.method === "rawResponse/completed") {
|
||||
const base = previous ?? { providerRequests: 0, inputTokens: 0, outputTokens: 0, cachedInputTokens: 0, reasoningTokens: 0 };
|
||||
this.#rawUsageByTurn.set(usageTurnId, {
|
||||
providerRequests: base.providerRequests + 1,
|
||||
inputTokens: base.inputTokens + value("inputTokens"),
|
||||
outputTokens: base.outputTokens + value("outputTokens"),
|
||||
cachedInputTokens: base.cachedInputTokens + value("cachedInputTokens"),
|
||||
reasoningTokens: base.reasoningTokens + value("reasoningOutputTokens"),
|
||||
});
|
||||
} else if (previous === undefined) {
|
||||
// Some provider versions publish only a terminal per-turn receipt.
|
||||
this.#rawUsageByTurn.set(usageTurnId, {
|
||||
providerRequests: 1,
|
||||
inputTokens: value("inputTokens"),
|
||||
outputTokens: value("outputTokens"),
|
||||
cachedInputTokens: value("cachedInputTokens"),
|
||||
reasoningTokens: value("reasoningOutputTokens"),
|
||||
});
|
||||
}
|
||||
}
|
||||
} else if (notification.method === "turn/started") {
|
||||
}
|
||||
if (notification.method === "turn/started") {
|
||||
if (turnId.length > 0) this.#activeTurnId = turnId;
|
||||
this.#emit({ turnId: turnId || this.#activeTurnId, kind: "activity", reason: "turn_started" });
|
||||
} else if (notification.method === "item/agentMessage/delta") {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary } from "./runnerd-codex-transport.js";
|
||||
|
||||
const fakeCodex = resolve(
|
||||
import.meta.dirname,
|
||||
"../../runner/target/debug/fake-codex-app-server",
|
||||
);
|
||||
|
||||
it("runs the lab provider boundary through authenticated durable PRP", async () => {
|
||||
const bundle = createCapabilityRunnerdCodexTransport({
|
||||
runnerBinary: defaultCapabilityRunnerdBinary(),
|
||||
codexCommand: fakeCodex,
|
||||
codexArgs: [],
|
||||
});
|
||||
bundle.transport.setServerRequestHandler(async (request) => ({
|
||||
success: true,
|
||||
contentItems: [{
|
||||
type: "inputText",
|
||||
text: JSON.stringify({ ok: true, result: { task: { title: "PRP lab task" } } }),
|
||||
}],
|
||||
}));
|
||||
try {
|
||||
await bundle.transport.request("initialize", {});
|
||||
const opened = await bundle.transport.request("thread/start", {
|
||||
cwd: tmpdir(),
|
||||
dynamicTools: [{
|
||||
name: "get_task_context",
|
||||
description: "Read the active task.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
}],
|
||||
});
|
||||
expect(opened.thread).toMatchObject({ modelProvider: "openai" });
|
||||
await bundle.transport.request("turn/start", {
|
||||
input: [{ type: "text", text: "Read the task." }],
|
||||
});
|
||||
const methods: string[] = [];
|
||||
for await (const notification of bundle.transport.notifications()) {
|
||||
methods.push(notification.method);
|
||||
if (notification.method === "turn/completed") break;
|
||||
}
|
||||
expect(methods).toContain("turn/completed");
|
||||
expect(bundle.evidence().diagnostics).toContain(
|
||||
"runnerd authenticated to the external mock control plane over durable PRP",
|
||||
);
|
||||
} finally {
|
||||
await bundle.transport.close();
|
||||
}
|
||||
expect(bundle.evidence()).toMatchObject({ runnerExited: true, runnerExitCode: 0 });
|
||||
}, 30_000);
|
||||
@@ -1,16 +1,30 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
ProcessCodexAppServerTransport,
|
||||
createSanitizedCodexEnvironment,
|
||||
type CodexAppServerTransport,
|
||||
type CodexTransportProcessInfo,
|
||||
import type {
|
||||
CodexAppServerTransport,
|
||||
CodexRpcNotification,
|
||||
CodexRpcServerRequest,
|
||||
CodexServerRequestHandler,
|
||||
CodexTransportProcessInfo,
|
||||
} from "../drivers/codex/app-server-transport.js";
|
||||
import { createSanitizedCodexEnvironment } from "../drivers/codex/app-server-transport.js";
|
||||
import { createIsolatedCodexAppServerArgs } from "../drivers/codex/codex-app-server-driver.js";
|
||||
import {
|
||||
DurableRecoveryMockCore,
|
||||
durableRecoveryInternals,
|
||||
spawnRunner,
|
||||
waitForProcess,
|
||||
type RunnerProcessHandle,
|
||||
} from "../mock-core/durable-recovery.js";
|
||||
|
||||
const packageRoot = fileURLToPath(new URL("../..", import.meta.url));
|
||||
const executableSuffix = process.platform === "win32" ? ".exe" : "";
|
||||
const MAX_NOTIFICATION_COUNT = 2_048;
|
||||
const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
export interface CapabilityRunnerdProcessEvidence {
|
||||
runnerPid: number | null;
|
||||
@@ -31,6 +45,15 @@ export interface CapabilityRunnerdCodexTransportOptions {
|
||||
closeGraceMs?: number;
|
||||
onDiagnostic?: (message: string) => void;
|
||||
onEvidence?: (evidence: Readonly<CapabilityRunnerdProcessEvidence>) => void;
|
||||
stateDirectory?: string;
|
||||
prpIdentity?: {
|
||||
runnerInstanceId: string;
|
||||
environmentLeaseId: string;
|
||||
runId: string;
|
||||
normalizedSessionId: string;
|
||||
turnId: string;
|
||||
itemId: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface CapabilityRunnerdCodexTransport {
|
||||
@@ -38,69 +61,405 @@ export interface CapabilityRunnerdCodexTransport {
|
||||
evidence(): Readonly<CapabilityRunnerdProcessEvidence>;
|
||||
}
|
||||
|
||||
export function defaultCapabilityRunnerdBinary(): string {
|
||||
return resolve(
|
||||
packageRoot,
|
||||
`runner/target/debug/paperclip-runnerd${executableSuffix}`,
|
||||
);
|
||||
class NotificationQueue implements AsyncIterable<CodexRpcNotification> {
|
||||
#values: Array<{ value: CodexRpcNotification; bytes: number }> = [];
|
||||
#waiters: Array<(value: IteratorResult<CodexRpcNotification>) => void> = [];
|
||||
#bytes = 0;
|
||||
#closed = false;
|
||||
|
||||
push(value: CodexRpcNotification): void {
|
||||
if (this.#closed) return;
|
||||
const bytes = Buffer.byteLength(JSON.stringify(value));
|
||||
const waiter = this.#waiters.shift();
|
||||
if (waiter !== undefined) {
|
||||
waiter({ value, done: false });
|
||||
return;
|
||||
}
|
||||
if (this.#values.length >= MAX_NOTIFICATION_COUNT || this.#bytes + bytes > MAX_NOTIFICATION_BYTES) {
|
||||
throw new Error("PRP provider notification queue bound exceeded");
|
||||
}
|
||||
this.#values.push({ value, bytes });
|
||||
this.#bytes += bytes;
|
||||
}
|
||||
|
||||
close(): void {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
this.#values = [];
|
||||
this.#bytes = 0;
|
||||
for (const waiter of this.#waiters.splice(0)) waiter({ value: undefined, done: true });
|
||||
}
|
||||
|
||||
[Symbol.asyncIterator](): AsyncIterator<CodexRpcNotification> {
|
||||
return {
|
||||
next: async () => {
|
||||
const queued = this.#values.shift();
|
||||
if (queued !== undefined) {
|
||||
this.#bytes -= queued.bytes;
|
||||
return { value: queued.value, done: false };
|
||||
}
|
||||
if (this.#closed) return { value: undefined, done: true };
|
||||
return new Promise((resolveValue) => this.#waiters.push(resolveValue));
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the real package-local runnerd as process owner. runnerd then starts
|
||||
* the real Codex app-server and transparently proxies its stdio JSON-RPC.
|
||||
*/
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: {};
|
||||
}
|
||||
|
||||
function commandDigest(value: unknown): string {
|
||||
return `sha256:${createHash("sha256").update(durableRecoveryInternals.canonicalJson(value)).digest("hex")}`;
|
||||
}
|
||||
|
||||
function unwrapToolResponse(response: Record<string, unknown>): unknown {
|
||||
const items = Array.isArray(response.contentItems) ? response.contentItems : [];
|
||||
const value = record(items[0]).text;
|
||||
if (typeof value !== "string") return response;
|
||||
try {
|
||||
return JSON.parse(value);
|
||||
} catch {
|
||||
return response;
|
||||
}
|
||||
}
|
||||
|
||||
class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
readonly #root: string;
|
||||
readonly #ownsRoot: boolean;
|
||||
readonly #queue = new NotificationQueue();
|
||||
readonly #startedAt = new Date().toISOString();
|
||||
readonly #evidence: CapabilityRunnerdProcessEvidence;
|
||||
#handler: CodexServerRequestHandler = async () => ({
|
||||
success: false,
|
||||
contentItems: [{ type: "inputText", text: "No Paperclip control-plane tool handler is installed." }],
|
||||
});
|
||||
#core: DurableRecoveryMockCore | null = null;
|
||||
#handle: RunnerProcessHandle | null = null;
|
||||
#pump: NodeJS.Timeout | null = null;
|
||||
#eventIndex = 0;
|
||||
#threadId = "";
|
||||
#sessionId: string | null = null;
|
||||
#turnId = "";
|
||||
#closed = false;
|
||||
|
||||
constructor(readonly options: CapabilityRunnerdCodexTransportOptions) {
|
||||
this.#ownsRoot = options.stateDirectory === undefined;
|
||||
this.#root = options.stateDirectory ?? mkdtempSync(resolve(tmpdir(), "paperclip-runner-lab-prp-"));
|
||||
mkdirSync(this.#root, { recursive: true, mode: 0o700 });
|
||||
this.#evidence = {
|
||||
runnerPid: null,
|
||||
runnerProcessGroupId: null,
|
||||
codexPid: null,
|
||||
runnerExited: false,
|
||||
runnerExitCode: null,
|
||||
runnerSignal: null,
|
||||
childEnvironmentKeys: Object.keys(createSanitizedCodexEnvironment(options.environment)).sort(),
|
||||
diagnostics: ["lab transport selected authenticated durable PRP"],
|
||||
};
|
||||
}
|
||||
|
||||
evidence(): CapabilityRunnerdProcessEvidence {
|
||||
return structuredClone(this.#evidence);
|
||||
}
|
||||
|
||||
async request(method: string, params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
if (this.#closed) throw new Error("PRP Codex transport is closed");
|
||||
if (method === "initialize") return { user: {} };
|
||||
if (method === "thread/start") return this.#start(params);
|
||||
if (method === "turn/start") return this.#startTurn(params);
|
||||
if (method === "turn/interrupt") {
|
||||
await this.#command("turn.interrupt", params);
|
||||
return {};
|
||||
}
|
||||
if (method === "thread/read") {
|
||||
if (this.#core === null) await this.#resume();
|
||||
return this.#commandResult("provider.thread.read", {});
|
||||
}
|
||||
if (method === "thread/resume") return { thread: { id: this.#threadId, sessionId: this.#sessionId } };
|
||||
throw new Error(`PRP Codex transport does not expose provider method ${method}`);
|
||||
}
|
||||
|
||||
notify(_method: string, _params?: Record<string, unknown>): void {}
|
||||
|
||||
notifications(): AsyncIterable<CodexRpcNotification> {
|
||||
return this.#queue;
|
||||
}
|
||||
|
||||
setServerRequestHandler(handler: CodexServerRequestHandler): void {
|
||||
this.#handler = handler;
|
||||
}
|
||||
|
||||
processInfo(): CodexTransportProcessInfo {
|
||||
return {
|
||||
pid: this.#evidence.runnerPid,
|
||||
processGroupId: this.#evidence.runnerProcessGroupId,
|
||||
startedAt: this.#startedAt,
|
||||
exited: this.#evidence.runnerExited,
|
||||
exitCode: this.#evidence.runnerExitCode,
|
||||
signal: this.#evidence.runnerSignal,
|
||||
};
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
this.#closed = true;
|
||||
if (this.#core !== null && this.#handle !== null) {
|
||||
this.#core.queueCommand("runner.suspend", {}, undefined, true);
|
||||
try {
|
||||
const result = await waitForProcess(this.#handle, this.options.closeGraceMs ?? 10_000);
|
||||
this.#evidence.runnerExited = true;
|
||||
this.#evidence.runnerExitCode = result.code;
|
||||
this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null;
|
||||
if (result.stderr.trim()) this.#diagnostic(result.stderr.trim().slice(-4_096));
|
||||
} catch (error) {
|
||||
this.#diagnostic(`runner shutdown failed: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
if (this.#pump !== null) clearInterval(this.#pump);
|
||||
this.#pump = null;
|
||||
this.#queue.close();
|
||||
await this.#core?.stop();
|
||||
this.#handle?.child.kill("SIGKILL");
|
||||
if (this.#ownsRoot) rmSync(this.#root, { recursive: true, force: true });
|
||||
this.#publish();
|
||||
}
|
||||
|
||||
async #start(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
if (this.#core !== null) throw new Error("PRP provider thread is already started");
|
||||
const token = randomUUID().replaceAll("-", "");
|
||||
const identity = this.options.prpIdentity ?? {
|
||||
runnerInstanceId: `runner_lab_${token}`,
|
||||
environmentLeaseId: `lease_lab_${token}`,
|
||||
runId: `run_lab_${token}`,
|
||||
normalizedSessionId: `session_lab_${token}`,
|
||||
turnId: `turn_lab_${token}`,
|
||||
itemId: `item_lab_${token}`,
|
||||
};
|
||||
const dynamicTools = Array.isArray(params.dynamicTools) ? params.dynamicTools.map(record) : [];
|
||||
const operations = dynamicTools.map((tool) => ({
|
||||
operationId: String(tool.name ?? ""),
|
||||
version: 1,
|
||||
description: String(tool.description ?? ""),
|
||||
inputSchema: record(tool.inputSchema),
|
||||
responseSchema: {},
|
||||
}));
|
||||
const core = new DurableRecoveryMockCore({
|
||||
stateDirectory: resolve(this.#root, "control-plane"),
|
||||
identity,
|
||||
fault: "none",
|
||||
onSemanticToolInput: async (call) => unwrapToolResponse(await this.#handler({
|
||||
id: call.callId,
|
||||
method: "item/tool/call",
|
||||
params: {
|
||||
threadId: this.#threadId,
|
||||
turnId: this.#turnId,
|
||||
callId: call.callId,
|
||||
tool: call.operationId,
|
||||
arguments: call.input,
|
||||
},
|
||||
})),
|
||||
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
||||
});
|
||||
this.#core = core;
|
||||
mkdirSync(resolve(this.#root, "runner"), { recursive: true, mode: 0o700 });
|
||||
core.queueCommand("run.prepare", {
|
||||
authorizedTools: {
|
||||
schema: "paperclip.runner.authorized-tools.v1",
|
||||
schemaVersion: 1,
|
||||
catalogDigest: commandDigest(operations),
|
||||
operations,
|
||||
},
|
||||
provider: {
|
||||
command: this.options.codexCommand ?? "codex",
|
||||
args: this.options.codexArgs ?? createIsolatedCodexAppServerArgs(this.options.environment),
|
||||
cwd: String(params.cwd ?? tmpdir()),
|
||||
model: typeof params.model === "string" ? params.model : null,
|
||||
instructions: String(params.baseInstructions ?? "You are a Paperclip agent."),
|
||||
},
|
||||
});
|
||||
core.queueCommand("session.open", { reuse: "same_session" });
|
||||
await core.start();
|
||||
const handle = spawnRunner({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory: resolve(this.#root, "runner"),
|
||||
identity,
|
||||
ticket: core.issueBootstrapTicket(),
|
||||
maxOutboxBytes: 256 * 1024,
|
||||
p0ReserveBytes: 64 * 1024,
|
||||
maxRuntimeMs: 60 * 60 * 1_000,
|
||||
runnerBinaryPath: this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(),
|
||||
});
|
||||
this.#handle = handle;
|
||||
this.#evidence.runnerPid = handle.child.pid ?? null;
|
||||
this.#evidence.runnerProcessGroupId = null;
|
||||
this.#publish();
|
||||
this.#pump = setInterval(() => this.#pumpEvents(), 5);
|
||||
await this.#waitCommand("run.prepare");
|
||||
await this.#waitCommand("session.open");
|
||||
await this.#waitForProviderIdentity();
|
||||
this.#diagnostic("runnerd authenticated to the external mock control plane over durable PRP");
|
||||
return { thread: { id: this.#threadId, sessionId: this.#sessionId, model: params.model, modelProvider: "openai" } };
|
||||
}
|
||||
|
||||
async #resume(): Promise<void> {
|
||||
const identity = this.options.prpIdentity;
|
||||
if (identity === undefined || !existsSync(resolve(this.#root, "runner", "runner-state.json"))) {
|
||||
throw new Error("PRP provider resume state is unavailable");
|
||||
}
|
||||
const core = new DurableRecoveryMockCore({
|
||||
stateDirectory: resolve(this.#root, "control-plane"),
|
||||
identity,
|
||||
fault: "none",
|
||||
onSemanticToolInput: async (call) => unwrapToolResponse(await this.#handler({
|
||||
id: call.callId,
|
||||
method: "item/tool/call",
|
||||
params: {
|
||||
threadId: this.#threadId,
|
||||
turnId: this.#turnId,
|
||||
callId: call.callId,
|
||||
tool: call.operationId,
|
||||
arguments: call.input,
|
||||
},
|
||||
})),
|
||||
connectionLeaseTtlMs: 60 * 60 * 1_000,
|
||||
});
|
||||
this.#core = core;
|
||||
this.#eventIndex = core.store.state.committedEvents.length;
|
||||
await core.start();
|
||||
const handle = spawnRunner({
|
||||
connectUrl: core.connectUrl,
|
||||
stateDirectory: resolve(this.#root, "runner"),
|
||||
identity,
|
||||
ticket: core.issueBootstrapTicket(),
|
||||
maxOutboxBytes: 256 * 1024,
|
||||
p0ReserveBytes: 64 * 1024,
|
||||
maxRuntimeMs: 60 * 60 * 1_000,
|
||||
runnerBinaryPath: this.options.runnerBinary ?? defaultCapabilityRunnerdBinary(),
|
||||
});
|
||||
this.#handle = handle;
|
||||
this.#evidence.runnerPid = handle.child.pid ?? null;
|
||||
this.#evidence.runnerProcessGroupId = null;
|
||||
this.#publish();
|
||||
this.#pump = setInterval(() => this.#pumpEvents(), 5);
|
||||
await this.#waitForProviderIdentity();
|
||||
this.#diagnostic("runnerd restored its durable PRP session and provider thread");
|
||||
}
|
||||
|
||||
async #startTurn(params: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
const input = Array.isArray(params.input) ? params.input.map(record) : [];
|
||||
const message = input.map((item) => typeof item.text === "string" ? item.text : "").join("\n");
|
||||
this.#turnId = `turn_lab_${randomUUID().replaceAll("-", "")}`;
|
||||
await this.#command("turn.start", { text: message });
|
||||
return { turn: { id: this.#turnId, status: "inProgress" } };
|
||||
}
|
||||
|
||||
async #command(type: string, payload: Record<string, unknown>): Promise<void> {
|
||||
const core = this.#core;
|
||||
if (core === null) throw new Error("PRP provider thread is not started");
|
||||
const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`;
|
||||
core.queueCommand(type, payload, commandId, true);
|
||||
await this.#waitCommand(type, commandId);
|
||||
}
|
||||
|
||||
async #commandResult(type: string, payload: Record<string, unknown>): Promise<Record<string, unknown>> {
|
||||
const core = this.#core;
|
||||
if (core === null) throw new Error("PRP provider thread is not started");
|
||||
const commandId = `command_lab_${randomUUID().replaceAll("-", "")}`;
|
||||
core.queueCommand(type, payload, commandId, true);
|
||||
await this.#waitCommand(type, commandId);
|
||||
const deadline = Date.now() + 30_000;
|
||||
while (Date.now() < deadline) {
|
||||
const event = core.store.state.committedEvents.find((candidate) => {
|
||||
if (candidate.eventType !== "provider.rpc_result") return false;
|
||||
return record(record(candidate.envelope.payload).payload).commandId === commandId;
|
||||
});
|
||||
const result = record(record(record(event?.envelope.payload).payload).result);
|
||||
if (event !== undefined) return result;
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
throw new Error(`PRP command ${type} omitted its provider result`);
|
||||
}
|
||||
|
||||
async #waitForProviderIdentity(): Promise<void> {
|
||||
const deadline = Date.now() + 30_000;
|
||||
while (Date.now() < deadline) {
|
||||
this.#pumpEvents();
|
||||
if (this.#threadId.length > 0 && this.#evidence.codexPid !== null) return;
|
||||
if (this.#handle?.child.exitCode !== null) throw new Error("runnerd exited before provider startup");
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
throw new Error("runnerd did not report its provider identity");
|
||||
}
|
||||
|
||||
async #waitCommand(type: string, commandId?: string): Promise<void> {
|
||||
const deadline = Date.now() + 30_000;
|
||||
while (Date.now() < deadline) {
|
||||
const command = this.#core?.store.state.commands.find((candidate) =>
|
||||
commandId === undefined ? candidate.type === type : candidate.commandId === commandId
|
||||
);
|
||||
if (command?.status === "completed") return;
|
||||
if (command !== undefined && command.status !== "pending") {
|
||||
throw new Error(`PRP command ${type} ${command.status}: ${JSON.stringify(command.result)}`);
|
||||
}
|
||||
if (this.#handle?.child.exitCode !== null) throw new Error(`runnerd exited while waiting for ${type}`);
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
throw new Error(`PRP command ${type} timed out`);
|
||||
}
|
||||
|
||||
#pumpEvents(): void {
|
||||
const events = this.#core?.store.state.committedEvents ?? [];
|
||||
while (this.#eventIndex < events.length) {
|
||||
const event = events[this.#eventIndex++];
|
||||
if (event.eventType === "provider.started") {
|
||||
const started = record(record(event.envelope.payload).payload);
|
||||
const pid = started.pid;
|
||||
const threadId = started.threadId;
|
||||
const sessionId = started.sessionId;
|
||||
if (typeof pid === "number") {
|
||||
this.#evidence.codexPid = pid;
|
||||
}
|
||||
if (typeof threadId === "string") this.#threadId = threadId;
|
||||
if (typeof sessionId === "string") this.#sessionId = sessionId;
|
||||
this.#publish();
|
||||
continue;
|
||||
}
|
||||
if (event.eventType !== "provider.event" && event.eventType !== "turn.completed") continue;
|
||||
const payload = record(record(event.envelope.payload).payload);
|
||||
const method = payload.method;
|
||||
if (typeof method !== "string") continue;
|
||||
const params = record(payload.params);
|
||||
if (method === "turn/started") {
|
||||
const providerTurnId = record(params.turn).id;
|
||||
if (typeof providerTurnId === "string") this.#turnId = providerTurnId;
|
||||
}
|
||||
this.#queue.push({ method, params });
|
||||
}
|
||||
}
|
||||
|
||||
#diagnostic(message: string): void {
|
||||
this.#evidence.diagnostics.push(message);
|
||||
if (this.#evidence.diagnostics.length > 64) this.#evidence.diagnostics.shift();
|
||||
this.options.onDiagnostic?.(message);
|
||||
this.#publish();
|
||||
}
|
||||
|
||||
#publish(): void {
|
||||
this.options.onEvidence?.(this.evidence());
|
||||
}
|
||||
}
|
||||
|
||||
export function defaultCapabilityRunnerdBinary(): string {
|
||||
return resolve(packageRoot, `runner/target/debug/paperclip-runnerd${executableSuffix}`);
|
||||
}
|
||||
|
||||
/** Uses the same external mock control-plane -> authenticated PRP -> runnerd -> provider topology as evals. */
|
||||
export function createCapabilityRunnerdCodexTransport(
|
||||
options: CapabilityRunnerdCodexTransportOptions = {},
|
||||
): CapabilityRunnerdCodexTransport {
|
||||
const environment = createSanitizedCodexEnvironment(options.environment);
|
||||
const evidence: CapabilityRunnerdProcessEvidence = {
|
||||
runnerPid: null,
|
||||
runnerProcessGroupId: null,
|
||||
codexPid: null,
|
||||
runnerExited: false,
|
||||
runnerExitCode: null,
|
||||
runnerSignal: null,
|
||||
childEnvironmentKeys: Object.keys(environment).sort(),
|
||||
diagnostics: [],
|
||||
};
|
||||
const publish = () => options.onEvidence?.(structuredClone(evidence));
|
||||
const diagnostic = (message: string) => {
|
||||
evidence.diagnostics.push(message);
|
||||
if (evidence.diagnostics.length > 64) evidence.diagnostics.shift();
|
||||
const match = /(?:capability|native) codex proxy started runner_pid=(\d+) codex_pid=(\d+)/.exec(message);
|
||||
if (match) {
|
||||
evidence.runnerPid = Number(match[1]);
|
||||
evidence.codexPid = Number(match[2]);
|
||||
}
|
||||
options.onDiagnostic?.(message);
|
||||
publish();
|
||||
};
|
||||
const processUpdate = (info: CodexTransportProcessInfo) => {
|
||||
evidence.runnerPid = info.pid;
|
||||
evidence.runnerProcessGroupId = info.processGroupId;
|
||||
evidence.runnerExited = info.exited;
|
||||
evidence.runnerExitCode = info.exitCode;
|
||||
evidence.runnerSignal = info.signal;
|
||||
publish();
|
||||
};
|
||||
const codexArgs = options.codexArgs ?? createIsolatedCodexAppServerArgs(options.environment);
|
||||
const runnerArgs = [
|
||||
"--codex-app-server-proxy",
|
||||
"--codex-command",
|
||||
options.codexCommand ?? "codex",
|
||||
...codexArgs.flatMap((argument) => ["--codex-arg", argument]),
|
||||
];
|
||||
const transport = new ProcessCodexAppServerTransport({
|
||||
command: options.runnerBinary ?? defaultCapabilityRunnerdBinary(),
|
||||
args: runnerArgs,
|
||||
environment,
|
||||
processGroup: true,
|
||||
closeGraceMs: options.closeGraceMs,
|
||||
onDiagnostic: diagnostic,
|
||||
onProcess: processUpdate,
|
||||
});
|
||||
return {
|
||||
transport,
|
||||
evidence: () => structuredClone(evidence),
|
||||
};
|
||||
const transport = new DurablePrpCodexTransport(options);
|
||||
return { transport, evidence: () => transport.evidence() };
|
||||
}
|
||||
@@ -103,6 +103,7 @@ class GatedTransport implements CodexAppServerTransport {
|
||||
return { turn: { id: "turn-1", status: "inProgress" } };
|
||||
}
|
||||
if (method === "turn/interrupt") {
|
||||
this.queue.push({ method: "rawResponse/completed", params: { threadId: "provider-thread", turnId: String(params.turnId), usage: { inputTokens: 3, outputTokens: 1, cachedInputTokens: 0, reasoningOutputTokens: 0 } } });
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: "provider-thread", turn: { id: String(params.turnId), status: "interrupted" } },
|
||||
@@ -150,6 +151,7 @@ class GatedTransport implements CodexAppServerTransport {
|
||||
item: { id: "message-1", type: "agentMessage", text: FINAL_TEXT },
|
||||
},
|
||||
});
|
||||
this.queue.push({ method: "rawResponse/completed", params: { threadId: "provider-thread", turnId, usage: { inputTokens: 3, outputTokens: 1, cachedInputTokens: 0, reasoningOutputTokens: 0 } } });
|
||||
this.queue.push({
|
||||
method: "turn/completed",
|
||||
params: { threadId: "provider-thread", turn: { id: turnId, status: "completed" } },
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
DurableRecoveryMockCore,
|
||||
durableRecoveryInternals,
|
||||
runDurableRecoveryRecovery,
|
||||
runDurableToolBridgeConformance,
|
||||
} from "./durable-recovery.js";
|
||||
|
||||
async function upgradeSocket(url: string): Promise<Socket> {
|
||||
@@ -365,6 +366,19 @@ async function runRunnerProcess(options: {
|
||||
}
|
||||
|
||||
describe.sequential("Durable transport and recovery", () => {
|
||||
it("repeatedly completes the provider bridge and explicit runner shutdown", async () => {
|
||||
for (let attempt = 0; attempt < 10; attempt += 1) {
|
||||
const result = await runDurableToolBridgeConformance();
|
||||
expect(result, `bridge attempt ${attempt + 1}`).toMatchObject({
|
||||
operationId: "get_task_context",
|
||||
toolInputCommitted: true,
|
||||
toolResultCompleted: true,
|
||||
providerTurnCompleted: true,
|
||||
runnerExitCode: 0,
|
||||
});
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it("accepts a short-lived bootstrap ticket once and rejects its reuse", async () => {
|
||||
const scratch =
|
||||
process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
|
||||
|
||||
@@ -183,14 +183,14 @@ interface MockCoreOptions {
|
||||
connectionLeaseTtlMs?: number;
|
||||
}
|
||||
|
||||
interface RunnerProcessResult {
|
||||
export interface RunnerProcessResult {
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
}
|
||||
|
||||
interface RunnerProcessHandle {
|
||||
export interface RunnerProcessHandle {
|
||||
child: ChildProcessWithoutNullStreams;
|
||||
completion: Promise<RunnerProcessResult>;
|
||||
}
|
||||
@@ -1342,7 +1342,7 @@ function runnerEnvironment(ticket: string): NodeJS.ProcessEnv {
|
||||
return environment;
|
||||
}
|
||||
|
||||
function spawnRunner(options: {
|
||||
export function spawnRunner(options: {
|
||||
connectUrl: string;
|
||||
stateDirectory: string;
|
||||
identity: DurableRecoveryIdentity;
|
||||
@@ -1406,7 +1406,7 @@ function spawnRunner(options: {
|
||||
return { child, completion };
|
||||
}
|
||||
|
||||
async function waitForProcess(
|
||||
export async function waitForProcess(
|
||||
handle: RunnerProcessHandle,
|
||||
timeoutMs = 15_000,
|
||||
): Promise<RunnerProcessResult> {
|
||||
@@ -1617,6 +1617,57 @@ export interface DurableEvalSessionInput {
|
||||
toolExposure?: "eager" | "lazy";
|
||||
}
|
||||
|
||||
export type DurableEvalInfrastructureFailureClass =
|
||||
| "provider_turn_timeout"
|
||||
| "runner_shutdown_timeout"
|
||||
| "runner_exit_failure";
|
||||
|
||||
export class DurableEvalInfrastructureError extends Error {
|
||||
readonly failureClass: DurableEvalInfrastructureFailureClass;
|
||||
readonly retryable: boolean;
|
||||
readonly diagnostics: Record<string, unknown>;
|
||||
|
||||
constructor(
|
||||
failureClass: DurableEvalInfrastructureFailureClass,
|
||||
message: string,
|
||||
retryable: boolean,
|
||||
diagnostics: Record<string, unknown>,
|
||||
) {
|
||||
super(message);
|
||||
this.name = "DurableEvalInfrastructureError";
|
||||
this.failureClass = failureClass;
|
||||
this.retryable = retryable;
|
||||
this.diagnostics = diagnostics;
|
||||
}
|
||||
}
|
||||
|
||||
function evalLifecycleDiagnostics(
|
||||
core: DurableRecoveryMockCore,
|
||||
handle: RunnerProcessHandle,
|
||||
): Record<string, unknown> {
|
||||
const commands = core.store.state.commands.map((command) => ({
|
||||
commandId: command.commandId,
|
||||
type: command.type,
|
||||
status: command.status,
|
||||
deliveryCount: core.store.state.commandDeliveryCounts[command.commandId] ?? 0,
|
||||
}));
|
||||
const events = core.store.state.committedEvents.slice(-20).map((event) => ({
|
||||
sourceSeq: event.sourceSeq,
|
||||
eventType: event.eventType,
|
||||
priority: event.priority,
|
||||
deliveryCount: event.deliveryCount,
|
||||
}));
|
||||
return {
|
||||
runnerPid: handle.child.pid ?? null,
|
||||
runnerExitCode: handle.child.exitCode,
|
||||
runnerSignal: handle.child.signalCode,
|
||||
connectionCount: core.store.state.connectionCount,
|
||||
ackedSourceSeq: core.store.state.ackedSourceSeq,
|
||||
commands,
|
||||
recentEvents: events,
|
||||
};
|
||||
}
|
||||
|
||||
/** Production-topology eval: the only Paperclip process in the sandbox is runnerd. */
|
||||
export async function runDurableEvalSession(input: DurableEvalSessionInput): Promise<Record<string, unknown>> {
|
||||
const identity = {
|
||||
@@ -1722,11 +1773,33 @@ export async function runDurableEvalSession(input: DurableEvalSessionInput): Pro
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 20));
|
||||
}
|
||||
if (!core.store.state.committedEvents.some((event) => event.eventType === "turn.completed")) {
|
||||
throw new Error(`durable provider turn timed out after ${input.turnTimeoutMs}ms (state: ${root})`);
|
||||
throw new DurableEvalInfrastructureError(
|
||||
"provider_turn_timeout",
|
||||
`durable provider turn timed out after ${input.turnTimeoutMs}ms`,
|
||||
true,
|
||||
evalLifecycleDiagnostics(core, handle),
|
||||
);
|
||||
}
|
||||
core.queueCommand("runner.shutdown", {}, undefined, true);
|
||||
const exited = await waitForProcess(handle, 30_000);
|
||||
if (exited.code !== 0) throw new Error(`runnerd exited ${String(exited.code)}: ${exited.stderr}`);
|
||||
let exited: RunnerProcessResult;
|
||||
try {
|
||||
exited = await waitForProcess(handle, 30_000);
|
||||
} catch (error) {
|
||||
throw new DurableEvalInfrastructureError(
|
||||
"runner_shutdown_timeout",
|
||||
`runnerd did not exit within 30000ms after runner.shutdown: ${error instanceof Error ? error.message : String(error)}`,
|
||||
true,
|
||||
evalLifecycleDiagnostics(core, handle),
|
||||
);
|
||||
}
|
||||
if (exited.code !== 0) {
|
||||
throw new DurableEvalInfrastructureError(
|
||||
"runner_exit_failure",
|
||||
`runnerd exited ${String(exited.code)}: ${exited.stderr}`,
|
||||
false,
|
||||
{ ...evalLifecycleDiagnostics(core, handle), stderr: exited.stderr.slice(-16_384) },
|
||||
);
|
||||
}
|
||||
const records = core.store.state.committedEvents.map((committed) => committed.envelope.payload as Record<string, unknown>);
|
||||
const evidence: Array<Record<string, unknown>> = [{
|
||||
id: "exposure-1", kind: "tool_exposure", at: new Date().toISOString(), turnId: null,
|
||||
|
||||
@@ -407,7 +407,7 @@ async function readEventStreamReplay(url: string, sessionId: string): Promise<st
|
||||
if (reader === undefined) throw new Error("event stream response did not include a body");
|
||||
let result = "";
|
||||
try {
|
||||
while (!result.includes("data:")) {
|
||||
while (!result.includes('"tokenBudget":4096')) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
result += new TextDecoder().decode(chunk.value, { stream: true });
|
||||
|
||||
@@ -683,7 +683,9 @@ export class LiveConsoleDemoServer {
|
||||
if (!output.overflow) response.write(`data: ${output.serialized}\n\n`);
|
||||
}
|
||||
entry.subscribers.add(response);
|
||||
request.once("close", () => entry.subscribers.delete(response));
|
||||
const unsubscribe = () => entry.subscribers.delete(response);
|
||||
request.once("close", unsubscribe);
|
||||
response.once("close", unsubscribe);
|
||||
return;
|
||||
}
|
||||
if (request.method !== "POST") {
|
||||
@@ -823,7 +825,13 @@ export class LiveConsoleDemoServer {
|
||||
if (entry.events.length >= MAX_BROWSER_EVENTS) entry.events.shift();
|
||||
entry.events.push(event);
|
||||
const serialized = `data: ${output.serialized}\n\n`;
|
||||
for (const subscriber of entry.subscribers) subscriber.write(serialized);
|
||||
for (const subscriber of entry.subscribers) {
|
||||
if (subscriber.destroyed || subscriber.writableEnded) {
|
||||
entry.subscribers.delete(subscriber);
|
||||
continue;
|
||||
}
|
||||
subscriber.write(serialized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -483,6 +483,7 @@ export function validateJsonSchema(schema: CapabilityJsonSchema, value: Capabili
|
||||
}
|
||||
|
||||
function matchesType(type: NonNullable<CapabilityJsonSchema["type"]>, value: CapabilityJsonValue): boolean {
|
||||
if (Array.isArray(type)) return type.some((candidate) => matchesType(candidate, value));
|
||||
switch (type) {
|
||||
case "null": return value === null;
|
||||
case "array": return Array.isArray(value);
|
||||
|
||||
Reference in new issue
Block a user