From a95fe6cabb8e2188efeecf17c6ce6a5f176861c3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 7 Oct 2026 09:16:05 -0500 Subject: [PATCH] fix(hermes): bound cold native admission separately Give Hermes native session opening a 60-second bound for verified private runtime copying and ACP initialization. Allow controller startup and per-turn restoration overhead while preserving ordinary command and stop deadlines. Add delayed-admission regressions and credential-free Linux native fixture CI. Record the failed final-head paid campaign without promoting qualification. Co-Authored-By: Paperclip --- .github/workflows/runner-hermes-native.yml | 132 ++++++++++++++++++ doc/plans/2026-10-06-hermes-native-runner.md | 74 +++++++++- packages/paperclip-runner/docs/hermes.md | 13 ++ .../runner-core/src/acpx_provider_session.rs | 12 +- .../runner-core/src/acpx_sidecar_transport.rs | 34 ++++- .../runner-core/src/bin/fake-acpx-sidecar.rs | 6 +- .../tests/acpx_provider_session.rs | 13 ++ .../tests/acpx_sidecar_transport.rs | 32 +++++ .../src/live/runnerd-codex-transport.ts | 78 ++++++++--- .../live/runnerd-command-admission.test.ts | 48 +++++++ tests/runner-e2e/workflow-security.test.ts | 17 +++ 11 files changed, 430 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/runner-hermes-native.yml create mode 100644 packages/paperclip-runner/src/live/runnerd-command-admission.test.ts diff --git a/.github/workflows/runner-hermes-native.yml b/.github/workflows/runner-hermes-native.yml new file mode 100644 index 00000000000..c9067f98fa3 --- /dev/null +++ b/.github/workflows/runner-hermes-native.yml @@ -0,0 +1,132 @@ +name: Hermes Native Transport + +on: + pull_request: + paths: + - .github/workflows/runner-hermes-native.yml + - packages/paperclip-runner/src/providers/hermes/** + - packages/paperclip-runner/src/drivers/acpx/** + - packages/paperclip-runner/src/live/runnerd-*.ts + - packages/paperclip-runner/runner/crates/runner-core/src/acpx_*.rs + - packages/paperclip-runner/scripts/*hermes* + - packages/paperclip-runner/acpx-profiles.json + - tests/runner-e2e/provision-hermes-linux.sh + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: hermes-native-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + native_transport: + name: Hermes native fixture (Linux amd64, no credentials) + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + package-manager-cache: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + NPM_CONFIG_UPDATE_NOTIFIER: "false" + with: + version: 9.15.4 + + - name: Install workspace dependencies + run: | + set -euo pipefail + if ! pnpm install --frozen-lockfile; then + pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile + pnpm install --frozen-lockfile + fi + + - name: Restore Rust dependencies (read only) + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: packages/paperclip-runner/runner -> target + cache-workspace-crates: false + cache-bin: false + save-if: false + + - name: Build verified runner and provider entrypoints + run: | + set -euo pipefail + pnpm --filter @paperclipai/paperclip-runner build:typescript + pnpm --filter @paperclipai/paperclip-runner build:binary + + - name: Provision the pinned Python closure without credentials + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends python3-venv bubblewrap + bash tests/runner-e2e/provision-hermes-linux.sh + + - name: Run native fixtures against a deterministic loopback model + id: fixtures + working-directory: packages/paperclip-runner + run: | + set -euo pipefail + mkdir -m 700 "$RUNNER_TEMP/hermes-fixture-home" + # No paid environment, secrets, or credentials reach either fixture. + # The execution host must pass the real bubblewrap namespace probe. + env -i PATH="$PATH" HOME="$RUNNER_TEMP/hermes-fixture-home" \ + PAPERCLIP_HERMES_QUALIFY=1 \ + node --test --test-concurrency=1 scripts/qualify-hermes-runtime.test.mjs \ + scripts/qualify-hermes-runnerd.test.mjs \ + 2>&1 | tee "$RUNNER_TEMP/hermes-native-fixture.log" + + - name: Record source and runtime provenance + if: always() + env: + HERMES_FIXTURE_OUTCOME: ${{ steps.fixtures.outcome }} + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/hermes-native-evidence" + if [ -f "$RUNNER_TEMP/hermes-native-fixture.log" ]; then + cp "$RUNNER_TEMP/hermes-native-fixture.log" "$RUNNER_TEMP/hermes-native-evidence/fixture.log" + fi + node --input-type=module <<'NODE' + import { readFile, writeFile } from 'node:fs/promises'; + import { createHash } from 'node:crypto'; + import { execFileSync } from 'node:child_process'; + import { join } from 'node:path'; + const root = 'packages/paperclip-runner'; + const version = JSON.parse(await readFile(join(root, 'src/providers/hermes/version.json'), 'utf8')); + const manifest = JSON.parse(await readFile(join(root, 'provider-assets/hermes/linux-x64/manifest.json'), 'utf8').catch(() => 'null')); + const evidence = { + qualification: 'native transport fixture only; no paid model, browser or Daytona proof', + sourceSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(), + platform: `${process.platform}-${process.arch}`, node: process.version, + fixtureOutcome: process.env.HERMES_FIXTURE_OUTCOME, + kernel: execFileSync('uname', ['-r'], { encoding: 'utf8' }).trim(), + osRelease: await readFile('/etc/os-release', 'utf8'), + runtime: version, + closureSha256: manifest ? createHash('sha256').update(JSON.stringify(manifest.entries)).digest('hex') : null, + files: manifest?.entries?.length ?? null, + model: 'deterministic loopback fixture', credentials: 'none', + }; + await writeFile(join(process.env.RUNNER_TEMP, 'hermes-native-evidence/provenance.json'), JSON.stringify(evidence, null, 2) + '\n'); + NODE + + - name: Upload fixture evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: hermes-native-linux-amd64-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/hermes-native-evidence/ + if-no-files-found: warn + retention-days: 14 diff --git a/doc/plans/2026-10-06-hermes-native-runner.md b/doc/plans/2026-10-06-hermes-native-runner.md index a2c08970c11..3285366057a 100644 --- a/doc/plans/2026-10-06-hermes-native-runner.md +++ b/doc/plans/2026-10-06-hermes-native-runner.md @@ -23,7 +23,8 @@ Keep the existing Hermes local/gateway adapters compatible. - [x] Complete final Rust/regression checks and resolve or classify failures. - [ ] Browser acceptance, Linux/Daytona execution and connection-method qualification. - [x] Reviewable draft PR stack, green CI and fresh Greptile 5/5 on the implementation heads. -- [ ] Complete the final local aggregate test invocation and release qualification. +- [x] Complete the final local aggregate test invocation and classify its failures. +- [ ] Complete live release qualification. ## Qualification evidence @@ -60,8 +61,8 @@ because fixtures pass. Native cron and gateway messaging are disabled. - Pending Hermes choice in existing runner configuration; existing connection, model, permission and transcript components. Ten Product E2E candidate cells - (five local, five Daytona) are registered. Four local cases have passing paid - attempts; the complete release matrix has not passed. + (five local, five Daytona) are registered. All five local cases have passing + paid attempts across different heads; the complete release matrix has not passed. ## Evidence and outstanding release gates @@ -342,3 +343,70 @@ cross-task memory/learned skills, routine firing and deduplication, and actual Linux amd64/Daytona execution and restoration. Exact approved Connection names and a compatible remote execution environment are still needed. The existing standalone Hermes local/gateway adapters retain their contracts. + +### Final-head campaign and startup follow-up, 2026-10-07 + +The campaign `hermes-local-paid-20261007-final-head` correctly recorded source +`a54d04785b62e2190f4e13647bf75ae510449d98` and passed **1/5** cases. It used +the same managed OpenRouter account, exact model and macOS browser/Runner path: + +| Case | Result | Failure or limitation | +| --- | --- | --- | +| Hello/completion | Pass | Done, exact final answer and cleanup passed | +| Question/resume | Fail | `session.open` exceeded its 30-second command deadline; cleanup passed | +| Plan/approve/complete | Fail | Reasoning streamed but the 120-second turn deadline expired; cleanup passed | +| Structured question/restart/resume | Fail | The isolated server hit system `ENFILE`; cleanup failed | +| File edit/validation | Fail | Dispatch was interrupted after the resource failure; process-group cleanup identity was uncertain | + +The file case's underlying task later reached success, but its independent +fixture oracle and cleanup did not pass. Its recorded failure is unchanged. +The owned launcher and remaining isolated server/database were retired with +verified exit. Failed private recovery roots remain preserved because their +original cleanup receipts failed. No additional paid campaign was launched. +The numeric budget receipt reports $4.539181805 remaining under the key's $5 +limit; this is shared-key accounting, not exact per-run billed cost. + +Credential-free production-host probes reproduced variable cold-start latency. +The first verified 445 MB runtime copy took 3.08 seconds. A complete admission +later took 33.69 seconds: 16.54 seconds for the verified private copy and 16.61 +seconds for native initialization/ACP handshake. Its no-auth loopback endpoint +received only model/backend metadata probes, with no inference requests. +These measurements reproduce an admission deadline problem; they do not prove +the cause of the host's file-table exhaustion. The read-only host counter still +reported 461,999 open files against a 491,520 limit. No system limit was changed +and no unrelated process was stopped. + +Hermes now has a separate 60-second native session-open deadline. The PRP +controller allows 75 seconds around cold admission, recovery, and later-turn +restoration. Ordinary commands and stop retain their prior deadlines, and the +post-acceptance turn-start event deadline remains 30 seconds. The 120-second +Product E2E turn oracle is unchanged. Focused regressions verify delayed startup, +ordinary-command timeout, fail-closed transport reuse and the finite outer +deadline. This addresses admission timing only; the paid planning timeout and +host resource failures still require a new passing campaign on a reliable host. + +Startup verification passed seven TypeScript deadline regressions, the complete +193-test controller transport selection, 27 focused Rust session/transport test +executions, and the complete 655-execution Rust workspace suite (two ignored). +Runner TypeScript/Rust typechecks, verified entrypoint builds, release binary +build, workflow authority tests (14), and actionlint pass. +Both rebuilt native production fixtures also pass: 35.98 seconds through Rust +PRP/sidecar and 144.68 seconds through the ACPX host. They use the deterministic +no-auth model fixture and retain their original assertions and deadlines. + +The new credential-free `Hermes Native Transport` PR workflow provisions the +pinned Linux amd64 closure and runs both production native fixtures on an +ephemeral Ubuntu host. Provider children receive an empty environment plus the +fixture PATH/home/opt-in flag. No paid environment or credentials are available. +It retains source/runtime provenance and fixture logs as CI artifacts while +excluding private native homes. Its actual execution result must be checked; +the workflow declaration alone is not Linux proof. The protected paid workflow +and its default-branch authorization remain unchanged by this addition. + +The three draft PRs had green CI and fresh Greptile 5/5 at routines +`3745f3c5a46bda7778ee132682d1b7ae23b088c1`, native +`0dea682031f8e35631faee7a05519ed4dce80d07`, and qualification +`a54d04785b62e2190f4e13647bf75ae510449d98`. That qualification CI initially +failed an unchanged signoff mock-heartbeat browser case; inspection and the +failed-job-only rerun passed. Startup changes require new checks and review. +Hermes remains pending qualification throughout. diff --git a/packages/paperclip-runner/docs/hermes.md b/packages/paperclip-runner/docs/hermes.md index 34102634ae7..a4b556cd41c 100644 --- a/packages/paperclip-runner/docs/hermes.md +++ b/packages/paperclip-runner/docs/hermes.md @@ -65,6 +65,13 @@ working copy and assigned tool bindings. The session still pins prompt, bundle, skill, connection, model, and permission identities. Unknown policy changes and changes within the same run are rejected. +Native session admission has a 60-second bound for the verified private Python +copy and ACP initialization. The controller allows 75 seconds around cold +startup, recovery and later-turn restoration. Once a turn is accepted, its +start-event deadline remains 30 seconds. Ordinary commands, cancellation and +process cleanup retain their existing bounds; this startup allowance does not +extend the task's execution deadline. + ## Interaction contract - Reasoning and assistant text use distinct message identities. Native tool @@ -141,3 +148,9 @@ Hermes cells using a managed OpenRouter connection. Each connection method, native control, attachment, persistence, remote restoration and permission mode must pass its release criterion with inspectable live evidence before promotion. The implementation record is included in that qualification PR. + +The `Hermes Native Transport` PR workflow runs the same native fixtures on Linux +amd64 with a stripped provider environment and deterministic loopback endpoints. +Its CI artifacts include exact source/runtime provenance and fixture logs. This +is credential-free transport and host-sandbox evidence; it does not replace +paid-provider, browser, subscription or Daytona qualification. diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs index 4b787104bf8..57ef63f268b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_provider_session.rs @@ -1287,10 +1287,14 @@ fn bootstrap( )?; verify_initialize_response(&initialized, transport.process_id())?; - let opened = transport.request( - GeneratedAcpxSidecarCommand::SessionOpen, - session_open_params(config, &sidecar_tools), - )?; + let open_params = session_open_params(config, &sidecar_tools); + let opened = if config.agent == "hermes" { + // The pinned Python closure is verified and privately copied before + // native initialization. This also applies to per-turn restoration. + transport.open_session(open_params, Duration::from_secs(60))? + } else { + transport.request(GeneratedAcpxSidecarCommand::SessionOpen, open_params)? + }; let identity = verify_open_response(&opened, transport.process_id(), config)?; let turn_controls = verified_turn_controls(opened.get("turnControls"), &config.agent)?; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index ca6a563d101..6eb573db09a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -224,13 +224,42 @@ impl AcpxSidecarTransport { &mut self, command: GeneratedAcpxSidecarCommand, params: Value, + ) -> Result { + self.request_with_timeout(command, params, self.request_timeout) + } + + /// Cold native admission has a separate bound; later commands retain the + /// configured transport deadline, including cancellation and shutdown. + pub fn open_session( + &mut self, + params: Value, + startup_timeout: Duration, + ) -> Result { + if startup_timeout < Duration::from_millis(1) || startup_timeout > Duration::from_secs(120) + { + return Err(LocalRunnerError::invalid( + "ACPX session startup timeout must be in the range 1 ms through 120 s", + )); + } + self.request_with_timeout( + GeneratedAcpxSidecarCommand::SessionOpen, + params, + startup_timeout, + ) + } + + fn request_with_timeout( + &mut self, + command: GeneratedAcpxSidecarCommand, + params: Value, + timeout: Duration, ) -> Result { if self.poisoned { return Err(LocalRunnerError::invalid( "ACPX sidecar transport is unavailable after a protocol failure", )); } - let result = self.request_inner(command, params); + let result = self.request_inner(command, params, timeout); match result { Ok(CommandOutcome::Success(value)) => Ok(value), Ok(CommandOutcome::Rejected(error)) => Err(error), @@ -277,6 +306,7 @@ impl AcpxSidecarTransport { &mut self, command: GeneratedAcpxSidecarCommand, params: Value, + timeout: Duration, ) -> Result { if !params.is_object() { return Err(LocalRunnerError::invalid( @@ -311,7 +341,7 @@ impl AcpxSidecarTransport { })?; self.next_request_id = request_id + 1; - let deadline = Instant::now() + self.request_timeout; + let deadline = Instant::now() + timeout; loop { let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index e16b748eaf0..51c24577e1b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -55,6 +55,9 @@ fn run() -> Result<(), Box> { .get("command") .and_then(Value::as_str) .ok_or("request command is missing")?; + if mode == "slow-commands" || (mode == "bootstrap-slow-open" && command == "session.open") { + std::thread::sleep(Duration::from_millis(600)); + } if let Some(journal) = admission_journal.as_mut() { write_json(journal, &json!({"request":request}))?; let response = if command == "tool.resolve" { @@ -261,6 +264,7 @@ fn run() -> Result<(), Box> { std::process::exit(9); } "bootstrap" + | "bootstrap-slow-open" | "goals" | "bootstrap-wrong-model" | "bootstrap-wrong-run" @@ -760,7 +764,7 @@ fn run() -> Result<(), Box> { next_sequence += 1; } } - "happy" => { + "happy" | "slow-commands" => { write_event(&mut stdout, next_sequence)?; next_sequence += 1; write_json(&mut stdout, &success(id, command, &request))?; diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs index 911b765b749..61f48e52af9 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_provider_session.rs @@ -82,6 +82,19 @@ fn start_error(config: &AcpxProviderSessionConfig) -> String { } } +#[test] +fn hermes_cold_bootstrap_has_a_separate_startup_deadline() { + let mut slow = config("bootstrap-slow-open"); + slow.transport.request_timeout = Duration::from_millis(250); + assert!(start_error(&slow).contains("timed out at session.open")); + slow.agent = "hermes".to_owned(); + slow.provider_policy = Some(AcpxProviderRuntimePolicy { read_only: false }); + let mut session = AcpxProviderSession::start(&slow) + .expect("Hermes startup admits the same delayed session response"); + assert_eq!(session.identity().agent_session_id, "agent-1"); + session.shutdown("startup deadline test").unwrap(); +} + #[test] fn controls_goals_and_observes_updates_without_an_active_prompt() { let mut session = AcpxProviderSession::start(&config("goals")).unwrap(); diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs index 48ddd249748..d591b5d364d 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/acpx_sidecar_transport.rs @@ -116,6 +116,38 @@ fn times_out_and_terminates_a_silent_sidecar() { assert!(started.elapsed() < Duration::from_secs(1)); } +#[test] +fn cold_session_admission_does_not_extend_later_command_deadlines() { + let mut transport = transport("slow-commands", Duration::from_millis(200)); + let result = transport + .open_session(json!({}), Duration::from_secs(2)) + .expect("cold admission should accept its delayed response"); + assert_eq!(result["command"], "session.open"); + let started = Instant::now(); + let error = transport + .request(GeneratedAcpxSidecarCommand::SessionRead, json!({})) + .expect_err("ordinary commands retain the short configured deadline"); + assert!(error.to_string().contains("timed out at session.read")); + assert!(started.elapsed() < Duration::from_secs(1)); + assert!(transport + .request(GeneratedAcpxSidecarCommand::SessionRead, json!({})) + .unwrap_err() + .to_string() + .contains("unavailable")); +} + +#[test] +fn an_invalid_startup_bound_does_not_send_or_poison_a_command() { + let mut transport = transport("happy", Duration::from_secs(1)); + for timeout in [Duration::ZERO, Duration::from_secs(121)] { + assert!(transport.open_session(json!({}), timeout).is_err()); + } + transport + .request(GeneratedAcpxSidecarCommand::Initialize, json!({})) + .expect("invalid local bounds do not consume or poison the wire"); + transport.shutdown().unwrap(); +} + #[test] fn an_empty_event_poll_does_not_poison_the_transport() { let mut transport = transport("silent", Duration::from_secs(1)); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 7fa34b74937..e3fcd378514 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -94,6 +94,45 @@ const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000; const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024; const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024; +function runnerdCommandTimeoutMs( + type: string, + provider?: string, + acpxAgent?: string, +): number { + // Session admission can copy the pinned Python tree and restore its native + // history. Leave room around the sidecar's 60 s open bound for PRP bootstrap. + // Subsequent turn execution, tool/control commands and stop retain their bounds. + return provider === "acpx" + && acpxAgent === "hermes" + && ["session.open", "runner.drain", "turn.start"].includes(type) + ? 75_000 + : 30_000; +} + +async function awaitRunnerdCommand(input: { + type: string; + command: () => { status: string; result?: unknown } | undefined; + throwIfFailed: () => void; + runnerHasExited: () => Promise; + deadline: number; + timeoutPrefix: () => string; +}): Promise { + while (Date.now() < input.deadline) { + input.throwIfFailed(); + const command = input.command(); + if (command?.status === "completed") return; + if (command !== undefined && command.status !== "pending") { + throw new Error( + `PRP command ${input.type} ${command.status}: ${JSON.stringify(command.result)}`, + ); + } + if (await input.runnerHasExited()) + throw new Error(`runnerd exited while waiting for ${input.type}`); + await new Promise(resolveWait => setTimeout(resolveWait, 10)); + } + throw new Error(`${input.timeoutPrefix()}: PRP command ${input.type} timed out`); +} + function readLocalProcessStartedAt(pid: number): string | null { if (!Number.isInteger(pid) || pid <= 0) return null; try { @@ -5739,7 +5778,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #startTurn( params: Record, ): Promise> { - const turnStartTimeoutMs = this.options.turnStartTimeoutMs ?? 30_000; + const turnStartTimeoutMs = this.options.turnStartTimeoutMs + ?? runnerdCommandTimeoutMs( + "turn.start", this.options.provider, this.options.acpxAgent, + ); if (!Number.isSafeInteger(turnStartTimeoutMs) || turnStartTimeoutMs <= 0) { throw new Error("turnStartTimeoutMs must be a positive safe integer"); } @@ -5994,29 +6036,25 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { async #waitCommand( type: string, commandId?: string, - deadline = Date.now() + 30_000, + deadline = Date.now() + runnerdCommandTimeoutMs( + type, this.options.provider, this.options.acpxAgent, + ), ): Promise { - while (Date.now() < deadline) { - this.#throwIfFailed(); - const command = + return awaitRunnerdCommand({ + type, + deadline, + throwIfFailed: () => this.#throwIfFailed(), + runnerHasExited: () => this.#runnerHasExited(), + command: () => commandId === undefined ? this.#core?.store.state.commands.find( (candidate) => candidate.type === type, ) - : this.#core?.getCommand(commandId); - if (command?.status === "completed") return; - if (command !== undefined && command.status !== "pending") { - throw new Error( - `PRP command ${type} ${command.status}: ${JSON.stringify(command.result)}`, - ); - } - if (await this.#runnerHasExited()) - throw new Error(`runnerd exited while waiting for ${type}`); - await new Promise((resolveWait) => setTimeout(resolveWait, 10)); - } - throw new Error( - `${this.#startupComplete ? "provider_transport_failed" : this.#startupFailureCode}: PRP command ${type} timed out`, - ); + : this.#core?.getCommand(commandId), + timeoutPrefix: () => this.#startupComplete + ? "provider_transport_failed" + : this.#startupFailureCode, + }); } @@ -6924,6 +6962,8 @@ export const runnerdLaunchProfileInternals = Object.freeze({ }); export const runnerdRecoveryInternals = Object.freeze({ + runnerdCommandTimeoutMs, + awaitRunnerdCommand, completedMaintenanceTerminalReceipt, completedMaintenanceTerminalReplayMatches, readControlPlaneState, diff --git a/packages/paperclip-runner/src/live/runnerd-command-admission.test.ts b/packages/paperclip-runner/src/live/runnerd-command-admission.test.ts new file mode 100644 index 00000000000..bff2d007d3f --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-command-admission.test.ts @@ -0,0 +1,48 @@ +import { expect, it, vi } from "vitest"; +import { runnerdRecoveryInternals } from "./runnerd-codex-transport.js"; + +it.each([ + ["hermes", "session.open", true], + ["hermes", "runner.drain", true], + ["hermes", "turn.start", true], + ["codex", "session.open", false], + ["hermes", "session.snapshot", false], + ["hermes", "turn.stop", false], +] as const)("bounds %s %s admission around a delayed durable receipt", async (agent, type, coldStartup) => { + vi.useFakeTimers(); + vi.setSystemTime(0); + try { + let status = "pending"; + const completed = setTimeout(() => { status = "completed"; }, 35_000); + const result = runnerdRecoveryInternals.awaitRunnerdCommand({ + type, command: () => ({ status }), throwIfFailed: () => undefined, + runnerHasExited: async () => false, + deadline: Date.now() + runnerdRecoveryInternals.runnerdCommandTimeoutMs(type, "acpx", agent), + timeoutPrefix: () => "provider_transport_failed", + }).then(() => "accepted", (error: Error) => error.message); + await vi.advanceTimersByTimeAsync(35_000); + if (coldStartup) expect(await result).toBe("accepted"); + else expect(await result).toBe(`provider_transport_failed: PRP command ${type} timed out`); + clearTimeout(completed); + expect(vi.getTimerCount()).toBe(0); + } finally { vi.useRealTimers(); } +}); + +it("still rejects a Hermes startup whose receipt never arrives", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + try { + let settled = false; + const result = runnerdRecoveryInternals.awaitRunnerdCommand({ + type: "session.open", command: () => ({ status: "pending" }), + throwIfFailed: () => undefined, runnerHasExited: async () => false, + deadline: Date.now() + runnerdRecoveryInternals.runnerdCommandTimeoutMs("session.open", "acpx", "hermes"), + timeoutPrefix: () => "runner_local_connect_failed", + }).then(() => "accepted", (error: Error) => { settled = true; return error.message; }); + await vi.advanceTimersByTimeAsync(74_999); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(await result).toBe("runner_local_connect_failed: PRP command session.open timed out"); + expect(vi.getTimerCount()).toBe(0); + } finally { vi.useRealTimers(); } +}); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 9d714488fa6..9dd30ccbc90 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -15,6 +15,23 @@ const everydayOracleImage = "python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285"; describe("public repository paid workflow security", () => { + it("keeps native Hermes PR fixtures outside paid authority and strips their child environment", async () => { + const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-hermes-native.yml"), "utf8"); + expect(workflow).toContain("pull_request:"); + expect(workflow).not.toContain("pull_request_target:"); + expect(workflow).not.toMatch(/\bsecrets\.|\benvironment:|\bwrite\b/); + expect(workflow).toContain("contents: read"); + expect(workflow).toContain("persist-credentials: false"); + const fixtures = workflow.slice(workflow.indexOf(" - name: Run native fixtures"), workflow.indexOf(" - name: Record source")); + expect(fixtures).toContain('env -i PATH="$PATH" HOME="$RUNNER_TEMP/hermes-fixture-home"'); + expect(fixtures).toContain("PAPERCLIP_HERMES_QUALIFY=1"); + expect(fixtures).not.toMatch(/(?:API_KEY|TOKEN|AUTH_JSON|KEEP)/); + expect(workflow.indexOf("Provision the pinned Python closure")).toBeLessThan(workflow.indexOf("Run native fixtures")); + expect(workflow).toContain("no paid model, browser or Daytona proof"); + const upload = workflow.slice(workflow.indexOf(" - name: Upload fixture evidence")); + expect(upload).toContain("hermes-native-evidence/"); + expect(upload).not.toMatch(/hermes-fixture-home|provider-assets|runtimeDirectory/); + }); it("provisions selected Hermes assets before credentials and uses the same selection for image identity and packs", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:"));