mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 00:05:49 +02:00
fix(hermes): bound cold native admission separately
Give Hermes native session opening a 60-second bound for verified private runtime copying and ACP initialization. Allow controller startup and per-turn restoration overhead while preserving ordinary command and stop deadlines. Add delayed-admission regressions and credential-free Linux native fixture CI. Record the failed final-head paid campaign without promoting qualification. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
02a7964775
commit
a95fe6cabb
11 files changed
+430
-29
No files matched your search
@@ -0,0 +1,132 @@
|
||||
name: Hermes Native Transport
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/runner-hermes-native.yml
|
||||
- packages/paperclip-runner/src/providers/hermes/**
|
||||
- packages/paperclip-runner/src/drivers/acpx/**
|
||||
- packages/paperclip-runner/src/live/runnerd-*.ts
|
||||
- packages/paperclip-runner/runner/crates/runner-core/src/acpx_*.rs
|
||||
- packages/paperclip-runner/scripts/*hermes*
|
||||
- packages/paperclip-runner/acpx-profiles.json
|
||||
- tests/runner-e2e/provision-hermes-linux.sh
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: hermes-native-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
native_transport:
|
||||
name: Hermes native fixture (Linux amd64, no credentials)
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout source
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
|
||||
env:
|
||||
NPM_CONFIG_AUDIT: "false"
|
||||
NPM_CONFIG_FUND: "false"
|
||||
NPM_CONFIG_UPDATE_NOTIFIER: "false"
|
||||
with:
|
||||
version: 9.15.4
|
||||
|
||||
- name: Install workspace dependencies
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! pnpm install --frozen-lockfile; then
|
||||
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
|
||||
pnpm install --frozen-lockfile
|
||||
fi
|
||||
|
||||
- name: Restore Rust dependencies (read only)
|
||||
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: packages/paperclip-runner/runner -> target
|
||||
cache-workspace-crates: false
|
||||
cache-bin: false
|
||||
save-if: false
|
||||
|
||||
- name: Build verified runner and provider entrypoints
|
||||
run: |
|
||||
set -euo pipefail
|
||||
pnpm --filter @paperclipai/paperclip-runner build:typescript
|
||||
pnpm --filter @paperclipai/paperclip-runner build:binary
|
||||
|
||||
- name: Provision the pinned Python closure without credentials
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends python3-venv bubblewrap
|
||||
bash tests/runner-e2e/provision-hermes-linux.sh
|
||||
|
||||
- name: Run native fixtures against a deterministic loopback model
|
||||
id: fixtures
|
||||
working-directory: packages/paperclip-runner
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -m 700 "$RUNNER_TEMP/hermes-fixture-home"
|
||||
# No paid environment, secrets, or credentials reach either fixture.
|
||||
# The execution host must pass the real bubblewrap namespace probe.
|
||||
env -i PATH="$PATH" HOME="$RUNNER_TEMP/hermes-fixture-home" \
|
||||
PAPERCLIP_HERMES_QUALIFY=1 \
|
||||
node --test --test-concurrency=1 scripts/qualify-hermes-runtime.test.mjs \
|
||||
scripts/qualify-hermes-runnerd.test.mjs \
|
||||
2>&1 | tee "$RUNNER_TEMP/hermes-native-fixture.log"
|
||||
|
||||
- name: Record source and runtime provenance
|
||||
if: always()
|
||||
env:
|
||||
HERMES_FIXTURE_OUTCOME: ${{ steps.fixtures.outcome }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$RUNNER_TEMP/hermes-native-evidence"
|
||||
if [ -f "$RUNNER_TEMP/hermes-native-fixture.log" ]; then
|
||||
cp "$RUNNER_TEMP/hermes-native-fixture.log" "$RUNNER_TEMP/hermes-native-evidence/fixture.log"
|
||||
fi
|
||||
node --input-type=module <<'NODE'
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
const root = 'packages/paperclip-runner';
|
||||
const version = JSON.parse(await readFile(join(root, 'src/providers/hermes/version.json'), 'utf8'));
|
||||
const manifest = JSON.parse(await readFile(join(root, 'provider-assets/hermes/linux-x64/manifest.json'), 'utf8').catch(() => 'null'));
|
||||
const evidence = {
|
||||
qualification: 'native transport fixture only; no paid model, browser or Daytona proof',
|
||||
sourceSha: execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(),
|
||||
platform: `${process.platform}-${process.arch}`, node: process.version,
|
||||
fixtureOutcome: process.env.HERMES_FIXTURE_OUTCOME,
|
||||
kernel: execFileSync('uname', ['-r'], { encoding: 'utf8' }).trim(),
|
||||
osRelease: await readFile('/etc/os-release', 'utf8'),
|
||||
runtime: version,
|
||||
closureSha256: manifest ? createHash('sha256').update(JSON.stringify(manifest.entries)).digest('hex') : null,
|
||||
files: manifest?.entries?.length ?? null,
|
||||
model: 'deterministic loopback fixture', credentials: 'none',
|
||||
};
|
||||
await writeFile(join(process.env.RUNNER_TEMP, 'hermes-native-evidence/provenance.json'), JSON.stringify(evidence, null, 2) + '\n');
|
||||
NODE
|
||||
|
||||
- name: Upload fixture evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: hermes-native-linux-amd64-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/hermes-native-evidence/
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
@@ -23,7 +23,8 @@ Keep the existing Hermes local/gateway adapters compatible.
|
||||
- [x] Complete final Rust/regression checks and resolve or classify failures.
|
||||
- [ ] Browser acceptance, Linux/Daytona execution and connection-method qualification.
|
||||
- [x] Reviewable draft PR stack, green CI and fresh Greptile 5/5 on the implementation heads.
|
||||
- [ ] Complete the final local aggregate test invocation and release qualification.
|
||||
- [x] Complete the final local aggregate test invocation and classify its failures.
|
||||
- [ ] Complete live release qualification.
|
||||
|
||||
## Qualification evidence
|
||||
|
||||
@@ -60,8 +61,8 @@ because fixtures pass.
|
||||
Native cron and gateway messaging are disabled.
|
||||
- Pending Hermes choice in existing runner configuration; existing connection,
|
||||
model, permission and transcript components. Ten Product E2E candidate cells
|
||||
(five local, five Daytona) are registered. Four local cases have passing paid
|
||||
attempts; the complete release matrix has not passed.
|
||||
(five local, five Daytona) are registered. All five local cases have passing
|
||||
paid attempts across different heads; the complete release matrix has not passed.
|
||||
|
||||
## Evidence and outstanding release gates
|
||||
|
||||
@@ -342,3 +343,70 @@ cross-task memory/learned skills, routine firing and deduplication, and actual
|
||||
Linux amd64/Daytona execution and restoration. Exact approved Connection names
|
||||
and a compatible remote execution environment are still needed. The existing
|
||||
standalone Hermes local/gateway adapters retain their contracts.
|
||||
|
||||
### Final-head campaign and startup follow-up, 2026-10-07
|
||||
|
||||
The campaign `hermes-local-paid-20261007-final-head` correctly recorded source
|
||||
`a54d04785b62e2190f4e13647bf75ae510449d98` and passed **1/5** cases. It used
|
||||
the same managed OpenRouter account, exact model and macOS browser/Runner path:
|
||||
|
||||
| Case | Result | Failure or limitation |
|
||||
| --- | --- | --- |
|
||||
| Hello/completion | Pass | Done, exact final answer and cleanup passed |
|
||||
| Question/resume | Fail | `session.open` exceeded its 30-second command deadline; cleanup passed |
|
||||
| Plan/approve/complete | Fail | Reasoning streamed but the 120-second turn deadline expired; cleanup passed |
|
||||
| Structured question/restart/resume | Fail | The isolated server hit system `ENFILE`; cleanup failed |
|
||||
| File edit/validation | Fail | Dispatch was interrupted after the resource failure; process-group cleanup identity was uncertain |
|
||||
|
||||
The file case's underlying task later reached success, but its independent
|
||||
fixture oracle and cleanup did not pass. Its recorded failure is unchanged.
|
||||
The owned launcher and remaining isolated server/database were retired with
|
||||
verified exit. Failed private recovery roots remain preserved because their
|
||||
original cleanup receipts failed. No additional paid campaign was launched.
|
||||
The numeric budget receipt reports $4.539181805 remaining under the key's $5
|
||||
limit; this is shared-key accounting, not exact per-run billed cost.
|
||||
|
||||
Credential-free production-host probes reproduced variable cold-start latency.
|
||||
The first verified 445 MB runtime copy took 3.08 seconds. A complete admission
|
||||
later took 33.69 seconds: 16.54 seconds for the verified private copy and 16.61
|
||||
seconds for native initialization/ACP handshake. Its no-auth loopback endpoint
|
||||
received only model/backend metadata probes, with no inference requests.
|
||||
These measurements reproduce an admission deadline problem; they do not prove
|
||||
the cause of the host's file-table exhaustion. The read-only host counter still
|
||||
reported 461,999 open files against a 491,520 limit. No system limit was changed
|
||||
and no unrelated process was stopped.
|
||||
|
||||
Hermes now has a separate 60-second native session-open deadline. The PRP
|
||||
controller allows 75 seconds around cold admission, recovery, and later-turn
|
||||
restoration. Ordinary commands and stop retain their prior deadlines, and the
|
||||
post-acceptance turn-start event deadline remains 30 seconds. The 120-second
|
||||
Product E2E turn oracle is unchanged. Focused regressions verify delayed startup,
|
||||
ordinary-command timeout, fail-closed transport reuse and the finite outer
|
||||
deadline. This addresses admission timing only; the paid planning timeout and
|
||||
host resource failures still require a new passing campaign on a reliable host.
|
||||
|
||||
Startup verification passed seven TypeScript deadline regressions, the complete
|
||||
193-test controller transport selection, 27 focused Rust session/transport test
|
||||
executions, and the complete 655-execution Rust workspace suite (two ignored).
|
||||
Runner TypeScript/Rust typechecks, verified entrypoint builds, release binary
|
||||
build, workflow authority tests (14), and actionlint pass.
|
||||
Both rebuilt native production fixtures also pass: 35.98 seconds through Rust
|
||||
PRP/sidecar and 144.68 seconds through the ACPX host. They use the deterministic
|
||||
no-auth model fixture and retain their original assertions and deadlines.
|
||||
|
||||
The new credential-free `Hermes Native Transport` PR workflow provisions the
|
||||
pinned Linux amd64 closure and runs both production native fixtures on an
|
||||
ephemeral Ubuntu host. Provider children receive an empty environment plus the
|
||||
fixture PATH/home/opt-in flag. No paid environment or credentials are available.
|
||||
It retains source/runtime provenance and fixture logs as CI artifacts while
|
||||
excluding private native homes. Its actual execution result must be checked;
|
||||
the workflow declaration alone is not Linux proof. The protected paid workflow
|
||||
and its default-branch authorization remain unchanged by this addition.
|
||||
|
||||
The three draft PRs had green CI and fresh Greptile 5/5 at routines
|
||||
`3745f3c5a46bda7778ee132682d1b7ae23b088c1`, native
|
||||
`0dea682031f8e35631faee7a05519ed4dce80d07`, and qualification
|
||||
`a54d04785b62e2190f4e13647bf75ae510449d98`. That qualification CI initially
|
||||
failed an unchanged signoff mock-heartbeat browser case; inspection and the
|
||||
failed-job-only rerun passed. Startup changes require new checks and review.
|
||||
Hermes remains pending qualification throughout.
|
||||
@@ -65,6 +65,13 @@ working copy and assigned tool bindings. The session still pins prompt, bundle,
|
||||
skill, connection, model, and permission identities. Unknown policy changes and
|
||||
changes within the same run are rejected.
|
||||
|
||||
Native session admission has a 60-second bound for the verified private Python
|
||||
copy and ACP initialization. The controller allows 75 seconds around cold
|
||||
startup, recovery and later-turn restoration. Once a turn is accepted, its
|
||||
start-event deadline remains 30 seconds. Ordinary commands, cancellation and
|
||||
process cleanup retain their existing bounds; this startup allowance does not
|
||||
extend the task's execution deadline.
|
||||
|
||||
## Interaction contract
|
||||
|
||||
- Reasoning and assistant text use distinct message identities. Native tool
|
||||
@@ -141,3 +148,9 @@ Hermes cells using a managed OpenRouter connection. Each connection method,
|
||||
native control, attachment, persistence, remote restoration and permission mode
|
||||
must pass its release criterion with inspectable live evidence before promotion.
|
||||
The implementation record is included in that qualification PR.
|
||||
|
||||
The `Hermes Native Transport` PR workflow runs the same native fixtures on Linux
|
||||
amd64 with a stripped provider environment and deterministic loopback endpoints.
|
||||
Its CI artifacts include exact source/runtime provenance and fixture logs. This
|
||||
is credential-free transport and host-sandbox evidence; it does not replace
|
||||
paid-provider, browser, subscription or Daytona qualification.
|
||||
@@ -1287,10 +1287,14 @@ fn bootstrap(
|
||||
)?;
|
||||
verify_initialize_response(&initialized, transport.process_id())?;
|
||||
|
||||
let opened = transport.request(
|
||||
GeneratedAcpxSidecarCommand::SessionOpen,
|
||||
session_open_params(config, &sidecar_tools),
|
||||
)?;
|
||||
let open_params = session_open_params(config, &sidecar_tools);
|
||||
let opened = if config.agent == "hermes" {
|
||||
// The pinned Python closure is verified and privately copied before
|
||||
// native initialization. This also applies to per-turn restoration.
|
||||
transport.open_session(open_params, Duration::from_secs(60))?
|
||||
} else {
|
||||
transport.request(GeneratedAcpxSidecarCommand::SessionOpen, open_params)?
|
||||
};
|
||||
let identity = verify_open_response(&opened, transport.process_id(), config)?;
|
||||
let turn_controls = verified_turn_controls(opened.get("turnControls"), &config.agent)?;
|
||||
|
||||
|
||||
@@ -224,13 +224,42 @@ impl AcpxSidecarTransport {
|
||||
&mut self,
|
||||
command: GeneratedAcpxSidecarCommand,
|
||||
params: Value,
|
||||
) -> Result<Value, LocalRunnerError> {
|
||||
self.request_with_timeout(command, params, self.request_timeout)
|
||||
}
|
||||
|
||||
/// Cold native admission has a separate bound; later commands retain the
|
||||
/// configured transport deadline, including cancellation and shutdown.
|
||||
pub fn open_session(
|
||||
&mut self,
|
||||
params: Value,
|
||||
startup_timeout: Duration,
|
||||
) -> Result<Value, LocalRunnerError> {
|
||||
if startup_timeout < Duration::from_millis(1) || startup_timeout > Duration::from_secs(120)
|
||||
{
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX session startup timeout must be in the range 1 ms through 120 s",
|
||||
));
|
||||
}
|
||||
self.request_with_timeout(
|
||||
GeneratedAcpxSidecarCommand::SessionOpen,
|
||||
params,
|
||||
startup_timeout,
|
||||
)
|
||||
}
|
||||
|
||||
fn request_with_timeout(
|
||||
&mut self,
|
||||
command: GeneratedAcpxSidecarCommand,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
) -> Result<Value, LocalRunnerError> {
|
||||
if self.poisoned {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
"ACPX sidecar transport is unavailable after a protocol failure",
|
||||
));
|
||||
}
|
||||
let result = self.request_inner(command, params);
|
||||
let result = self.request_inner(command, params, timeout);
|
||||
match result {
|
||||
Ok(CommandOutcome::Success(value)) => Ok(value),
|
||||
Ok(CommandOutcome::Rejected(error)) => Err(error),
|
||||
@@ -277,6 +306,7 @@ impl AcpxSidecarTransport {
|
||||
&mut self,
|
||||
command: GeneratedAcpxSidecarCommand,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
) -> Result<CommandOutcome, LocalRunnerError> {
|
||||
if !params.is_object() {
|
||||
return Err(LocalRunnerError::invalid(
|
||||
@@ -311,7 +341,7 @@ impl AcpxSidecarTransport {
|
||||
})?;
|
||||
self.next_request_id = request_id + 1;
|
||||
|
||||
let deadline = Instant::now() + self.request_timeout;
|
||||
let deadline = Instant::now() + timeout;
|
||||
loop {
|
||||
let remaining = deadline.saturating_duration_since(Instant::now());
|
||||
if remaining.is_zero() {
|
||||
|
||||
@@ -55,6 +55,9 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.get("command")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or("request command is missing")?;
|
||||
if mode == "slow-commands" || (mode == "bootstrap-slow-open" && command == "session.open") {
|
||||
std::thread::sleep(Duration::from_millis(600));
|
||||
}
|
||||
if let Some(journal) = admission_journal.as_mut() {
|
||||
write_json(journal, &json!({"request":request}))?;
|
||||
let response = if command == "tool.resolve" {
|
||||
@@ -261,6 +264,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
std::process::exit(9);
|
||||
}
|
||||
"bootstrap"
|
||||
| "bootstrap-slow-open"
|
||||
| "goals"
|
||||
| "bootstrap-wrong-model"
|
||||
| "bootstrap-wrong-run"
|
||||
@@ -760,7 +764,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
|
||||
next_sequence += 1;
|
||||
}
|
||||
}
|
||||
"happy" => {
|
||||
"happy" | "slow-commands" => {
|
||||
write_event(&mut stdout, next_sequence)?;
|
||||
next_sequence += 1;
|
||||
write_json(&mut stdout, &success(id, command, &request))?;
|
||||
|
||||
@@ -82,6 +82,19 @@ fn start_error(config: &AcpxProviderSessionConfig) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hermes_cold_bootstrap_has_a_separate_startup_deadline() {
|
||||
let mut slow = config("bootstrap-slow-open");
|
||||
slow.transport.request_timeout = Duration::from_millis(250);
|
||||
assert!(start_error(&slow).contains("timed out at session.open"));
|
||||
slow.agent = "hermes".to_owned();
|
||||
slow.provider_policy = Some(AcpxProviderRuntimePolicy { read_only: false });
|
||||
let mut session = AcpxProviderSession::start(&slow)
|
||||
.expect("Hermes startup admits the same delayed session response");
|
||||
assert_eq!(session.identity().agent_session_id, "agent-1");
|
||||
session.shutdown("startup deadline test").unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn controls_goals_and_observes_updates_without_an_active_prompt() {
|
||||
let mut session = AcpxProviderSession::start(&config("goals")).unwrap();
|
||||
|
||||
@@ -116,6 +116,38 @@ fn times_out_and_terminates_a_silent_sidecar() {
|
||||
assert!(started.elapsed() < Duration::from_secs(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cold_session_admission_does_not_extend_later_command_deadlines() {
|
||||
let mut transport = transport("slow-commands", Duration::from_millis(200));
|
||||
let result = transport
|
||||
.open_session(json!({}), Duration::from_secs(2))
|
||||
.expect("cold admission should accept its delayed response");
|
||||
assert_eq!(result["command"], "session.open");
|
||||
let started = Instant::now();
|
||||
let error = transport
|
||||
.request(GeneratedAcpxSidecarCommand::SessionRead, json!({}))
|
||||
.expect_err("ordinary commands retain the short configured deadline");
|
||||
assert!(error.to_string().contains("timed out at session.read"));
|
||||
assert!(started.elapsed() < Duration::from_secs(1));
|
||||
assert!(transport
|
||||
.request(GeneratedAcpxSidecarCommand::SessionRead, json!({}))
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
.contains("unavailable"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_invalid_startup_bound_does_not_send_or_poison_a_command() {
|
||||
let mut transport = transport("happy", Duration::from_secs(1));
|
||||
for timeout in [Duration::ZERO, Duration::from_secs(121)] {
|
||||
assert!(transport.open_session(json!({}), timeout).is_err());
|
||||
}
|
||||
transport
|
||||
.request(GeneratedAcpxSidecarCommand::Initialize, json!({}))
|
||||
.expect("invalid local bounds do not consume or poison the wire");
|
||||
transport.shutdown().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_event_poll_does_not_poison_the_transport() {
|
||||
let mut transport = transport("silent", Duration::from_secs(1));
|
||||
|
||||
@@ -94,6 +94,45 @@ const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000;
|
||||
const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024;
|
||||
const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024;
|
||||
|
||||
function runnerdCommandTimeoutMs(
|
||||
type: string,
|
||||
provider?: string,
|
||||
acpxAgent?: string,
|
||||
): number {
|
||||
// Session admission can copy the pinned Python tree and restore its native
|
||||
// history. Leave room around the sidecar's 60 s open bound for PRP bootstrap.
|
||||
// Subsequent turn execution, tool/control commands and stop retain their bounds.
|
||||
return provider === "acpx"
|
||||
&& acpxAgent === "hermes"
|
||||
&& ["session.open", "runner.drain", "turn.start"].includes(type)
|
||||
? 75_000
|
||||
: 30_000;
|
||||
}
|
||||
|
||||
async function awaitRunnerdCommand(input: {
|
||||
type: string;
|
||||
command: () => { status: string; result?: unknown } | undefined;
|
||||
throwIfFailed: () => void;
|
||||
runnerHasExited: () => Promise<boolean>;
|
||||
deadline: number;
|
||||
timeoutPrefix: () => string;
|
||||
}): Promise<void> {
|
||||
while (Date.now() < input.deadline) {
|
||||
input.throwIfFailed();
|
||||
const command = input.command();
|
||||
if (command?.status === "completed") return;
|
||||
if (command !== undefined && command.status !== "pending") {
|
||||
throw new Error(
|
||||
`PRP command ${input.type} ${command.status}: ${JSON.stringify(command.result)}`,
|
||||
);
|
||||
}
|
||||
if (await input.runnerHasExited())
|
||||
throw new Error(`runnerd exited while waiting for ${input.type}`);
|
||||
await new Promise(resolveWait => setTimeout(resolveWait, 10));
|
||||
}
|
||||
throw new Error(`${input.timeoutPrefix()}: PRP command ${input.type} timed out`);
|
||||
}
|
||||
|
||||
function readLocalProcessStartedAt(pid: number): string | null {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return null;
|
||||
try {
|
||||
@@ -5739,7 +5778,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
async #startTurn(
|
||||
params: Record<string, unknown>,
|
||||
): Promise<Record<string, unknown>> {
|
||||
const turnStartTimeoutMs = this.options.turnStartTimeoutMs ?? 30_000;
|
||||
const turnStartTimeoutMs = this.options.turnStartTimeoutMs
|
||||
?? runnerdCommandTimeoutMs(
|
||||
"turn.start", this.options.provider, this.options.acpxAgent,
|
||||
);
|
||||
if (!Number.isSafeInteger(turnStartTimeoutMs) || turnStartTimeoutMs <= 0) {
|
||||
throw new Error("turnStartTimeoutMs must be a positive safe integer");
|
||||
}
|
||||
@@ -5994,29 +6036,25 @@ class DurablePrpCodexTransport implements CodexAppServerTransport {
|
||||
async #waitCommand(
|
||||
type: string,
|
||||
commandId?: string,
|
||||
deadline = Date.now() + 30_000,
|
||||
deadline = Date.now() + runnerdCommandTimeoutMs(
|
||||
type, this.options.provider, this.options.acpxAgent,
|
||||
),
|
||||
): Promise<void> {
|
||||
while (Date.now() < deadline) {
|
||||
this.#throwIfFailed();
|
||||
const command =
|
||||
return awaitRunnerdCommand({
|
||||
type,
|
||||
deadline,
|
||||
throwIfFailed: () => this.#throwIfFailed(),
|
||||
runnerHasExited: () => this.#runnerHasExited(),
|
||||
command: () =>
|
||||
commandId === undefined
|
||||
? this.#core?.store.state.commands.find(
|
||||
(candidate) => candidate.type === type,
|
||||
)
|
||||
: this.#core?.getCommand(commandId);
|
||||
if (command?.status === "completed") return;
|
||||
if (command !== undefined && command.status !== "pending") {
|
||||
throw new Error(
|
||||
`PRP command ${type} ${command.status}: ${JSON.stringify(command.result)}`,
|
||||
);
|
||||
}
|
||||
if (await this.#runnerHasExited())
|
||||
throw new Error(`runnerd exited while waiting for ${type}`);
|
||||
await new Promise((resolveWait) => setTimeout(resolveWait, 10));
|
||||
}
|
||||
throw new Error(
|
||||
`${this.#startupComplete ? "provider_transport_failed" : this.#startupFailureCode}: PRP command ${type} timed out`,
|
||||
);
|
||||
: this.#core?.getCommand(commandId),
|
||||
timeoutPrefix: () => this.#startupComplete
|
||||
? "provider_transport_failed"
|
||||
: this.#startupFailureCode,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -6924,6 +6962,8 @@ export const runnerdLaunchProfileInternals = Object.freeze({
|
||||
});
|
||||
|
||||
export const runnerdRecoveryInternals = Object.freeze({
|
||||
runnerdCommandTimeoutMs,
|
||||
awaitRunnerdCommand,
|
||||
completedMaintenanceTerminalReceipt,
|
||||
completedMaintenanceTerminalReplayMatches,
|
||||
readControlPlaneState,
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { runnerdRecoveryInternals } from "./runnerd-codex-transport.js";
|
||||
|
||||
it.each([
|
||||
["hermes", "session.open", true],
|
||||
["hermes", "runner.drain", true],
|
||||
["hermes", "turn.start", true],
|
||||
["codex", "session.open", false],
|
||||
["hermes", "session.snapshot", false],
|
||||
["hermes", "turn.stop", false],
|
||||
] as const)("bounds %s %s admission around a delayed durable receipt", async (agent, type, coldStartup) => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
try {
|
||||
let status = "pending";
|
||||
const completed = setTimeout(() => { status = "completed"; }, 35_000);
|
||||
const result = runnerdRecoveryInternals.awaitRunnerdCommand({
|
||||
type, command: () => ({ status }), throwIfFailed: () => undefined,
|
||||
runnerHasExited: async () => false,
|
||||
deadline: Date.now() + runnerdRecoveryInternals.runnerdCommandTimeoutMs(type, "acpx", agent),
|
||||
timeoutPrefix: () => "provider_transport_failed",
|
||||
}).then(() => "accepted", (error: Error) => error.message);
|
||||
await vi.advanceTimersByTimeAsync(35_000);
|
||||
if (coldStartup) expect(await result).toBe("accepted");
|
||||
else expect(await result).toBe(`provider_transport_failed: PRP command ${type} timed out`);
|
||||
clearTimeout(completed);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
|
||||
it("still rejects a Hermes startup whose receipt never arrives", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
try {
|
||||
let settled = false;
|
||||
const result = runnerdRecoveryInternals.awaitRunnerdCommand({
|
||||
type: "session.open", command: () => ({ status: "pending" }),
|
||||
throwIfFailed: () => undefined, runnerHasExited: async () => false,
|
||||
deadline: Date.now() + runnerdRecoveryInternals.runnerdCommandTimeoutMs("session.open", "acpx", "hermes"),
|
||||
timeoutPrefix: () => "runner_local_connect_failed",
|
||||
}).then(() => "accepted", (error: Error) => { settled = true; return error.message; });
|
||||
await vi.advanceTimersByTimeAsync(74_999);
|
||||
expect(settled).toBe(false);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(await result).toBe("runner_local_connect_failed: PRP command session.open timed out");
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally { vi.useRealTimers(); }
|
||||
});
|
||||
@@ -15,6 +15,23 @@ const everydayOracleImage =
|
||||
"python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285";
|
||||
|
||||
describe("public repository paid workflow security", () => {
|
||||
it("keeps native Hermes PR fixtures outside paid authority and strips their child environment", async () => {
|
||||
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-hermes-native.yml"), "utf8");
|
||||
expect(workflow).toContain("pull_request:");
|
||||
expect(workflow).not.toContain("pull_request_target:");
|
||||
expect(workflow).not.toMatch(/\bsecrets\.|\benvironment:|\bwrite\b/);
|
||||
expect(workflow).toContain("contents: read");
|
||||
expect(workflow).toContain("persist-credentials: false");
|
||||
const fixtures = workflow.slice(workflow.indexOf(" - name: Run native fixtures"), workflow.indexOf(" - name: Record source"));
|
||||
expect(fixtures).toContain('env -i PATH="$PATH" HOME="$RUNNER_TEMP/hermes-fixture-home"');
|
||||
expect(fixtures).toContain("PAPERCLIP_HERMES_QUALIFY=1");
|
||||
expect(fixtures).not.toMatch(/(?:API_KEY|TOKEN|AUTH_JSON|KEEP)/);
|
||||
expect(workflow.indexOf("Provision the pinned Python closure")).toBeLessThan(workflow.indexOf("Run native fixtures"));
|
||||
expect(workflow).toContain("no paid model, browser or Daytona proof");
|
||||
const upload = workflow.slice(workflow.indexOf(" - name: Upload fixture evidence"));
|
||||
expect(upload).toContain("hermes-native-evidence/");
|
||||
expect(upload).not.toMatch(/hermes-fixture-home|provider-assets|runtimeDirectory/);
|
||||
});
|
||||
it("provisions selected Hermes assets before credentials and uses the same selection for image identity and packs", async () => {
|
||||
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
|
||||
const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:"));
|
||||
|
||||
Reference in new issue
Block a user