diff --git a/.github/actions/setup-ci-dependencies/action.yml b/.github/actions/setup-ci-dependencies/action.yml index 3367001b4b5..0a78e542c91 100644 --- a/.github/actions/setup-ci-dependencies/action.yml +++ b/.github/actions/setup-ci-dependencies/action.yml @@ -65,6 +65,14 @@ runs: pnpm install --frozen-lockfile fi + - name: Replay workspace install hooks after restore + if: steps.installed.outputs.cache-hit == 'true' + shell: bash + working-directory: ${{ inputs.working-directory }} + run: | + pnpm -r --if-present run postinstall + pnpm run postinstall + - name: Restore certified shared build outputs (read only) if: inputs.restore-cache == 'true' && steps.inputs.outputs.cacheable == 'true' id: builds diff --git a/.github/scripts/tests/ci-dependency-cache.test.mjs b/.github/scripts/tests/ci-dependency-cache.test.mjs index 1b79b00c401..36195ad32e9 100644 --- a/.github/scripts/tests/ci-dependency-cache.test.mjs +++ b/.github/scripts/tests/ci-dependency-cache.test.mjs @@ -71,6 +71,36 @@ test('unmodelled workspace lifecycle scripts disable materialized reuse', () => } }); +test('known Copilot hooks permit caching, track their helpers, and still run on restore', () => { + const files = { ...entries, + 'server/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'packages/paperclip-runner/package.json': JSON.stringify({ scripts: { postinstall: 'node scripts/install-copilot-assets.mjs' } }), + 'server/scripts/install-copilot-assets.mjs': 'server bootstrap', + 'packages/paperclip-runner/scripts/install-copilot-assets.mjs': 'runner bootstrap', + 'packages/paperclip-runner/scripts/materialize-copilot-binary.mjs': 'native materializer', + 'packages/paperclip-runner/scripts/copilot-inner-distribution.mjs': 'closure pins', + }; + const compute = (input) => cachePlan({ files: Object.keys(input), + read: file => input[file] == null ? null : Buffer.from(input[file]), scope, + packageDirs: ['.', 'server', 'packages/paperclip-runner'], + }); + const original = compute(files); + assert.equal(original.cacheable, true); + for (const file of Object.keys(files).filter(file => /^(server|packages\/paperclip-runner)\/scripts\//.test(file))) { + for (const replacement of ['changed', null]) { + assert.notEqual(compute({ ...files, [file]: replacement }).dependencyKey, original.dependencyKey, file); + } + } + assert.equal(cachePlan({ files: ['elsewhere/package.json'], + read: () => Buffer.from(files['server/package.json']), scope, packageDirs: ['elsewhere'], + }).cacheable, false, 'the command is allowed only in its reviewed owners'); + const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); + const replay = action.split(' - name: Replay workspace install hooks after restore')[1].split(' - name:')[0]; + assert.match(replay, /if: steps\.installed\.outputs\.cache-hit == 'true'/); + assert.match(replay, /pnpm -r --if-present run postinstall\s+pnpm run postinstall/); + assert.ok(action.indexOf('Replay workspace install hooks') < action.indexOf('Validate workspace links')); +}); + test('cache restores are exact and only a frozen canonical master push can publish entries', () => { const action = readFileSync(new URL('../../actions/setup-ci-dependencies/action.yml', import.meta.url), 'utf8'); const installed = action.slice(action.indexOf(' - name: Restore installed'), action.indexOf(' - name: Locate pnpm')); @@ -97,6 +127,15 @@ test('paired test timings remain available when lifecycle hooks disable material assert.match(step(name), /if: steps.qualification.outputs.cacheable == 'true'/); } assert.match(step('Explain disabled materialized reuse'), /cacheable != 'true'/); - assert.doesNotMatch(step('Prepare unchanged baseline'), /if:/); - assert.doesNotMatch(step('Compare complete commands and executed case inventories'), /if:/); + for (const name of ['Prepare unchanged baseline', 'Compare complete commands and executed case inventories']) { + assert.match(step(name), /if: steps\.test_scope\.outputs\.run == 'true'/); + assert.doesNotMatch(step(name), /if:.*cacheable/); + } + const scope = step('Select changed server test work'); + assert.match(scope, /git diff --name-only HEAD\^1 HEAD --/); + for (const file of ['scripts/benchmark-ci-work.mjs', 'scripts/run-vitest-stable.mjs', + 'server/src/__tests__/setup-supertest.ts', 'server/src/__tests__/workspace-runtime.test.ts', + 'server/src/services/workspace-git-snapshot-streaming.test.ts']) assert.ok(scope.includes(file)); + assert.doesNotMatch(scope, /scripts\/ci-dependency-cache|\.github\/actions\/setup-ci-dependencies/); + assert.match(step('Exercise restored dependencies for cache-only changes'), /pnpm exec vitest run --project @paperclipai\/shared/); }); diff --git a/.github/scripts/tests/pr-runner-rust-cache.test.mjs b/.github/scripts/tests/pr-runner-rust-cache.test.mjs index 6089bcec638..1136e5cf150 100644 --- a/.github/scripts/tests/pr-runner-rust-cache.test.mjs +++ b/.github/scripts/tests/pr-runner-rust-cache.test.mjs @@ -48,11 +48,27 @@ test("every rust-cache restore in the PR workflow belongs to a guarded reader jo const keyInputs = [ /uses: Swatinem\/rust-cache@([0-9a-f]{40}) # v[0-9.]+/, /workspaces: (\$\{\{ steps\.runner_rust_workspace\.outputs\.path \}\} -> target)/, - /shared-key: (release-runner-v2)/, + /shared-key: (release-runner-v2-.+)/, /cache-workspace-crates: (false)/, /cache-bin: (false)/, ]; +test("virtual workspace profiles and ancestor Cargo configuration partition every dependency cache", () => { + for (const [name, workflow] of [["PR", prWorkflow], ["release", releaseWorkflow]]) { + const keys = [...workflow.matchAll(/^\s+shared-key: (.+)$/gm)].map(([, key]) => key); + assert.ok(keys.length > 0); + for (const key of keys) { + assert.ok(key.includes("hashFiles('packages/paperclip-runner/runner/Cargo.toml'"), name); + for (const directory of ['.', 'packages', 'packages/paperclip-runner', 'packages/paperclip-runner/runner']) { + const prefix = directory === '.' ? '' : `${directory}/`; + for (const file of ['config', 'config.toml']) { + assert.ok(key.includes(`'${prefix}.cargo/${file}'`), `${name}: ${prefix}.cargo/${file}`); + } + } + } + } +}); + test("every PR reader restores the Rust cache under the same key the master push writes", () => { for (const [name, pr] of readers) { for (const pattern of keyInputs) { diff --git a/.github/workflows/ci-work-benchmark.yml b/.github/workflows/ci-work-benchmark.yml index fe7ad35782d..96dffab12d5 100644 --- a/.github/workflows/ci-work-benchmark.yml +++ b/.github/workflows/ci-work-benchmark.yml @@ -7,6 +7,9 @@ on: - .github/actions/setup-ci-dependencies/** - .github/workflows/ci-work-benchmark.yml - scripts/ci-dependency-cache.mjs + - .github/workflows/pr-trusted.yml + - .github/workflows/release-verify.yml + - scripts/runner-debug-crate-cache.mjs - scripts/benchmark-ci-work.mjs - scripts/run-vitest-stable.mjs - server/src/__tests__/setup-supertest.ts @@ -39,6 +42,21 @@ jobs: fetch-depth: 2 persist-credentials: false + - name: Select changed server test work + id: test_scope + run: | + changed=$(git diff --name-only HEAD^1 HEAD -- \ + scripts/benchmark-ci-work.mjs \ + scripts/run-vitest-stable.mjs \ + server/src/__tests__/setup-supertest.ts \ + server/src/__tests__/workspace-runtime.test.ts \ + server/src/services/workspace-git-snapshot-streaming.test.ts) + if [ -n "$changed" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 with: @@ -123,6 +141,7 @@ jobs: echo "Materialized dependency reuse is disabled by an unmodelled install hook. Retaining cold setup and comparing test commands only." >> "$GITHUB_STEP_SUMMARY" - name: Prepare unchanged baseline + if: steps.test_scope.outputs.run == 'true' run: | base_sha=$(git rev-parse HEAD^1) git worktree add --detach ../baseline "$base_sha" @@ -135,12 +154,17 @@ jobs: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps - name: Compare complete commands and executed case inventories + if: steps.test_scope.outputs.run == 'true' env: BENCHMARK_OUTPUT: ${{ runner.temp }}/ci-work-benchmark run: | node scripts/benchmark-ci-work.mjs ../baseline "$BENCHMARK_OUTPUT" cat "$BENCHMARK_OUTPUT/summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Exercise restored dependencies for cache-only changes + if: steps.test_scope.outputs.run != 'true' && steps.qualification.outputs.cacheable == 'true' + run: pnpm exec vitest run --project @paperclipai/shared + - name: Retain timings, inventories and failure logs if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a @@ -148,3 +172,120 @@ jobs: name: ci-work-benchmark path: ${{ runner.temp }}/ci-work-benchmark retention-days: 14 + + rust_prime: + if: >- + contains(github.event.pull_request.labels.*.name, 'profile-ci') && + (github.event.action != 'labeled' || github.event.label.name == 'profile-ci') + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'true' + - name: Build the exact debug units + working-directory: packages/paperclip-runner + run: cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Save workspace artifacts in this PR only + id: artifacts + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + + rust_restore: + needs: rust_prime + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Select the pinned compiler and workspace path + id: workspace + working-directory: packages/paperclip-runner + run: | + toolchain=$(rustup show active-toolchain | awk '{print $1}') + echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV" + rustup toolchain list | awk '{print $1}' | grep -vx "$toolchain" | xargs -r -n1 rustup toolchain uninstall || true + pinned="$HOME/paperclip-runner-rust" + ln -s "$GITHUB_WORKSPACE/packages/paperclip-runner/runner" "$pinned" + echo "path=$pinned" >> "$GITHUB_OUTPUT" + - name: Qualify the dependency cache + id: dependencies + uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + workspaces: ${{ steps.workspace.outputs.path }} -> target + shared-key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} + cache-workspace-crates: false + cache-bin: false + save-if: 'false' + - name: Compute workspace artifact identity + id: debug + run: node scripts/runner-debug-crate-cache.mjs plan + - name: Restore workspace artifacts in this PR only + id: artifacts + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + key: qualification-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.debug.outputs.key }} + path: ${{ steps.debug.outputs.paths }} + - name: Verify fresh-host cache reuse + env: + DEPENDENCIES_HIT: ${{ steps.dependencies.outputs.cache-hit }} + ARTIFACTS_HIT: ${{ steps.artifacts.outputs.cache-hit }} + run: | + test "$DEPENDENCIES_HIT" = true + test "$ARTIFACTS_HIT" = true + node scripts/runner-debug-crate-cache.mjs mark-current + cd packages/paperclip-runner + started=$(date +%s) + cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins --message-format=json > "$RUNNER_TEMP/rust-restored-build.jsonl" + echo "Restored Rust build: $(($(date +%s) - started)) seconds." >> "$GITHUB_STEP_SUMMARY" + node --input-type=module <<'NODE' + import { readFileSync } from 'node:fs'; + const artifacts = readFileSync(`${process.env.RUNNER_TEMP}/rust-restored-build.jsonl`, 'utf8') + .trim().split('\n').map(line => JSON.parse(line)).filter(row => row.reason === 'compiler-artifact'); + if (!artifacts.length || artifacts.some(row => !row.fresh)) { + throw new Error(`Cache hit rebuilt units: ${JSON.stringify(artifacts.filter(row => !row.fresh).map(row => row.package_id))}`); + } + console.log(`All ${artifacts.length} Cargo units were fresh on a new host.`); + NODE + - name: Retain Cargo reuse evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: rust-cache-qualification + path: ${{ runner.temp }}/rust-restored-build.jsonl + retention-days: 14 diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 455512ee2de..420f1d98259 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -370,7 +370,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -784,7 +784,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1035,7 +1035,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false @@ -1376,7 +1376,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Mirror the master writer: these also feed the cache key. cache-workspace-crates: false cache-bin: false diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index a8a29e433b5..b0319840ec1 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -92,7 +92,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-typecheck-v2 + shared-key: release-typecheck-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -398,7 +398,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 with: workspaces: ${{ steps.runner_rust_workspace.outputs.path }} -> target - shared-key: release-runner-v2 + shared-key: release-runner-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }} # Rebuild workspace code and rerun every check. Cache only compiled # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false @@ -680,7 +680,7 @@ jobs: uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 with: workspaces: ${{ steps.native_rust_workspace.outputs.path }} -> target - shared-key: release-native-v2-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} + shared-key: release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', '.cargo/config', '.cargo/config.toml', 'packages/.cargo/config', 'packages/.cargo/config.toml', 'packages/paperclip-runner/.cargo/config', 'packages/paperclip-runner/.cargo/config.toml', 'packages/paperclip-runner/runner/.cargo/config', 'packages/paperclip-runner/runner/.cargo/config.toml') }}-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }} cache-workspace-crates: false cache-bin: false cache-on-failure: false diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 8976eccb990..ec278667695 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1095,6 +1095,22 @@ verifying Vite/HMR behavior. For normal issue work, start with the smallest targeted check that proves the change. Reserve repo-wide typecheck/build/test runs for PR-ready handoff or changes broad enough that narrow checks do not cover the risk. +The Runner transport suite keeps attachment recovery, warm attachment, live +adoption, and suspension in separate test files. Vitest can schedule these +independent fixtures on its existing file workers; cases within each file stay +serial. The package sequencer starts the ten files measured above 50 seconds in +the November 4, 2026 AWS run first, ordered by approximate duration. This includes +transport deadlines, settlement, provider launch, recovery, and streaming waits, +so fresh CI runners do not have to infer duration from file size. Package CI uses +up to four workers, bounded by available CPUs; local runs retain Vitest defaults. +Vitest's file selection and shard membership are unchanged. Real lease/deadline +waits and assertions are unchanged. Run the complete +family, including the separate deadline and settlement regressions, with: + +```sh +pnpm --filter @paperclipai/paperclip-runner exec vitest run runnerd-codex-transport +``` + #### Embedded Postgres template cache To run the suite with one owned PostgreSQL server and a separate disposable @@ -3187,7 +3203,7 @@ Run the credential-free real-process restart suite with: ```sh pnpm --filter @paperclipai/paperclip-runner build:runner-binaries pnpm exec vitest run server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts -pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport.test.ts -t 'adopts a live runner' +pnpm --filter @paperclipai/paperclip-runner exec vitest run src/live/runnerd-codex-transport-adoption.test.ts -t 'adopts a live runner' ``` The suite uses isolated PostgreSQL state, isolated `PAPERCLIP_HOME` roots, real diff --git a/doc/RELEASE-AUTOMATION-SETUP.md b/doc/RELEASE-AUTOMATION-SETUP.md index 324a6adb07e..021188f2e53 100644 --- a/doc/RELEASE-AUTOMATION-SETUP.md +++ b/doc/RELEASE-AUTOMATION-SETUP.md @@ -368,7 +368,14 @@ readiness can succeed. A failed lane does not cancel the other lane. Both lanes restore Cargo dependencies with the pinned Rust Cache action. The compiler comes from the Runner package's `rust-toolchain.toml` before the action computes its key. Compiler and Cargo metadata changes select a new cache. The -`release-runner-v2` shared key avoids separate copies for these lanes. +`release-runner-v2` shared key avoids separate copies for these lanes. Its suffix +hashes the virtual workspace's `runner/Cargo.toml` and ancestor Cargo configuration. +Rust Cache discovers member manifests through Cargo metadata, which omits this +virtual manifest. Without the suffix, changing `[profile.dev]` can restore an +exact but obsolete dependency cache on every run. GitHub caches are immutable, +so the successful rebuild cannot replace that old entry. The profile/configuration +hash also partitions fallback restores. Every reader and writer uses the same +inputs, including the separate typecheck and cross-target caches. Only the Rust lane saves this cache. After verification it also runs `build:rust` to warm the debug dependencies used by the protocol lane; its own tests already warm release dependencies. The cache writer is shorter than the protocol lane. @@ -385,13 +392,36 @@ paths and the PR lanes can restore master's entry. The guard tests under Rust cache step in both workflows. Workspace crates and installed Cargo binaries are excluded from this entry. -Every run runs all assigned checks, including on a cache hit. Only an -own-repository master-push run verifying that push's exact SHA can restore the -cache, and only a successful Rust lane saves it. PR, tag, and manual candidate +Every run runs all assigned checks, including on a cache hit. In release +verification, only an own-repository master-push run verifying that push's exact +SHA can restore the cache, and only a successful Rust lane saves it. The PR +workflow restores master's cache read only. Tag and manual candidate release verification compile without this cache. A miss or eviction costs compilation time but does not change the checks. To discard old dependency caches, increment the shared-key version and let the next successful master verification warm it. +### Installed JavaScript dependencies + +`setup-ci-dependencies` can restore exact `node_modules` trees and separately +certified shared/SDK build outputs. `scripts/ci-dependency-cache.mjs` hashes +manifests, resolution, patches, install scripts, and the runtime/checkout layout. +The root workspace-link hook and the server/Runner Copilot asset hooks are known +postinstall commands. An exact dependency restore replays these hooks before +validating workspace links and build outputs. Copilot defers installation in an +unbuilt source checkout; its normal build installs and verifies the assets. +Provider assets are not included in the dependency cache. Any unrecognized +workspace lifecycle hook still disables materialized reuse. Only a frozen install +from the canonical master push can publish the shared entries. + +For CI cache changes, the `profile-ci` label runs the paired setup benchmark +and exercises restored dependencies with the shared-package tests. The longer +server test comparison runs when its measured scripts or fixtures change; +cache-only changes do not require unchanged test commands to run faster. +It also builds Rust units on one GitHub-hosted runner and restores the dependency +and workspace caches on a second runner with a fresh checkout. Both restores +must be exact, and Cargo must report every unit as fresh. These qualification +caches are scoped to the PR and run; they cannot populate master's caches. + ### Runner release crate cache The dependency entry leaves the Runner's own crate out, and a fresh checkout diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.mjs new file mode 100644 index 00000000000..c8069693070 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.mjs @@ -0,0 +1,27 @@ +import { BaseSequencer } from "vitest/node"; + +const slowFiles = [ + "runnerd-codex-transport.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "cursor-cloud-runner-launch.test.ts", + "runnerd-codex-transport-deadlines.test.ts", + "opencode-server-driver.test.ts", + "acpx-controller-recovery.test.ts", + "runnerd-codex-transport-warm-attachment.test.ts", + "runnerd-opencode-streaming-tools.test.ts", + "live-session.test.ts", + "runnerd-codex-transport-adoption.test.ts", +]; + +const priority = (file) => { + const name = file.moduleId.replaceAll("\\", "/").split("/").at(-1); + const index = slowFiles.indexOf(name); + return index < 0 ? slowFiles.length : index; +}; + +export class RunnerSequencer extends BaseSequencer { + async sort(files) { + const sorted = await super.sort(files); + return sorted.sort((a, b) => priority(a) - priority(b)); + } +} diff --git a/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs new file mode 100644 index 00000000000..781fe81d8d1 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-sequencer.test.mjs @@ -0,0 +1,61 @@ +import assert from "node:assert/strict"; +import { test } from "vitest"; +import { BaseSequencer } from "vitest/node"; +import { RunnerSequencer } from "./vitest-sequencer.mjs"; + +const project = { name: "runner", config: { isolate: true, sequence: { groupOrder: 0 } } }; +const files = [ + "large.test.ts", + "runnerd-codex-transport.test.ts", + "small.test.ts", + "runnerd-codex-transport-settlement.test.ts", + "runnerd-codex-transport-deadlines.test.ts", +].map((name) => ({ moduleId: `/repo/src/live/${name}`, project })); +const ctx = { + config: { root: "/repo", shard: { index: 1, count: 2 } }, + cache: { + getFileTestResults: () => undefined, + getFileStats: (key) => ({ size: 100 - files.findIndex((file) => key.endsWith(file.moduleId.slice(6))) }), + }, +}; + +test("starts real-wait transport files first without changing test membership", async () => { + const sorted = await new RunnerSequencer(ctx).sort(files); + assert.deepEqual(sorted, [files[1], files[3], files[4], files[0], files[2]]); + assert.equal(new Set(sorted).size, files.length); + assert.deepEqual(await new BaseSequencer(ctx).sort(files), files); +}); + +test("selected subsets stay selected and retain default ordering", async () => { + const selected = [files[2], files[0]]; + assert.deepEqual(await new RunnerSequencer(ctx).sort(selected), await new BaseSequencer(ctx).sort(selected)); + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[4]]), [files[4]]); + assert.deepEqual(await new RunnerSequencer(ctx).sort([]), []); +}); + +test("keeps Vitest shard membership and schedules waits within each shard", async () => { + const all = []; + for (const index of [1, 2]) { + const shardCtx = { ...ctx, config: { ...ctx.config, shard: { index, count: 2 } } }; + const sequencer = new RunnerSequencer(shardCtx); + const shard = await sequencer.shard(files); + assert.deepEqual(shard, await new BaseSequencer(shardCtx).shard(files)); + const sorted = await sequencer.sort(shard); + assert.deepEqual(new Set(sorted), new Set(shard)); + all.push(...sorted); + } + assert.equal(all.length, files.length); + assert.deepEqual(new Set(all), new Set(files)); +}); + +test("recognizes Windows paths", async () => { + const deadline = { ...files[4], moduleId: "C:\\repo\\src\\live\\runnerd-codex-transport-deadlines.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], deadline]), [deadline, files[0]]); +}); + +test("starts measured slow provider files before short files without adding work", async () => { + const streaming = { ...files[0], moduleId: "/repo/src/live/runnerd-opencode-streaming-tools.test.ts" }; + const launch = { ...files[0], moduleId: "/repo/src/control-plane/cursor-cloud-runner-launch.test.ts" }; + assert.deepEqual(await new RunnerSequencer(ctx).sort([files[0], streaming, files[4], launch]), + [launch, files[4], streaming, files[0]]); +}); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts new file mode 100644 index 00000000000..424db66444b --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts @@ -0,0 +1,556 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +async function verifyLiveRunnerAdoption( + mismatchedCheckpoint: boolean, + mismatchedArtifact = false, + goalMidTurn = false, + detachBeforeCleanup = false, + startWithoutCheckpoint = false, + preparedTask = false, + adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", + toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", + uncertainAtClose = false, + semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", + stopExitProofFault?: "missing" | "false", +) { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); + const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; + const completionPath = join(stateDirectory, "pending-completion.json"); + if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ + schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", + completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [], + })); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + const checkpoint = vi.fn(async () => undefined); + server.on("upgrade", (request, socket, head) => { + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Expected adoption test listener"); + const registration = vi.fn(async (next: DurablePrpControlPlane) => { + authority = next; + return { + connectUrl: `ws://127.0.0.1:${address.port}/runner`, + ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), + release: async () => { + if (authority === next) authority = null; + }, + }; + }); + const identity = { + runnerInstanceId: "runner-live-adopt", + environmentLeaseId: "lease-live-adopt", + runId: "run-live-adopt", + normalizedSessionId: "session-live-adopt", + turnId: "turn-live-adopt", + itemId: "item-live-adopt", + }; + const sharedOptions = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), + stateDirectory, + prpIdentity: identity, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + controlPlaneRegistration: registration, + }; + const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ + configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), + })); + const options = { ...sharedOptions, ...(preparedTask ? { + prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, + detached: process.platform !== "win32", stdio: "ignore" }); + return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }) }; + }, + } : {}) }; + const first = createCapabilityRunnerdCodexTransport(options); + const pendingTool = vi.fn(() => new Promise>(() => undefined)); + if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); + let releaseRecovery!: () => void; + const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); + const recoveryDenied = new Error("execution authority revoked after Stop"); + const recoveredTool = vi.fn(async () => { + if (semanticRecoveryMode) { + // Model server authority denying replay after Stop cleared the task lock. + // The gate holds that denial until reattachment has returned its session, + // before the caller invokes the eventual onSession cancellation callback. + await recoveryGate; + throw recoveryDenied; + } + return { result: { receipt: "known-outcome" }, isError: false }; + }); + const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); + let runnerPid: number | null = null; + let runnerStartedAt: string | null = null; + let adopted: ReturnType | null = + null; + let restoreIntervalSpies: () => void = () => undefined; + try { + let opened: Record; + try { + opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), + }); + } catch (error) { + const stderr = await readFile( + join(stateDirectory, "diagnostics", "runnerd.stderr.log"), + "utf8", + ).catch(() => ""); + throw new Error( + `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, + ); + } + runnerPid = first.evidence().runnerPid; + runnerStartedAt = first.transport.processInfo().startedAt; + expect(runnerPid).toEqual(expect.any(Number)); + + if (goalMidTurn) { + await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/started") break; + } + } + + if (toolHandlerOrder) { + await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); + await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); + } + await first.detachControllerForRestart(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + if (goalMidTurn) { + await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); + // runnerd need not poll the provider into its PRP outbox while disconnected. + // Wait for flushed provider output, not a platform-dependent final poll + // racing the disconnect. Adoption must still bind that buffered item. + await vi.waitFor(async () => { + expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); + }, { timeout: 5_000 }); + } + + const controlPlaneStatePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + const compactProviderIdentityEvents = async () => { + const controlPlaneState = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { committedEvents: Array<{ eventType: string }> }; + controlPlaneState.committedEvents = + controlPlaneState.committedEvents.filter( + (event) => + event.eventType !== "harness.ready" && + event.eventType !== "session.started" && + event.eventType !== "session.resumed", + ); + await writeFile( + controlPlaneStatePath, + `${JSON.stringify(controlPlaneState, null, 2)}\n`, + { mode: 0o600 }, + ); + }; + if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); + + if (preparedTask) { + if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); + prepareLaunch.mockClear(); + registration.mockClear(); + if (adoptionAuthorization === "missing-binding") { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + delete retained.preparedLaunchConfigurations; + await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); + } + } + const authorizeRunnerAdoption = vi.fn(async () => { + expect(registration).not.toHaveBeenCalled(); + if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); + }); + const duplicateLauncher = vi.fn(() => { + throw new Error("duplicate runner spawn attempted"); + }); + const openedThread = opened.thread as Record; + const signal = vi.fn(() => true); + const probeError = new Error("fixture remote process identity is unavailable"); + let rejectLiveness = false; + const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; + const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; + restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; + adopted = createCapabilityRunnerdCodexTransport({ + ...options, + ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), + ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), + ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), + // Hash different stable bytes without replacing the real runner artifact + // used by concurrent tests. Adoption must never execute this path. + ...(mismatchedArtifact + ? { + runnerBinary: resolve(import.meta.dirname, "../../package.json"), + runnerReconnectGraceMs: 150, + } + : {}), + ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), + resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], + resumeProviderSession: startWithoutCheckpoint ? undefined : { + driverSessionId: String(openedThread.id), + providerSessionId: mismatchedCheckpoint + ? "wrong-provider-session" + : String(openedThread.sessionId), + }, + runnerProcessLauncher: duplicateLauncher, + adoptExistingRunner: { + pid: runnerPid!, + processGroupId: runnerPid, + startedAt: runnerStartedAt, + signal, + isAlive: () => { + if (rejectLiveness) return Promise.reject(probeError); + try { + process.kill(runnerPid!, 0); + return true; + } catch { + return false; + } + }, + }, + }); + if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { + await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( + adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" + ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); + expect(registration).not.toHaveBeenCalled(); + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); + return; + } + if (mismatchedArtifact) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + expect(authority?.activeRunnerConnectionCount()).toBe(0); + await expect( + adopted.transport.request("turn/start", { + input: [{ type: "text", text: "must not be dispatched" }], + }), + ).rejects.toThrow("native_adopted_runner_authentication_timeout"); + await adopted.transport.close(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse( + await readFile(controlPlaneStatePath, "utf8"), + ) as { + identity: unknown; + commands: Array<{ type: string; status: string }>; + }; + expect(retained.identity).toEqual(identity); + expect( + retained.commands.some( + (command) => + command.type === "runner.drain" && command.status === "pending", + ), + ).toBe(true); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["runner.suspend"]), + ); + expect(retained.commands.map((command) => command.type)).not.toEqual( + expect.arrayContaining(["turn.stop"]), + ); + return; + } + if (mismatchedCheckpoint) { + await expect( + adopted.transport.request("thread/read", {}), + ).rejects.toThrow("native_adopted_provider_identity_mismatch"); + expect(duplicateLauncher).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + return; + } + if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); + await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( + expect.objectContaining({ + thread: expect.objectContaining({ id: "codex-thread-1" }), + }), + ); + if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); + if (semanticRecoveryMode) { + const retained = authority as DurablePrpControlPlane | null; + expect(retained).not.toBeNull(); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + if (semanticRecoveryMode === "denied-before-cancel") { + await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); + // This is a real Core replay callback, not an artificial transport error. + // Deliver its denial before giving the caller its cancellation turn. + releaseRecovery(); + await vi.waitFor(async () => { + await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); + }); + await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); + expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); + } else { + // Let all startup/handler microtasks and a real reconciliation read run. + // Cleanup must not enter replay even while its authority would deny it. + await adopted.transport.request("thread/read", {}); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + releaseRecovery(); + await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); + expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); + } + expect(redispatchedTool).not.toHaveBeenCalled(); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (semanticRecoveryMode === "cleanup-only") { + const faultedStopProof = vi.fn(); + if (stopExitProofFault) { + const queue = retained!.queueCommand.bind(retained!); + vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { + const command = queue(type, ...args); + if (type === "turn.stop") { + let result = command.result; + Object.defineProperty(command, "result", { + enumerable: true, + configurable: true, + get: () => result, + set: value => { + const receipt = value as { result?: Record } | null; + if (receipt?.result && typeof receipt.result === "object") { + faultedStopProof(receipt.result.providerExitConfirmed); + const stopped = { ...receipt.result }; + if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; + else stopped.providerExitConfirmed = false; + result = { ...receipt, result: stopped }; + } else result = value; + }, + }); + } + return command; + }); + await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect(faultedStopProof).toHaveBeenCalledWith(true); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); + } else { + await expect(adopted.transport.close()).resolves.toBeUndefined(); + expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); + expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ + status: "completed", result: { result: { providerExitConfirmed: true } }, + }]); + } + expect(retained!.semanticToolResultsSettled()).toBe(false); + expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); + expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); + if (!stopExitProofFault) { + expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); + const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); + } + const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).not.toHaveBeenCalled(); + } + return; + } + if (toolHandlerOrder) { + await vi.waitFor(() => { + expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); + const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); + expect(result?.status).toBe("completed"); + }, { timeout: 5_000 }); + if (pendingCompletion) { + expect(recoveredTool).not.toHaveBeenCalled(); + expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); + } else expect(redispatchedTool).not.toHaveBeenCalled(); + } + expect(adopted.evidence().runnerPid).toBe(runnerPid); + expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); + if (startWithoutCheckpoint) { + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + for (const type of ["run.prepare", "session.open"]) { + expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); + } + } + if (goalMidTurn) { + const observed = await Promise.race([ + (async () => { + for await (const notification of adopted!.transport.notifications()) { + if (notification.method === "item/started") return notification; + } + throw new Error("recovered goal item was lost"); + })(), + new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), + ]); + expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); + } + expect(duplicateLauncher).not.toHaveBeenCalled(); + if (preparedTask) { + if (adoptionAuthorization === "legacy") { + expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); + expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); + } else { + expect(prepareLaunch).not.toHaveBeenCalled(); + expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); + } + expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); + } + if (uncertainAtClose) { + const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const retainedAuthority = authority as DurablePrpControlPlane | null; + rejectLiveness = true; + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); + expect(intervals!.mock.results.length).toBeGreaterThan(0); + for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); + const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); + let queueDeadline: ReturnType | undefined; + try { + expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); + } finally { clearTimeout(queueDeadline); } + await expect(adopted.transport.close()).rejects.toBe(probeError); + expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(signal).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + expect(duplicateLauncher).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + expect(retained.identity).toEqual(before.identity); + expect(retained.commands).toEqual(before.commands); + return; + } + expect(adopted.evidence().diagnostics).toContain( + `adopted runner ${runnerPid} authenticated to its durable PRP authority`, + ); + if (!startWithoutCheckpoint) { + expect(adopted.evidence().diagnostics).toContain( + "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", + ); + expect(adopted.evidence().diagnostics).toContain( + "confirmed adopted provider identity against authenticated recovery session.snapshot", + ); + } + if (detachBeforeCleanup) { + await adopted.detachControllerForRestart(); + await adopted.transport.close("old controller finalizer"); + expect(signal).not.toHaveBeenCalled(); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); + const commandTypes = retained.commands.map((command: { type: string }) => command.type); + expect(commandTypes).not.toContain("turn.stop"); + expect(commandTypes).not.toContain("runner.suspend"); + } + } finally { + releaseRecovery(); + await adopted?.transport.close().catch(() => undefined); + if (uncertainAtClose) { + const providerPid = adopted?.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } + } + await (authority as DurablePrpControlPlane | null)?.stop(); + } + restoreIntervalSpies(); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // The adopted runner normally exits after its durable suspend command. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +it( + "adopts a live runner on the same durable authority without spawning a duplicate", + () => verifyLiveRunnerAdoption(false), + 30_000, +); + +it("adopts a prepared runner after losing its controller launch cache without preparing resources again", + () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); + +it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", + mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); + +it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); + +it( + "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", + () => verifyLiveRunnerAdoption(false, true), + 15_000, +); + +it( + "rejects a live runner whose provider identity mismatches the compacted checkpoint", + () => verifyLiveRunnerAdoption(true), + 30_000, +); + +it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); + +it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); + +it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); + +it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); + +it("exposes semantic recovery before cancellation when the authority has been revoked", () => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); + +it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); + +it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => + verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts new file mode 100644 index 00000000000..72b95a8206c --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts @@ -0,0 +1,1077 @@ +import { + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type PreparedRunnerProviderLaunch, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; +import { cp, lstat, mkdtemp, readFile, readdir, readlink, rm, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it.each([ + ...[ + "before-result", + "after-result", + "after-activation", + "before-confirmation", + "after-confirmation", + ].flatMap((lossPoint) => + [false, true].map((routed) => ({ + lossPoint, + routed, + recoveryFault: "none", + })), + ), + ...["endpoint", "missing-capability", "listen", "malformed-core"].map( + (recoveryFault) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault, + }), + ), + ...["before_bootstrap", "before_spawn", "before_authentication"].map( + (stage) => ({ + lossPoint: "after-result", + routed: true, + recoveryFault: `authorize_${stage}`, + }), + ), + ...[ + "attach-wait", + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? ["attach-capability"] + : []), + ].map((recoveryFault) => ({ + lossPoint: "before-result", + routed: true, + recoveryFault, + })), + ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER + ? [ + { + lossPoint: "after-result", + routed: false, + recoveryFault: "legacy-parser", + }, + ] + : []), + ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ + lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, + })), + { + lossPoint: "after-result", + routed: true, + recoveryFault: "none", + externalArtifact: true, + }, + { + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup: true, + }, + ...[ + "missing_snapshot", + "rejected_snapshot", + "wrong_thread", + "callback_failure", + "callback_async_failure", + ].map((ordinaryFollowup) => ({ + lossPoint: "after-confirmation", + routed: true, + recoveryFault: "none", + ordinaryFollowup, + })), +])( + "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", + async (testCase) => { + const { lossPoint, routed, recoveryFault } = testCase; + const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; + const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; + const configurations = new Map(); + const ordinaryFollowup = + "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; + const snapshotFault = + typeof ordinaryFollowup === "string" && + !ordinaryFollowup.startsWith("callback_"); + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-restart-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const routes = new Map< + string, + { core: DurablePrpControlPlane; generation: symbol } + >(); + const routeCalls: string[] = []; + let recovering = false; + let recoveryClaimCurrent = true; + let recoveryFenceActive = true; + const completionSnapshotIds: string[] = []; + let rejectHeldCompletion: ((error: Error) => void) | undefined; + let markHeldCompletionEntered!: () => void; + const heldCompletionEntered = new Promise((resolve) => { + markHeldCompletionEntered = resolve; + }); + let snapshotObserverSpy: ReturnType | undefined; + const routeServer = createServer((_request, response) => + response.writeHead(404).end(), + ); + routeServer.on("upgrade", (request, socket, head) => { + const entry = routes.get(request.url ?? ""); + if (!entry) { + socket.destroy(); + return; + } + entry.core.handleUpgrade(request, socket, request.url!, head); + }); + if (routed) + await new Promise((resolveListen) => + routeServer.listen(0, "127.0.0.1", resolveListen), + ); + const routeAddress = routeServer.address(); + const routePort = + routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; + const registration = async ( + core: DurablePrpControlPlane, + identity = core.store.state.identity, + ) => { + if ( + recovering && + ordinaryFollowup && + recoveryFenceActive && + !recoveryClaimCurrent + ) { + throw new Error("fixture old recovery claim is no longer current"); + } + const path = `/api/runner/v1/connect/${identity.runId}`; + const generation = Symbol(); + routes.set(path, { core, generation }); + routeCalls.push(identity.runId); + return { + connection: + recovering && recoveryFault === "listen" + ? { + mode: "listen" as const, + listenAddress: "0.0.0.0" as const, + listenPort: routePort, + listenPath: path, + } + : { + mode: "connect" as const, + connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, + }, + release: () => { + if (routes.get(path)?.generation === generation) routes.delete(path); + }, + }; + }; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + if (recovering && snapshotFault) { + const getCommand = core.getCommand.bind(core); + snapshotObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + const command = getCommand(id); + if ( + command?.type !== "session.snapshot" || + command.status !== "completed" + ) + return command; + const observed = structuredClone(command); + if (ordinaryFollowup === "rejected_snapshot") { + observed.status = "failed"; + observed.result = { + result: { message: "fixture snapshot observation rejected" }, + }; + } else if (ordinaryFollowup === "wrong_thread") { + observed.result = { + ...observed.result, + result: { + ...(observed.result?.result as Record), + driverSessionId: "foreign-thread", + providerSessionId: "foreign-thread", + }, + }; + } else observed.result = { result: { status: "session_open" } }; + return observed; + }); + } + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch({ ...options, + ...(executionPrepared ? { processLauncher: undefined } : {}), + ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), + }); + handles.push(handle); + return handle; + }); + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const stopOwnedProvider = async (pid: number) => { + // Exact runner completion can race the OS reaping its already-signalled + // provider. Absence, not the outcome of a redundant signal, is required. + try { + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 500, + interval: 10, + }); + } catch { + try { + process.kill(-pid, "SIGKILL"); + } catch (error) { + if ( + !["ESRCH", "EPERM"].includes( + String((error as NodeJS.ErrnoException).code), + ) + ) + throw error; + } + await vi.waitFor(() => expect(dead(-pid)).toBe(true), { + timeout: 2_000, + interval: 10, + }); + } + expect(dead(pid)).toBe(true); + }; + const fingerprintTree = async (root: string): Promise => { + const rows: unknown[] = []; + const visit = async (path: string, relative: string) => { + const metadata = await lstat(path); + rows.push({ + path: relative, + inode: metadata.ino, + mode: metadata.mode, + mtimeMs: metadata.mtimeMs, + digest: metadata.isFile() + ? createHash("sha256") + .update(await readFile(path)) + .digest("hex") + : metadata.isSymbolicLink() + ? createHash("sha256") + .update(await readlink(path)) + .digest("hex") + : null, + }); + if (metadata.isDirectory()) + for (const child of (await readdir(path)).sort()) + await visit(join(path, child), `${relative}/${child}`); + }; + await visit(root, "."); + return rows; + }; + const within = async (promise: Promise, timeout = 5_000) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error("warm restart fixture timed out")), + timeout, + ); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + }; + const options = { + runnerBinary: + recoveryFault === "attach-capability" + ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! + : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary()), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + prpIdentity: (() => { + const runId = randomUUID(); + return { + runnerInstanceId: randomUUID(), + environmentLeaseId: + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === + "transient" + ? runId + : randomUUID(), + runId, + normalizedSessionId: randomUUID(), + turnId: `turn-${runId}`, + itemId: `item-${runId}`, + }; + })(), + ...(routed + ? { + controlPlaneRegistration: registration, + warmTransitionRegistrationMode: "routed_connect" as const, + } + : {}), + ...(executionPrepared ? { + runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), + readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), + // The fixture's spawn spy supplies the real local process owner. + runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, + prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { + const existing = configurations.get(requirements.identity.runId); + if (existing) { + expect(requirements).toEqual(existing.requirements); + if (recovering) expect(recovery).toEqual(existing.prepared); + return existing.prepared; + } + const prepared = { configurationId: `prepared-${requirements.identity.runId}`, + requirementsDigest: runnerLaunchRequirementsDigest(requirements), + provider: createRunnerProviderLaunch(requirements, { + options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, + root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), + codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", + }), + }; + configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); + return prepared; + }, + } : {}), + }; + const first = createCapabilityRunnerdCodexTransport(options); + let resumed: + ReturnType | undefined; + const providerPids = new Set(); + let commitSpy: ReturnType | undefined; + let commandObserverSpy: ReturnType | undefined; + let detached: Promise | undefined; + let cleanupProven = false; + const legacyProbeDirectories: string[] = []; + try { + const opened = await within( + first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + const openedThread = opened.thread as { id: string; sessionId: string }; + const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; + const firstThreadSnapshot = await within( + first.transport.request("thread/read", {}), + ); + const firstEvidence = first.evidence(); + const cleanRunnerStateBytes = await readFile( + join(stateDirectory, "runner/runner-state.json"), + ); + const runnerProcessStartedAt = + process.platform === "darwin" + ? new Date( + execFileSync( + "ps", + ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], + { encoding: "utf8", timeout: 1_500 }, + ).trim(), + ).toISOString() + : null; + if ((first.evidence().codexPid ?? 0) > 0) + providerPids.add(first.evidence().codexPid!); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + const nextRunId = randomUUID(); + const desired = { + ...oldIdentity, + runId: nextRunId, + turnId: `turn-${nextRunId}`, + itemId: `item-${nextRunId}`, + }; + if ( + recoveryFault === "attach-capability" || + recoveryFault === "attach-wait" + ) { + if (recoveryFault === "attach-wait") { + const getCommand = core.getCommand.bind(core); + commandObserverSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((id) => { + if ( + core.store.state.commands.find( + (entry) => entry.commandId === id, + )?.type === "run.attach" + ) { + throw new Error("fixture result observer unavailable"); + } + return getCommand(id); + }); + } + const bootstrapCount = core.store.state.freshBootstraps; + await expect( + first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }), + ).rejects.toThrow( + recoveryFault === "attach-capability" + ? "capability is required" + : "result observer unavailable", + ); + expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${oldIdentity.runId}`, + ]); + expect(core.store.state.freshBootstraps).toBe(bootstrapCount); + expect(handles).toHaveLength(1); + if (recoveryFault === "attach-capability") + expect( + core.store.state.commands.some( + (entry) => entry.type === "run.attach", + ), + ).toBe(false); + return; + } + const commit = core.store.commit.bind(core.store); + let lossObserved = false; + let signalLoss!: () => void; + const loss = new Promise((resolveLoss) => { + signalLoss = resolveLoss; + }); + commitSpy = vi + .spyOn(core.store, "commit") + .mockImplementation((candidate) => { + // The selected process-crash boundary stays unavailable until the + // exact owned runner is joined. A later replay must not silently + // settle this fixture before its retained pair is exported. + if (lossObserved) + throw new Error( + "fixture controller persistence unavailable after loss", + ); + const phase = candidate.warmTransition?.phase; + const target = + lossPoint === "after-activation" ? "activated" : "prepared"; + const atBoundary = lossPoint.includes("confirmation") + ? candidate.completedWarmTransition !== undefined && + candidate.warmTransition === undefined + : phase === target; + if (!lossObserved && atBoundary) { + lossObserved = true; + if ( + lossPoint !== "before-result" && + lossPoint !== "before-confirmation" + ) + commit(candidate); + // A dead controller cannot accept a reconnect through the second + // route installed for the in-flight authority rotation. Detaching + // only the current route leaves that overlapping fixture listener + // alive; completed-receipt replay needs no further store commit. + for (const [path, entry] of routes) { + if (entry.core === core) routes.delete(path); + } + core.disconnectActiveRunner(); + detached = first.transport.detachControllerForRestart!(); + signalLoss(); + // A committed confirmation is still before its ACK. Returning from + // this hook lets the current frame handler send that ACK even after + // close begins, collapsing the intended crash window on fast peers. + throw new Error("fixture interrupted exact result commit"); + } + commit(candidate); + }); + const attachment = first.transport.attachRun!({ + runId: desired.runId, + turnId: desired.turnId, + itemId: desired.itemId, + }); + void attachment.catch(() => undefined); + await within(loss); + await within(detached!); + expect(lossObserved).toBe(true); + const runnerPath = join(stateDirectory, "runner/runner-state.json"); + const runner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(runner.warmTransition.phase).toBe( + lossPoint === "after-activation" || lossPoint.includes("confirmation") + ? "activating" + : "prepared", + ); + const persisted = JSON.parse(await readFile(core.store.path, "utf8")); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + )?.status, + ).toBe( + lossPoint === "before-result" + ? "pending" + : lossPoint === "after-result" + ? "completed" + : undefined, + ); + // Stop only this fixture's exact owned process handles. No stored receipt + // or PID absence is used as authority to terminate an unknown owner. + await durableControlPlane + .waitForProcess(handles[0]!, 100) + .catch(() => undefined); + await within(handles[0]!.completion); + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + await within(attachment.catch(() => undefined)); + const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); + expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); + expect(joinedCore.schema).toBe(persisted.schema); + expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); + expect(joinedCore.completedWarmTransition).toEqual( + persisted.completedWarmTransition, + ); + commitSpy.mockRestore(); + const fixtureOutput = + process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; + if (fixtureOutput && recoveryFault === "none") { + const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); + await cp(options.runnerBinary, retainedArtifact, { force: false }); + expect( + createHash("sha256") + .update(await readFile(retainedArtifact)) + .digest("hex"), + ).toBe( + createHash("sha256") + .update(await readFile(options.runnerBinary)) + .digest("hex"), + ); + const retained = join( + fixtureOutput, + `${lossPoint}-${routed ? "routed" : "local"}`, + ); + await cp(stateDirectory, retained, { + recursive: true, + errorOnExist: true, + force: false, + }); + await writeFile( + join(retained, "transition-fixture-metadata.json"), + JSON.stringify( + { + schema: "paperclip.test.warm-transition-fixture.v1", + oldIdentity, + newIdentity: desired, + lossPoint, + routed, + runner: { + pid: handles[0]!.child.pid, + processGroupId: handles[0]!.processGroupId, + startedAt: runnerProcessStartedAt, + spawnObservedAt: handles[0]!.startedAt, + completion: await handles[0]!.completion, + processAbsent: dead(handles[0]!.child.pid!), + groupAbsent: dead(-handles[0]!.processGroupId!), + }, + providers: [...providerPids].map((pid) => ({ + pid, + processGroupId: pid, + startedAt: firstEvidence.providerProcessStartedAt, + processAbsent: dead(pid), + groupAbsent: dead(-pid), + })), + artifact: { + path: retainedArtifact, + version: runner.warmTransition.receipt.runnerVersion, + digest: runner.warmTransition.receipt.runnerDigest, + }, + thread, + firstThreadSnapshot, + firstEvidence, + }, + null, + 2, + ), + { mode: 0o600 }, + ); + } + routes.clear(); + const routeCallCount = routeCalls.length; + const beforeCalls = (await readFile(callsPath, "utf8")) + .trim() + .split(/\r?\n/); + expect( + beforeCalls.filter((call) => call === "process-start"), + ).toHaveLength(1); + if (recoveryFault === "legacy-parser") { + const legacyBinary = + process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; + const legacyDigest = `sha256:${createHash("sha256") + .update(await readFile(legacyBinary)) + .digest("hex")}`; + for (const [mode, bytes] of [ + ["pending", await readFile(runnerPath)], + ["clean", cleanRunnerStateBytes], + ] as const) { + const probe = await mkdtemp( + join(tmpdir(), "runnerd-legacy-schema-probe-"), + ); + legacyProbeDirectories.push(probe); + await writeFile(join(probe, "runner-state.json"), bytes, { + mode: 0o600, + }); + const handle = durableControlPlane.spawnRunner({ + connection: runner.warmTransition.receipt.connection, + stateDirectory: probe, + identity: oldIdentity, + runnerBinaryPath: legacyBinary, + runnerVersion: "0.3.0", + runnerDigest: legacyDigest, + ticket: "bootstrap_legacy_parser_probe", + maxOutboxBytes: runner.maxOutboxBytes, + p0ReserveBytes: runner.p0ReserveBytes, + maxRuntimeMs: 200, + reconnectGraceMs: 200, + }); + const result = await within(handle.completion, 5_000); + expect(result.code).not.toBe(0); + if (mode === "pending") { + expect(result.stderr).toContain( + "durable state binding does not match", + ); + expect(await readFile(join(probe, "runner-state.json"))).toEqual( + bytes, + ); + } else { + expect(result.stderr).not.toContain( + "durable state binding does not match", + ); + expect( + JSON.parse( + await readFile(join(probe, "runner-state.json"), "utf8"), + ).nextSourceSeq, + ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); + } + } + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + return; + } + recovering = true; + if (recoveryFault === "malformed-core") + await writeFile(core.store.path, "{", { mode: 0o600 }); + const beforeRecoveryBytes = await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ); + const beforeRecoveryTree = + recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); + const authorizationStages: string[] = []; + let releaseHeldAuthorization: (() => void) | undefined; + let markHeldAuthorizationEntered!: () => void; + const heldAuthorizationEntered = new Promise((resolve) => { + markHeldAuthorizationEntered = resolve; + }); + const authorizationFault = recoveryFault.startsWith("authorize_"); + const failedAuthorizationStage = recoveryFault.slice("authorize_".length); + const beforeProviderBytes = await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ); + resumed = createCapabilityRunnerdCodexTransport({ + ...options, + ...(externalArtifact ? { + runnerBinary: "/controller-missing-runner", + runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + } : {}), + ...(ordinaryFollowup + ? { + authorizeWarmTransitionRecovery: async () => { + if (!recoveryClaimCurrent) + throw new Error( + "fixture old recovery claim is no longer current", + ); + }, + onWarmTransitionRecoveryCompleted: (completion: { + transitionId: string; + }) => { + expect(completion.transitionId).toBe( + runner.warmTransition.receipt.transitionId, + ); + const completedSnapshot = cores[1]!.store.state.commands + .filter((command) => command.type === "session.snapshot") + .at(-1)!; + expect(completedSnapshot.status).toBe("completed"); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + completionSnapshotIds.push(completedSnapshot.commandId); + if ( + ordinaryFollowup === "callback_failure" && + completionSnapshotIds.length === 1 + ) { + throw new Error( + "fixture recovery completion callback failed", + ); + } + if ( + ordinaryFollowup === "callback_async_failure" && + completionSnapshotIds.length === 1 + ) { + return new Promise((_resolve, reject) => { + rejectHeldCompletion = reject; + markHeldCompletionEntered(); + }); + } + recoveryFenceActive = false; + }, + } + : {}), + ...(authorizationFault + ? { + authorizeWarmTransitionRecovery: async (stage: string) => { + authorizationStages.push(stage); + if ( + stage === "before_bootstrap" && + failedAuthorizationStage === "before_bootstrap" + ) { + await new Promise((resolveGate) => { + releaseHeldAuthorization = resolveGate; + markHeldAuthorizationEntered(); + }); + } + if (stage === failedAuthorizationStage) + throw new Error("fixture recovery authority revoked"); + }, + } + : {}), + ...(recoveryFault === "missing-capability" + ? { warmTransitionRegistrationMode: undefined } + : {}), + ...(recoveryFault !== "none" + ? { + environment: { + CODEX_API_KEY: "synthetic-refused-route-credential", + }, + } + : {}), + prpIdentity: desired, + resumeProviderSession: { + driverSessionId: thread.id, + providerSessionId: thread.sessionId, + }, + }); + if (recoveryFault !== "none") { + const recoveryRequest = within( + resumed.transport.request("thread/read", {}), + ); + void recoveryRequest.catch(() => undefined); + if ( + authorizationFault && + failedAuthorizationStage === "before_bootstrap" + ) { + await within(Promise.race([ + heldAuthorizationEntered, + recoveryRequest.then(() => { + throw new Error("warm restart authorization callback was not entered"); + }), + ])); + expect(releaseHeldAuthorization).toBeTypeOf("function"); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + try { + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not queue before bootstrap" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.resolveRuntimeRequest!({ + requestId: "must-not-resolve", + turnId: desired.turnId, + resolution: { action: "cancel" }, + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + expect(handles).toHaveLength(1); + } finally { + releaseHeldAuthorization!(); + } + } + const failedRecovery = expect(recoveryRequest).rejects; + if (authorizationFault) { + await failedRecovery.toThrow( + failedAuthorizationStage === "before_authentication" + ? "native_runner_warm_transition_recovery_pending" + : "fixture recovery authority revoked", + ); + const expectedStages = [ + "before_bootstrap", + "before_spawn", + "before_authentication", + ]; + expect([...new Set(authorizationStages)]).toEqual( + expectedStages.slice( + 0, + expectedStages.indexOf(failedAuthorizationStage) + 1, + ), + ); + expect(handles).toHaveLength( + failedAuthorizationStage === "before_authentication" ? 2 : 1, + ); + expect( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + ), + ).toEqual(beforeProviderBytes); + const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); + expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + if (failedAuthorizationStage !== "before_authentication") + expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); + if (failedAuthorizationStage === "before_bootstrap") + expect(await readFile(core.store.path)).toEqual( + beforeRecoveryBytes[0], + ); + await within(resumed.transport.close()).catch(() => undefined); + expect([...routes.keys()]).toEqual([]); + return; + } + if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); + else + await failedRecovery.toThrow( + recoveryFault === "missing-capability" + ? "requires_exact_owned_endpoint" + : "registered_endpoint_mismatch", + ); + expect(handles).toHaveLength(1); + expect( + await Promise.all( + [core.store.path, runnerPath].map((path) => readFile(path)), + ), + ).toEqual(beforeRecoveryBytes); + expect(await fingerprintTree(stateDirectory)).toEqual( + beforeRecoveryTree, + ); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual(beforeCalls); + expect([...routes.keys()]).toEqual([]); + return; + } + if ( + snapshotFault || + ordinaryFollowup === "callback_failure" || + ordinaryFollowup === "callback_async_failure" + ) { + const firstRead = within(resumed.transport.request("thread/read", {})); + void firstRead.catch(() => undefined); + if (ordinaryFollowup === "callback_async_failure") { + await within(Promise.race([ + heldCompletionEntered, + firstRead.then(() => { + throw new Error("warm restart completion callback was not entered"); + }), + ])); + expect(rejectHeldCompletion).toBeTypeOf("function"); + try { + expect(recoveryFenceActive).toBe(true); + const queuedBefore = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must not pass held completion" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map( + (command) => command.commandId, + ), + ).toEqual(queuedBefore); + } finally { + rejectHeldCompletion!( + new Error("fixture recovery completion callback failed"), + ); + } + } + await expect(firstRead).rejects.toThrow( + snapshotFault + ? ordinaryFollowup === "rejected_snapshot" + ? "snapshot observation rejected" + : "completion_unproven" + : "recovery completion callback failed", + ); + expect(recoveryFenceActive).toBe(true); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); + const beforeDeniedWork = cores[1]!.store.state.commands.map( + (command) => command.commandId, + ); + const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); + await expect( + resumed.transport.request("turn/start", { + input: [{ text: "must remain fenced" }], + }), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.request("thread/resume", {}), + ).rejects.toThrow("warm_transition_completion_pending"); + await expect( + resumed.transport.attachRun!({ + runId: "must-not-attach", + turnId: "must-not-attach", + itemId: "must-not-attach", + }), + ).rejects.toThrow("warm_transition_completion_pending"); + expect( + cores[1]!.store.state.commands.map((command) => command.commandId), + ).toEqual(beforeDeniedWork); + expect(await readFile(callsPath, "utf8")).toBe( + beforeDeniedProviderCalls, + ); + snapshotObserverSpy?.mockRestore(); + const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + expect(recoveryFenceActive).toBe(false); + expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); + expect(new Set(completionSnapshotIds).size).toBe( + completionSnapshotIds.length, + ); + expect(await readFile(callsPath, "utf8")).toBe( + providerCallsBeforeRetry, + ); + return; + } + const read = await within( + resumed.transport.request("thread/read", {}), + 10_000, + ); + if ((resumed.evidence().codexPid ?? 0) > 0) + providerPids.add(resumed.evidence().codexPid!); + expect(read.thread).toMatchObject(thread); + expect(cores).toHaveLength(2); + expect(cores[1]!.store.state.identity).toEqual(desired); + expect(cores[1]!.store.state.warmTransition).toBeUndefined(); + expect( + cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, + ).toBe(runner.warmTransition.receipt.transitionId); + if (routed) { + expect(routeCalls.slice(routeCallCount)).toEqual([ + oldIdentity.runId, + desired.runId, + ]); + expect([...routes.keys()]).toEqual([ + `/api/runner/v1/connect/${desired.runId}`, + ]); + } + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(2); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + if (ordinaryFollowup) { + recoveryClaimCurrent = false; + const resumedCore = cores[1]!; + resumedCore.disconnectActiveRunner(); + await vi.waitFor( + () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), + { timeout: 2_000 }, + ); + expect(recoveryFenceActive).toBe(false); + expect( + (await within(resumed.transport.request("thread/read", {}))).thread, + ).toMatchObject(thread); + const thirdRunId = randomUUID(); + await within( + resumed.transport.attachRun!({ + runId: thirdRunId, + turnId: `turn-${thirdRunId}`, + itemId: `item-${thirdRunId}`, + }), + ); + expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); + expect( + (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), + ).toEqual([...calls, "thread/goal/get"]); + } + } finally { + snapshotObserverSpy?.mockRestore(); + commitSpy?.mockRestore(); + commandObserverSpy?.mockRestore(); + try { + await within(resumed?.transport.close() ?? Promise.resolve()).catch( + () => undefined, + ); + await within(first.transport.detachControllerForRestart!()).catch( + () => undefined, + ); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within(handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + for (const pid of providerPids) { + await stopOwnedProvider(pid); + } + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + if (routed) + await new Promise((resolveClose) => + routeServer.close(() => resolveClose()), + ); + coreSpy.mockRestore(); + launchSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + if (cleanupProven) + for (const directory of legacyProbeDirectories) + await rm(directory, { recursive: true, force: true }); + } + } + }, + 30_000, +); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts new file mode 100644 index 00000000000..dba8ec470b3 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts @@ -0,0 +1,769 @@ +import { mkdtemp, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { ControlPlanePort } from "../contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; +import type { NativeSession, NativeSessionBackend } from "../contracts/native-session-backend.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; +import { executeNativeSession } from "../native-session-runtime.js"; +import { createCodexTaskEnvelope } from "../contracts/codex.js"; +import { CodexAppServerDriver } from "../drivers/codex/codex-app-server-driver.js"; +import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { defaultCapabilityRunnerdBinary, expectTurnStarted, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; + +it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-stopped-resume-active-"), + ); + const identity = { + runnerInstanceId: "runner-stopped-active", + environmentLeaseId: "lease-stopped-active", + runId: "run-stopped-first", + normalizedSessionId: "session-stopped-active", + turnId: "turn-stopped-first", + itemId: "item-stopped-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--linger-after-turn-start", + "--resume-unowned-turn-when-marked", + "--emit-tool-call-on-resume", + ), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + }; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: identity, + }); + let second: + ReturnType | undefined; + const semanticHandler = vi.fn(async () => ({ + success: true, + contentItems: [], + })); + first.transport.setServerRequestHandler(semanticHandler); + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the current task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + }); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + await expectTurnStarted( + first.transport.notifications()[Symbol.asyncIterator](), + ); + await first.transport.close(); + const providerPath = join( + stateDirectory, + "runner", + "codex-provider-state.json", + ); + expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); + const secondIdentity = { + ...identity, + runId: "run-stopped-second", + turnId: "turn-stopped-second", + itemId: "item-stopped-second", + }; + second = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: secondIdentity, + }); + second.transport.setServerRequestHandler(semanticHandler); + await expect(second.transport.request("thread/read", {})).rejects.toThrow( + "prepared provider checkpoint resumed unexpected active work", + ); + // Exercise the normal durable transfer before inspecting the rejection: + // provider pendingEvents is an acknowledged queue, not an event journal. + await vi.waitFor( + async () => { + const provider = JSON.parse(await readFile(providerPath, "utf8")); + expect(provider.pendingEvents).toEqual([]); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ) as { + committedEvents: Array<{ + eventType: string; + envelope: { payload: { payload: { code?: string } } }; + }>; + }; + const rejections = control.committedEvents.filter( + (event) => + event.eventType === "harness.diagnostic" && + event.envelope.payload.payload.code === + "prepared_provider_checkpoint_has_active_work", + ); + expect(rejections).toEqual([ + expect.objectContaining({ + logicalEffectCount: 1, + envelope: expect.objectContaining({ + ...secondIdentity, + payload: expect.objectContaining({ + payload: expect.objectContaining({ + code: "prepared_provider_checkpoint_has_active_work", + paperclipAccepted: false, + providerReportedActive: true, + }), + }), + }), + }), + ]); + }, + { timeout: 5_000 }, + ); + const closed = JSON.parse(await readFile(providerPath, "utf8")); + expect(closed).toMatchObject({ + lifecycle: "closed", + threadId: (opened.thread as Record).id, + activeProviderTurnId: null, + }); + expect(semanticHandler).not.toHaveBeenCalled(); + expect(closed.pendingEvents).toEqual([]); + } finally { + await Promise.allSettled([ + first.transport.close(), + second?.transport.close(), + ]); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { + const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", + }; + + const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [ + { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, + ], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; + + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; + + const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.sessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], + }; + + function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, + ): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; + } + const stateDirectory = await mkdtemp( + join(tmpdir(), "native-close-quarantine-"), + ); + const readRunnerState = vi.fn(async () => ({ + schema: "paperclip.runner.durable.state.v1", + runnerInstanceId: "runner-close-quarantine", + environmentLeaseId: "lease-close-quarantine", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-close-quarantine", + itemId: "item-close-quarantine", + lifecycle: "ready", + })); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", + ), + codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: await readRunnerState(), + readRunnerState, + // A checkpoint owner requires durable suspension proof. A local transport + // without a checkpoint can simply terminate its process on close. + controlPlaneRegistration: async (authority) => { + await authority.start(); + return { + connectUrl: authority.connectUrl, + checkpoint: async () => {}, + release: async () => {}, + }; + }, + }); + try { + await bundle.transport.request("thread/start", { + cwd: stateDirectory, + dynamicTools: [], + }); + const failedClose = bundle.transport.close(); + const failure = await failedClose.catch((error: unknown) => error); + expect(failure).toMatchObject({ + code: "native_session_close_unrecoverable", + }); + expect(bundle.transport.close()).toBe(failedClose); + + vi.useFakeTimers(); + const close = vi.fn(({ reason }: { reason: string }) => + bundle.transport.close(reason), + ); + const capabilities = { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return capabilities; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "real-memoized-close-quarantine", + version: "1", + capabilities, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + requireSessionCloseBeforeReturn: true, + }); + await expect(execute()).rejects.toBe(failure); + const readsAfterClose = readRunnerState.mock.calls.length; + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + recovery: "operator_required", + }); + await vi.advanceTimersByTimeAsync(600_000); + await expect(execute()).rejects.toMatchObject({ + code: "native_session_cleanup_quarantined", + }); + expect(openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); + } finally { + vi.useRealTimers(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 10_000); + +it.each(["not_suspended", "wrong_identity"] as const)( + "does not report local runner close healthy with %s durable evidence", + async (mode) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-local-close-unproven-"), + ); + const identity = { + runnerInstanceId: "runner-close-unproven", + environmentLeaseId: "lease-close-unproven", + runId: "run-close-unproven", + normalizedSessionId: "session-close-unproven", + turnId: "turn-close-unproven", + itemId: "item-close-unproven", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + closeGraceMs: 400, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + readRunnerState: async () => ({ + schema: "paperclip.runner.durable.state.v1", + ...identity, + ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), + lifecycle: mode === "not_suspended" ? "ready" : "suspended", + }), + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await expect(bundle.transport.close()).rejects.toThrow( + "runner did not durably suspend before checkpoint", + ); + const control = JSON.parse( + await readFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + "utf8", + ), + ); + expect(control.identity).toEqual(identity); + expect(await readdir(stateDirectory)).toContain("runner"); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 10_000, +); + +it.each([ + { + label: "interrupted", + args: ["--hold-turn"], + terminal: "turn.interrupted", + disposition: "needs_review", + }, + { + label: "failed", + args: ["--fail-turn-immediately"], + terminal: "turn.failed", + disposition: "needs_review", + }, + { + label: "completed", + args: [], + terminal: "turn.completed", + disposition: "done", + }, +])("retains a $label runner's result before closing its provider turn", async ({ + label, args, terminal, disposition, +}) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, ...args), + stateDirectory, + }); + const driver = new CodexAppServerDriver({ + taskEnvelope: createCodexTaskEnvelope({ + objective: "Stop and retain unfinished work.", + }), + environment: { + PATH: process.env.PATH, + HOME: join(tmpdir(), "runnerd-stop-host-home"), + PAPERCLIP_WORKSPACE_CWD: stateDirectory, + }, + approvalPolicy: "never", + transportFactory: () => bundle.transport, + }); + const session = await driver.openSession({ + runId: "run-stop-result", + normalizedSessionId: "session-stop-result", + workingDirectory: stateDirectory, + }); + try { + const turn = await session.startTurn({ + message: { role: "user", text: "Keep working until interrupted." }, + }); + if (label === "interrupted") { + await session.interrupt({ + turnId: turn.turnId, + reason: "Stopped by the user", + }); + } + const events: PrpEvent[] = []; + for await (const event of session.events()) { + events.push(event); + if (event.eventType === "session.failed" || event.eventType === terminal) { + break; + } + } + // Let already committed durable suffix events reach the facade as well. + await bundle.transport.request("thread/read", {}); + const snapshot = await session.snapshot(); + expect(events.some((event) => event.eventType === "session.failed")).toBe(false); + expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); + expect(events.at(-1)?.eventType).toBe(terminal); + expect(snapshot).toMatchObject({ + activeTurnId: null, + semanticResult: { + turnId: turn.turnId, + result: { reportedWorkDisposition: disposition }, + }, + }); + } finally { + await session.close().catch(() => undefined); + await rm(stateDirectory, { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 25, + }); + } +}, 30_000); + +it.each([0, 193 * 1024 * 1024])( + "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", + async (extraJournalBytes) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-exact-authority-resume-"), + ); + const identity = { + runnerInstanceId: "runner-exact-resume", + environmentLeaseId: "lease-exact-resume", + runId: "run-exact-resume", + normalizedSessionId: "session-exact-resume", + turnId: "turn-exact-resume", + itemId: "item-exact-resume", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), + stateDirectory, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + prpIdentity: identity, + }; + const first = createCapabilityRunnerdCodexTransport(options); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let providerThread: { id: string; sessionId: string } | null = null; + try { + const opened = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const thread = opened.thread as Record; + providerThread = { + id: String(thread.id), + sessionId: String(thread.sessionId), + }; + } finally { + await first.transport.close(); + } + if (providerThread === null) { + throw new Error( + "exact-authority fixture did not return a provider thread", + ); + } + + const statePath = join( + stateDirectory, + "control-plane", + "control-plane-state.json", + ); + if (extraJournalBytes > 0) { + const padding = Buffer.alloc(1024 * 1024, 0x20); + const stateHandle = await open(statePath, "a"); + try { + for (let remaining = extraJournalBytes; remaining > 0;) { + const bytesToWrite = Math.min(remaining, padding.length); + await stateHandle.write(padding, 0, bytesToWrite); + remaining -= bytesToWrite; + } + } finally { + await stateHandle.close(); + } + expect((await stat(statePath)).size).toBeGreaterThan( + 64 * 1024 * 1024, + ); + } + const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ type: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect( + beforeResume.commands.some((command) => command.type === "run.attach"), + ).toBe(false); + const priorResumeEvents = beforeResume.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ).length; + const priorSnapshots = beforeResume.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + const resumed = createCapabilityRunnerdCodexTransport({ + ...options, + resumeProviderSession: { + driverSessionId: providerThread.id, + providerSessionId: providerThread.sessionId, + }, + }); + resumed.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await resumed.transport.request("thread/read", {}); + expect(read.thread).toMatchObject(providerThread); + const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { + commands: Array<{ commandId: string; type: string; status: string }>; + committedEvents: Array<{ eventType: string }>; + }; + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + commandId: expect.stringMatching(/^command_resume_probe_/), + type: "runner.drain", + status: "completed", + }), + ); + expect(afterResume.commands).toContainEqual( + expect.objectContaining({ + type: "session.snapshot", + status: "completed", + }), + ); + expect( + afterResume.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshots + 2); + // The authenticated snapshot above proves the live provider identity. + // Control-first dispatch may deliver that command before the independent + // session event is ingested. Still require exactly one durable event; + // don't mistake an immediate file read for an event-delivery barrier. + await vi.waitFor( + async () => { + const delivered = JSON.parse(await readFile(statePath, "utf8")) as { + committedEvents: Array<{ eventType: string }>; + }; + expect( + delivered.committedEvents.filter( + (event) => event.eventType === "session.resumed", + ), + ).toHaveLength(priorResumeEvents + 1); + }, + { timeout: 3_000, interval: 25 }, + ); + } finally { + await resumed.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } + }, + 30_000, +); + +it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-close-grace-too-small-"), + ); + const identity = { + runnerInstanceId: "runner-close-grace-too-small", + environmentLeaseId: "lease-close-grace-too-small", + runId: "run-close-grace-too-small", + normalizedSessionId: "session-close-grace-too-small", + turnId: "turn-close-grace-too-small", + itemId: "item-close-grace-too-small", + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + // No real durable command round trip can complete this fast. A wider + // budget for the provider-drain proof must not turn this barrier into + // one that always passes; it still needs the actual proof to arrive. + closeGraceMs: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: identity, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + const closed = bundle.transport.close(); + await expect(closed).rejects.toMatchObject({ + code: "native_session_close_unrecoverable", + semanticToolsSettled: true, + }); + // Retirement preserves the original rejected promise and its proof. + expect(bundle.transport.close()).toBe(closed); + } finally { + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("persists an active provider as settled before bounded suspension", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-active-suspension-"), + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), + stateDirectory, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + await bundle.transport.request("initialize", {}); + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }); + await bundle.transport.request("turn/start", { + input: [{ type: "text", text: "Wait for another instruction." }], + }); + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + await expectTurnStarted(notifications); + await bundle.transport.close(); + + const providerState = JSON.parse( + await readFile( + join(stateDirectory, "runner", "codex-provider-state.json"), + "utf8", + ), + ) as Record; + expect(providerState).toMatchObject({ + lifecycle: "prepared", + activeProviderTurnId: null, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts new file mode 100644 index 00000000000..7739d4e2851 --- /dev/null +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts @@ -0,0 +1,1649 @@ +import { mkdir, mkdtemp, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { createHash, randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; +import type { PrpEvent } from "../protocol/replay-contract.js"; +import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; +import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; +import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; +import { + defaultCapabilityRunnerdBinary, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; + +it.each([ + ["held-ack", "normal"], + ["lost-ack", "normal"], + ["rejected-attach", "normal"], + ["lost-ack", "after-activation"], +] as const)( + "preserves old warm-attach authority and event ownership across %s (%s observer)", + async (mode, observer) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const effects = new Map(); + const handles: ReturnType[] = []; + let armed = false; + let heldEvent: PrpEvent | null = null; + let releaseCommit!: () => void; + let enteredCommit!: () => void; + const commitGate = new Promise((resolveCommit) => { + releaseCommit = resolveCommit; + }); + const commitEntered = new Promise((resolveEntered) => { + enteredCommit = resolveEntered; + }); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore({ + ...options, + onCommittedEvent: async (event) => { + await options.onCommittedEvent?.(event); + const prior = effects.get(event.sourceEventId); + if (prior) { + expect(event).toEqual(prior.event); + prior.deliveries += 1; + } else { + effects.set(event.sourceEventId, { + event: structuredClone(event), + deliveries: 1, + }); + } + if ( + armed && + mode !== "rejected-attach" && + heldEvent === null && + event.eventType === "run.attached" + ) { + heldEvent = structuredClone(event); + enteredCommit(); + await commitGate; + if (mode === "lost-ack") { + // The external durable effect exists, but this connection + // disappears before its local cursor/ACK can be published. + throw new Error( + "fixture lost the old authority ACK after commit", + ); + } + } + }, + }); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolveWait, rejectWait) => { + timer = setTimeout( + () => rejectWait(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + }); + const readRunner = async () => + JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ) as { + runId: string; + ackedSourceSeq: number; + processedCommands: Record; + outbox: { envelope: { payload: PrpEvent } }[]; + }; + let providerPid: number | null = null; + let primaryError: unknown; + let cleanupProven = false; + try { + const opened = await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ) as { thread: { id: string } }; + const core = cores[0]!; + expect(cores).toHaveLength(1); + const oldIdentity = structuredClone(core.store.state.identity); + const runnerPid = bundle.evidence().runnerPid; + providerPid = bundle.evidence().codexPid; + const rotations: (typeof core.store.state)[] = []; + const preparedStates: (typeof core.store.state)[] = []; + const commit = core.store.commit.bind(core.store); + vi.spyOn(core.store, "commit").mockImplementation((candidate) => { + commit(candidate); + // Capture the durable old epoch at publication. The authenticated + // successor may activate before attachRun observes the completed command. + if ( + candidate.identity.runId === oldIdentity.runId && + candidate.warmTransition?.phase === "prepared" + ) { + preparedStates.push(structuredClone(core.store.state)); + } + }); + if (observer === "after-activation") { + const getCommand = core.getCommand.bind(core); + vi.spyOn(core, "getCommand").mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + command.status === "completed" && + core.store.state.completedWarmTransition === undefined + ) { + // Deterministically exercise an observer that misses the old epoch. + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + const rotate = core.rotateRunIdentity.bind(core); + vi.spyOn(core, "rotateRunIdentity").mockImplementation( + (identity, template) => { + rotations.push(structuredClone(core.store.state)); + return rotate(identity, template); + }, + ); + if (mode === "rejected-attach") { + const queue = core.queueCommand.bind(core); + vi.spyOn(core, "queueCommand").mockImplementation( + (type, payload = {}, id, immediate) => + queue( + type, + type === "run.attach" + ? { + ...payload, + provider: { + ...(payload.provider as Record), + model: "foreign-profile", + }, + } + : payload, + id, + immediate, + ), + ); + } + armed = true; + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-ack-next", + turnId: "turn-warm-ack-next", + itemId: "item-warm-ack-next", + }); + void attachment.catch(() => undefined); + if (mode === "rejected-attach") { + await expect(within("rejected attach", attachment)).rejects.toThrow( + "run.attach cannot change the durable Codex provider profile", + ); + expect(rotations).toHaveLength(0); + expect(core.store.state.identity).toEqual(oldIdentity); + expect((await readRunner()).runId).toBe(oldIdentity.runId); + const read = await within( + "read under unchanged authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("failed"); + } else { + await within("old authority commit barrier", commitEntered); + const retained = await readRunner(); + expect(retained.runId).toBe(oldIdentity.runId); + expect( + Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") + ?.status, + ).toBe("completed"); + expect( + retained.outbox.some( + (entry) => + entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(true); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + expect(rotations).toHaveLength(0); + if (mode === "lost-ack") core.disconnectActiveRunner(); + releaseCommit(); + await within("warm attach after old ACK", attachment, 10_000); + expect(rotations).toHaveLength(1); + expect(preparedStates.length).toBeGreaterThan(0); + const retired = preparedStates[0]!; + if (observer === "after-activation") { + expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); + expect( + rotations[0]!.committedEvents.some( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + ), + ).toBe(false); + } + const attachedEvent = retired.committedEvents.find( + (entry) => entry.sourceEventId === heldEvent!.sourceEventId, + )!; + expect(attachedEvent.logicalEffectCount).toBe(1); + expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( + attachedEvent.sourceSeq, + ); + expect( + retired.committedEvents.slice(-4).map((entry) => entry.eventType), + ).toEqual([ + "session.resumed", + "session.capabilities.updated", + "session.goal.snapshot", + "run.attached", + ]); + expect( + retired.committedEvents.every( + (entry) => entry.envelope.runId === oldIdentity.runId, + ), + ).toBe(true); + if (mode === "lost-ack") { + expect(retired.connectionCount).toBeGreaterThanOrEqual(2); + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); + } else { + expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); + } + await vi.waitFor(async () => + expect((await readRunner()).runId).toBe("run-warm-ack-next"), + ); + const read = await within( + "read under new authority", + bundle.transport.request("thread/read", {}), + ); + expect((read.thread as { id: string }).id).toBe(opened.thread.id); + } + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + } catch (error) { + primaryError = error; + throw error; + } finally { + releaseCommit(); + try { + await within( + "warm fixture close", + bundle.transport.close(), + 10_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact warm fixture runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } catch (error) { + if (primaryError === undefined) throw error; + console.error( + "Warm ACK fixture cleanup unproven; retaining its private state directory.", + ); + } finally { + try { + for (const core of cores) await core.stop().catch(() => undefined); + } finally { + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + } + }, + 30_000, +); + +it.each([false, true])( + "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", + async (holdObserver) => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-attach-result-loss-"), + ); + const callsPath = join(stateDirectory, "calls.log"); + const cores: DurablePrpControlPlane[] = []; + const handles: ReturnType[] = []; + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + const coreSpy = vi + .spyOn(durableControlPlane, "DurablePrpControlPlane") + .mockImplementation(function ( + options: ConstructorParameters[0], + ) { + const core = new OriginalCore(options); + cores.push(core); + return core; + } as unknown as typeof OriginalCore); + const launch = durableControlPlane.spawnRunner; + const launchSpy = vi + .spyOn(durableControlPlane, "spawnRunner") + .mockImplementation((options) => { + const handle = launch(options); + handles.push(handle); + return handle; + }); + const within = async ( + label: string, + promise: Promise, + timeout = 5_000, + ) => { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timeout`)), + timeout, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + }; + const dead = (pid: number) => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--call-log", + callsPath, + "--record-process-start", + ), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 2_000, + }); + let providerPid: number | null = null; + let cleanupProven = false; + let saveSpy: ReturnType | undefined; + let observerSpy: { mockRestore(): void } | undefined; + let attachment: Promise | undefined; + try { + await within( + "initial thread", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [], + }), + ); + expect(cores).toHaveLength(1); + const core = cores[0]!; + const oldIdentity = structuredClone(core.store.state.identity); + if (holdObserver) { + const getCommand = core.getCommand.bind(core); + observerSpy = vi + .spyOn(core, "getCommand") + .mockImplementation((commandId) => { + const command = getCommand(commandId); + if ( + command?.type === "run.attach" && + core.store.state.completedWarmTransition?.command.commandId !== + commandId + ) { + return { ...command, status: "pending", result: null }; + } + return command; + }); + } + providerPid = bundle.evidence().codexPid; + const store = core.store as typeof core.store & { + commit(candidate: typeof core.store.state): void; + }; + const commit = store.commit.bind(store); + let lostResult = false; + let observeLoss!: () => void; + const loss = new Promise((resolveLoss) => { + observeLoss = resolveLoss; + }); + saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { + const attach = candidate.commands.find( + (entry) => entry.type === "run.attach", + ); + if (!lostResult && attach?.status === "completed") { + lostResult = true; + // The authenticated result reached the receiver, but the durable write + // did not. The clone has not been exposed in memory or on disk. + const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); + expect(persisted.identity).toEqual(oldIdentity); + expect( + persisted.commands.find( + (entry: { type: string }) => entry.type === "run.attach", + ).status, + ).toBe("pending"); + core.disconnectActiveRunner(); + observeLoss(); + throw new Error( + "fixture lost attach result before durable controller commit", + ); + } + commit(candidate); + }); + attachment = bundle.transport.attachRun!({ + runId: "run-result-loss-next", + turnId: "turn-result-loss-next", + itemId: "item-result-loss-next", + }); + void attachment.catch(() => undefined); + await within("lost attach result", loss); + expect(lostResult).toBe(true); + expect(core.store.state.identity).toEqual(oldIdentity); + expect( + core.store.state.commands.find((entry) => entry.type === "run.attach") + ?.status, + ).toBe("pending"); + const outcome = await within( + "exact result replay after reconnect", + attachment.then( + () => ({ status: "completed" as const }), + (error: unknown) => ({ + status: "failed" as const, + message: String(error), + }), + ), + 10_000, + ); + const runner = JSON.parse( + await readFile( + join(stateDirectory, "runner/runner-state.json"), + "utf8", + ), + ); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter((call) => call === "process-start")).toHaveLength(1); + expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); + expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); + expect( + { + outcome, + runnerRunId: runner.runId, + controllerRunId: core.store.state.identity.runId, + }, + "lost attach result must replay without leaving the two durable authorities split", + ).toEqual({ + outcome: { status: "completed" }, + runnerRunId: "run-result-loss-next", + controllerRunId: "run-result-loss-next", + }); + if (holdObserver) + expect( + core.store.state.completedWarmTransition?.receipt.newIdentity.runId, + ).toBe("run-result-loss-next"); + } finally { + saveSpy?.mockRestore(); + observerSpy?.mockRestore(); + try { + await within( + "result-loss fixture close", + bundle.transport.close(), + 5_000, + ).catch(() => undefined); + for (const handle of handles) { + await durableControlPlane + .waitForProcess(handle, 250) + .catch(() => undefined); + await within("exact result-loss runner exit", handle.completion); + if (handle.processGroupId && !dead(-handle.processGroupId)) + process.kill(-handle.processGroupId, "SIGKILL"); + await vi.waitFor(() => { + expect(handle.child.pid && dead(handle.child.pid)).toBe(true); + expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( + true, + ); + }); + } + if (providerPid && !dead(-providerPid)) + process.kill(-providerPid, "SIGKILL"); + if (providerPid) + await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); + cleanupProven = true; + } finally { + for (const core of cores) await core.stop().catch(() => undefined); + launchSpy.mockRestore(); + coreSpy.mockRestore(); + if (cleanupProven) + await rm(stateDirectory, { recursive: true, force: true }); + } + } + }, + 30_000, +); + +it("rotates PRP authority in place for a warm cross-run attachment", async () => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + const within = async (label: string, promise: Promise) => + await Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), + ), + ]); + try { + await within("initialize", bundle.transport.request("initialize", {})); + await within( + "thread start", + bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ], + completionContract: { + revision: "sha256:warm-three-turn-contract", + criterionIds: ["objective"], + }, + }), + ); + const runnerPid = bundle.evidence().runnerPid; + const providerPid = bundle.evidence().codexPid; + const notifications = bundle.transport + .notifications() + [Symbol.asyncIterator](); + const waitForCompletion = async (label: string) => { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + const next = await Promise.race([ + notifications.next(), + new Promise((_, reject) => + setTimeout( + () => reject(new Error(`${label} notification timeout`)), + 1_000, + ), + ), + ]); + if (next.value?.method === "turn/completed") return; + } + throw new Error(`${label} completion timeout`); + }; + await within( + "first turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "first run" }], + }), + ); + await waitForCompletion("first run"); + + await within( + "warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-second", + turnId: "turn-warm-second", + itemId: "item-warm-second", + }), + ); + await within( + "second turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "second run" }], + }), + ); + await waitForCompletion("second run"); + + await within( + "second warm attach", + bundle.transport.attachRun!({ + runId: "run-warm-third", + turnId: "turn-warm-third", + itemId: "item-warm-third", + }), + ); + await within( + "third turn start", + bundle.transport.request("turn/start", { + input: [{ type: "text", text: "third run" }], + }), + ); + await waitForCompletion("third run"); + + expect(bundle.evidence()).toMatchObject({ + runnerPid, + codexPid: providerPid, + runnerExited: false, + }); + } finally { + await bundle.transport.close(); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("recovers an active second warm business run after deleting its controller context assets", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); + const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); + const assetRoot = join(root, "retired-controller-assets"); + const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); + await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); + await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null, generation = 0; + let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + const registered = new WeakSet(); + server.on("upgrade", (request, socket, head) => authority + ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); + const registration = async (core: DurablePrpControlPlane) => { + if (!registered.has(core)) { + core.setCheckpointPersistence(async value => { checkpoint = value; }); + registered.add(core); + } + authority = core; const current = ++generation; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; + }; + const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", + runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; + const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; + const first = createCapabilityRunnerdCodexTransport(options); + const within = async (label: string, promise: Promise) => { + let timer: ReturnType | undefined; + try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } + finally { clearTimeout(timer); } + }; + const waitComplete = async (transport: typeof first.transport) => { + for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; + }; + let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; + let recovered: ReturnType | undefined; + const result = { ok: true, task: { id: "task-1" } }; + first.transport.setServerRequestHandler(async () => ++calls === 1 + ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } + : new Promise>(() => undefined)); + try { + const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", + inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); + runnerPid = first.evidence().runnerPid; + providerPid = first.evidence().codexPid; + expect(providerPid).toEqual(expect.any(Number)); + const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); + const startedAt = first.transport.processInfo().startedAt; + const firstCompleted = waitComplete(first.transport); + await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); + await within("healthy first turn completion", firstCompleted); + const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + await within("warm second business attachment", first.transport.attachRun!(attachment)); + const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); + await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); + const activeCore = authority!; + const savedIdentity = { ...activeCore.store.state.identity }; + expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); + await activeCore.flushCheckpoint(); + await within("controller detach", first.detachControllerForRestart()); + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + await rm(assetRoot, { recursive: true, force: true }); + expect(existsSync(assetRoot)).toBe(false); + checkpoint!.restoreSessionDirectory(nextRoot); + const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); + const openedThread = opened.thread as Record; + const activeTurn = active.turn as Record; + expect(openedThread.id).toEqual(expect.any(String)); + expect(activeTurn.id).toEqual(expect.any(String)); + const shared = { ...options, stateDirectory: nextRoot, + resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, + resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), + runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), + prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, + runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, + isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, + signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; + // The old server's synthetic pair must continue failing the strict Runner guard. + const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, + turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); + await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); + await wrong.detachControllerForRestart(); + // Leave the pending tool outcome unknown. Reattachment must authenticate + // and preserve it without fabricating a receipt or redispatching its handler. + recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); + await within("replacement thread/read", recovered.transport.request("thread/read", {})); + expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); + expect(authority!.activeRunnerConnectionCount()).toBe(1); + expect(() => process.kill(providerPid!, 0)).not.toThrow(); + expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); + expect(calls).toBe(2); + expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); + expect(recovered.transport.processInfo().startedAt).toBe(startedAt); + expect(authority!.store.state.identity).toEqual(savedIdentity); + expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); + expect(noSpawn).not.toHaveBeenCalled(); + } catch (error) { + const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); + const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); + throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ + transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), + identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), + events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), + runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, + outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), + })}`); + } finally { + await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); + await within("first cleanup", first.transport.close()).catch(() => undefined); + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await (authority as DurablePrpControlPlane | null)?.stop(); + if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } + server.closeAllConnections(); + if (server.listening) await new Promise(done => server.close(() => done())); + for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); + +it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-reattach-before-probe-"), + ); + const server = createServer(); + const authorities = new Map(); + let blockFirstAuthorityReconnect = false; + let resolveRejectedReconnect!: () => void; + const rejectedReconnect = new Promise((resolvePromise) => { + resolveRejectedReconnect = resolvePromise; + }); + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + if (route === "/runner-1" && blockFirstAuthorityReconnect) { + resolveRejectedReconnect(); + socket.destroy(); + return; + } + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm reconnect test listener"); + } + let registrationCount = 0; + const diagnostics: string[] = []; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, + onDiagnostic: (message) => diagnostics.push(message), + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + release: () => { + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + const firstAuthority = authorities.get("/runner-1"); + if (!firstAuthority) throw new Error("Missing first warm authority"); + const priorSnapshotCount = firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ).length; + + blockFirstAuthorityReconnect = true; + firstAuthority.disconnectActiveRunner(); + const attachment = bundle.transport.attachRun!({ + runId: "run-warm-after-reconnect", + turnId: "turn-warm-after-reconnect", + itemId: "item-warm-after-reconnect", + }); + await Promise.race([ + rejectedReconnect, + new Promise((_, reject) => + setTimeout( + () => reject(new Error("runner did not attempt to reconnect")), + 5_000, + ), + ), + ]); + + // No command may be queued while its sole authenticated consumer is + // absent. The generic 30-second command timeout used to turn this state + // into same-run recovery and replace the healthy warm runner process. + expect( + firstAuthority.store.state.commands.filter( + (command) => command.type === "session.snapshot", + ), + ).toHaveLength(priorSnapshotCount); + + blockFirstAuthorityReconnect = false; + await Promise.race([ + attachment, + new Promise((_, reject) => + setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), + ), + ]); + expect(bundle.evidence()).toMatchObject({ + runnerPid, + runnerExited: false, + }); + expect(diagnostics).toContain( + "warm runner connection interrupted; waiting for re-authentication before authority rotation", + ); + expect(diagnostics).toContain( + "warm runner re-authenticated before authority rotation", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it("releases both PRP authorities when warm rotation activation fails", async () => { + const stateDirectory = await mkdtemp( + join(tmpdir(), "runnerd-warm-attach-activation-failure-"), + ); + const server = createServer(); + const authorities = new Map(); + const released: string[] = []; + server.on("upgrade", (request, socket, head) => { + const route = request.url ?? ""; + const authority = authorities.get(route); + if (!authority) { + socket.destroy(); + return; + } + authority.handleUpgrade(request, socket, route, head); + }); + await new Promise((resolveListen) => + server.listen(0, "127.0.0.1", resolveListen), + ); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Expected warm activation failure test listener"); + } + let registrationCount = 0; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory), + stateDirectory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + controlPlaneRegistration: async (authority) => { + registrationCount += 1; + const route = `/runner-${registrationCount}`; + authorities.set(route, authority); + return { + connectUrl: `ws://127.0.0.1:${address.port}${route}`, + ...(registrationCount === 1 + ? {} + : { + activate: () => { + throw new Error("rotation activation failed"); + }, + }), + release: () => { + released.push(route); + if (authorities.get(route) === authority) authorities.delete(route); + }, + }; + }, + }); + bundle.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let runnerPid: number | null = null; + try { + await bundle.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: codexSemanticToolSpecs(), + }); + runnerPid = bundle.evidence().runnerPid; + + await expect( + bundle.transport.attachRun!({ + runId: "run-warm-activation-failure", + turnId: "turn-warm-activation-failure", + itemId: "item-warm-activation-failure", + }), + ).rejects.toThrow("rotation activation failed"); + expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); + expect(authorities.size).toBe(0); + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "rotation activation failed", + ); + } finally { + await bundle.transport.close().catch(() => undefined); + if (runnerPid) { + try { + process.kill(-runnerPid, "SIGKILL"); + } catch { + // A successful durable close already stopped the runner process group. + } + } + server.closeAllConnections(); + if (server.listening) { + await new Promise((resolveClose) => + server.close(() => resolveClose()), + ); + } + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); + +it.each([ + { + binding: "runner instance", + priorRunnerInstanceId: "runner-other", + priorEnvironmentLeaseId: "lease-current", + }, + { + binding: "environment lease", + priorRunnerInstanceId: "runner-current", + priorEnvironmentLeaseId: "lease-other", + }, +])( + "quarantines a mismatched $binding instead of reusing its provider session", + async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { + const container = await mkdtemp( + join(tmpdir(), "runnerd-mismatched-authority-"), + ); + const stateDirectory = join(container, "state"); + await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); + await writeFile( + join(stateDirectory, "control-plane", "control-plane-state.json"), + JSON.stringify({ + identity: { + runnerInstanceId: priorRunnerInstanceId, + environmentLeaseId: priorEnvironmentLeaseId, + runId: "run-old", + normalizedSessionId: "session-current", + turnId: "turn-old", + itemId: "item-old", + }, + }), + { mode: 0o600 }, + ); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + stateDirectory, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, + prpIdentity: { + runnerInstanceId: "runner-current", + environmentLeaseId: "lease-current", + runId: "run-new", + normalizedSessionId: "session-current", + turnId: "turn-new", + itemId: "item-new", + }, + }); + try { + await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_state_quarantined", + ); + expect(await readdir(stateDirectory)).toEqual([]); + expect( + (await readdir(container)).some((entry) => + entry.startsWith("state.quarantine-"), + ), + ).toBe(true); + } finally { + await bundle.transport.close(); + await rm(container, { recursive: true, force: true }); + } + }, +); + +it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); + const tracePath = join(stateDirectory, "provider-trace.ndjson"); + const skillRoot = join(stateDirectory, "runtime-skill"); + const instructionRoot = join(stateDirectory, "runtime-instructions"); + await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); + await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); + await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); + const credentialsCleaned = pendingTail === "credentials-cleaned"; + const sourceCodexHome = join(stateDirectory, "selected-account-first"); + if (credentialsCleaned) { + await mkdir(sourceCodexHome); + await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + const baseIdentity = { + runnerInstanceId: "runner-cold-attach", + environmentLeaseId: "lease-cold-attach", + runId: "run-cold-first", + normalizedSessionId: "session-cold-attach", + turnId: "turn-cold-first", + itemId: "item-cold-first", + }; + const options = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs( + stateDirectory, + "--include-skill-instructions", + "--durable-turn-ids", + ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), + "-c", + 'shell_environment_policy.set={PATH="/run/A"}', + ), + stateDirectory, + ...(credentialsCleaned ? { sourceCodexHome } : {}), + environment: { + PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", + PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, + PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_NAME: "assigned", + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", + PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", + } : {}), + }, + lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, + runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), + }; + const dynamicTools = [ + { + name: "get_task_context", + description: "Read the active task.", + inputSchema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + ]; + const first = createCapabilityRunnerdCodexTransport({ + ...options, + prpIdentity: baseIdentity, + }); + first.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + let firstProviderThread: { id: string; sessionId: string } | null = null; + try { + const started = await first.transport.request("thread/start", { + cwd: tmpdir(), + dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], + completionContract: { + revision: "contract-first", + criterionIds: ["criterion-first"], + }, + }); + const startedThread = started.thread as Record; + firstProviderThread = { + id: String(startedThread.id), + sessionId: String(startedThread.sessionId), + }; + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + await first.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of first.transport.notifications()) { + if (event.method === "turn/completed") break; + } + } finally { + await first.transport.close(); + } + if (!firstProviderThread) { + throw new Error("cold attach fixture did not return a provider thread"); + } + const successorCodexHome = join(stateDirectory, "selected-account-next"); + if (credentialsCleaned) { + await rm(join(stateDirectory, "codex-home", "auth.json")); + await rm(sourceCodexHome, { recursive: true }); + await mkdir(successorCodexHome); + await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); + } + + // Model a retained provider-owned suffix: rotation must deliver and durably + // accept it before attempting run.attach, rather than dropping or ACKing it. + const retainedDelta = "cold-recovery-retained-tail"; + if (pendingTail) { + const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); + const provider = JSON.parse(await readFile(providerFile, "utf8")); + expect(provider.pendingEvents).toEqual([]); + provider.pendingEvents.push({ + executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, + eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", + priority: "p0", + payload: pendingTail === "unknown-semantic" ? { semantic_tool: { + schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", + callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, + correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, + turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, + content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, + } } : { provider: "codex", kind: "usage", delta: retainedDelta, + providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, + runDeltaAvailable: true, runDeltaBasis: "run" }, + }); + await writeFile(providerFile, JSON.stringify(provider)); + } + const retainedEvents: PrpEvent[] = []; + let releaseRetained!: () => void, retainedEntered!: () => void; + const retainedAccepted = new Promise(done => { releaseRetained = done; }); + const retainedAdmission = new Promise(done => { retainedEntered = done; }); + const secondIdentity = { + ...baseIdentity, + runId: "run-cold-second", + turnId: "turn-cold-second", + itemId: "item-cold-second", + }; + let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; + let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; + const rotatedOptions = { + ...options, + ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), + onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { + authority.setCheckpointPersistence(async checkpoint => { + stagedCheckpoint = checkpoint; + if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); + }); + } : undefined, + onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { + if (event.payload.delta === retainedDelta) { + retainedEvents.push(structuredClone(event)); + retainedEntered(); + if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); + await retainedAccepted; + } + } : undefined, + environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", + ...(credentialsCleaned ? { + PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", + PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", + } : {}), + }, + codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-second", + criterionIds: ["criterion-second"], + }, + prpIdentity: secondIdentity, + }; + let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + if (pendingTail === "restart-before-bootstrap") { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); + expect(rotated.evidence().runnerPid).toBeNull(); + expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); + expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); + await rotated.detachControllerForRestart().catch(() => undefined); + rejectStagedCommit = false; + await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); + stagedCheckpoint!.restoreSessionDirectory(stateDirectory); + rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); + } + const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); + rotated.transport.setServerRequestHandler(currentHandler); + if (pendingTail === "accounting-rejected") { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + try { + await Promise.race([retainedAdmission, reading]); + const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(source.identity).toEqual(baseIdentity); + expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + expect(currentHandler).not.toHaveBeenCalled(); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await expect(reading).rejects.toThrow(); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + if (pendingTail === "unknown-semantic") { + try { + await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); + expect(currentHandler).not.toHaveBeenCalled(); + const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retained.identity).toEqual(baseIdentity); + expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); + expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); + expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); + } finally { + await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + } + return; + } + let recoveryError: unknown; + try { + const reading = rotated.transport.request("thread/read", {}); + void reading.catch(() => undefined); + if (pendingTail) { + await Promise.race([retainedAdmission, reading]); + const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); + expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => + entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); + const retainedCore = stagedCheckpoint ? { + identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, + commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), + } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(retainedCore.identity).toEqual(baseIdentity); + expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); + if (credentialsCleaned) { + const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); + expect(config).toContain("https://paperclip.example/mcp/current"); + expect(config).toContain("current-assigned-gateway-credential"); + expect(config).not.toContain("first-assigned-gateway-credential"); + expect(currentHandler).not.toHaveBeenCalled(); + } + } + releaseRetained(); + const read = await reading; + expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); + if (pendingTail) expect(retainedEvents[0]).toMatchObject({ + runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, + }); + const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); + expect(persistedProvider.pendingEvents).toEqual([]); + expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); + expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + await rotated.transport.request("turn/start", { + input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], + }); + for await (const event of rotated.transport.notifications()) { + if (event.method === "paperclip/runResult") break; + } + expect(rotated.evidence().diagnostics).toContain( + "runnerd attached the durable provider session to a fresh PRP run authority", + ); + if (pendingTail) { + const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state + : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); + expect(target.stagedRunAttachment).toBeUndefined(); + expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); + } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); + } catch (error) { + recoveryError = error; + throw error; + } finally { + releaseRetained(); + await rotated.transport.close().catch(error => { + if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); + throw error; + }); + } + const requestFrames = (await readFile(tracePath, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record) + .filter( + (entry) => + entry.kind === "frame" && entry.direction === "client_to_provider", + ) + .map( + (entry) => + JSON.parse( + Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), + ) as Record, + ) + ; + const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); + for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { + expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); + } + const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); + expect(turnFrames).toHaveLength(2); + for (const frame of turnFrames) { + expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); + } + expect(resumeFrames.length).toBeGreaterThanOrEqual(1); + for (const frame of resumeFrames) { + expect(frame).toEqual( + expect.objectContaining({ + method: "thread/resume", + params: expect.objectContaining({ threadId: firstProviderThread.id }), + }), + ); + } + + if (pendingTail) { + await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); + await rm(stateDirectory, { recursive: true, force: true }); + return; + } + + // A remote process owner keeps runner-state outside the controller's local + // session root. Resume must defer to its explicit state reader instead of + // rejecting recovery before the remote checkpoint can be made available. + const externallyOwnedRunnerStateDirectory = join( + stateDirectory, + "externally-owned-runner", + ); + await rename( + join(stateDirectory, "runner"), + externallyOwnedRunnerStateDirectory, + ); + const readRunnerState = async () => + JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + "utf8", + ), + ) as Record; + const mismatched = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: { + ...secondIdentity, + runId: "run-cold-other", + turnId: "turn-cold-other", + itemId: "item-cold-other", + }, + }); + await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( + "native_runner_prp_run_rotation_unavailable", + ); + await mismatched.transport.close(); + + const externalIdentity = { + ...secondIdentity, + runId: "run-cold-external", + turnId: "turn-cold-external", + itemId: "item-cold-external", + }; + const rejectedExternalRotationSteps: string[] = []; + let externalArchiveDirectory: string | null = null; + const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + rejectedExternalRotationSteps.push("prepared"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + await expect( + stat(join(stateDirectory, "control-plane")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect( + await stat( + join( + stateDirectory, + "authority-epochs", + `epoch-${archiveKey}`, + "control-plane", + ), + ), + ).toBeDefined(); + externalArchiveDirectory = join( + stateDirectory, + "external-authority-epochs", + archiveKey, + ); + await mkdir(externalArchiveDirectory, { recursive: true }); + await rename( + join(externallyOwnedRunnerStateDirectory, "runner-state.json"), + join(externalArchiveDirectory, "runner-state.json"), + ); + rejectedExternalRotationSteps.push("remote-archived"); + throw new Error("controller crashed after external archive"); + }, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + await expect( + rejectedExternalRotation.transport.request("thread/read", {}), + ).rejects.toThrow("controller crashed after external archive"); + await rejectedExternalRotation.transport.close(); + expect(rejectedExternalRotationSteps).toEqual([ + "prepared", + "remote-archived", + ]); + await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); + await expect( + stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), + ).rejects.toThrow(); + + const externalRotationSteps: string[] = []; + const externallyRotated = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + prepareExternalRunnerState: async () => { + throw new Error("retry must not prepare a new external runner"); + }, + archiveExternalRunnerState: async ({ archiveKey }) => { + expect(externalArchiveDirectory).toBe( + join(stateDirectory, "external-authority-epochs", archiveKey), + ); + externalRotationSteps.push("archived"); + return JSON.parse( + await readFile( + join(externalArchiveDirectory!, "runner-state.json"), + "utf8", + ), + ) as Record; + }, + resumeDynamicTools: dynamicTools, + resumeCompletionContract: { + revision: "contract-external", + criterionIds: ["criterion-external"], + }, + prpIdentity: externalIdentity, + }); + externallyRotated.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await externallyRotated.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: firstProviderThread.id, + sessionId: firstProviderThread.sessionId, + cwd: tmpdir(), + }); + expect(externalRotationSteps).toEqual(["archived"]); + } finally { + await externallyRotated.transport.close(); + } + + const restored = createCapabilityRunnerdCodexTransport({ + ...options, + runnerStateDirectory: externallyOwnedRunnerStateDirectory, + readRunnerState, + resumeDynamicTools: dynamicTools, + prpIdentity: externalIdentity, + }); + restored.transport.setServerRequestHandler(async () => ({ + success: true, + contentItems: [], + })); + try { + const read = await restored.transport.request("thread/read", {}); + expect(read.thread).toMatchObject({ + id: "codex-thread-1", + cwd: tmpdir(), + }); + expect( + (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) + .mode & 0o222, + ).toBe(0); + expect( + ( + await stat( + join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), + ) + ).mode & 0o222, + ).toBe(0); + const providerState = JSON.parse( + await readFile( + join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), + "utf8", + ), + ) as { toolBridge?: { authorized?: Record } }; + expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ + "get_task_context", + "paperclip_block", + "paperclip_finish", + ]); + await restored.transport.request("turn/start", { + input: [{ type: "text", text: "restored process" }], + }); + for await (const event of restored.transport.notifications()) { + if (event.method === "turn/completed") break; + } + expect(restored.evidence()).toMatchObject({ + runnerExited: false, + codexPid: expect.any(Number), + }); + } finally { + await restored.transport.close(); + await releaseMaterializedNativeRuntimeSkills( + join(stateDirectory, "codex-home", "skills"), + ); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index da352598f0c..9a3f342ef43 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1,80 +1,41 @@ -import { bindRunnerLaunchResources, createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type PreparedRunnerProviderLaunch, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; +import { + bindRunnerLaunchResources, + createRunnerProviderLaunch, + runnerLaunchRequirementsDigest, + type RunnerProviderLaunchRequirements, +} from "./runner-provider-launch.js"; import { coldAdmissionTimeoutMs, waitForRunnerCommand } from "./runnerd-codex-transport.js"; import { configuredEnvironmentProjection } from "../configured-environment.js"; -import { - chmod, - cp, - mkdir, - lstat, - mkdtemp, - open, - readFile, - readdir, - readlink, - realpath, - rename, - rm, - stat, - truncate, - writeFile, -} from "node:fs/promises"; -import { createHash, randomUUID } from "node:crypto"; +import { chmod, cp, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, stat, truncate, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { execFileSync, spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { bindAcpxAgentFiles } from "../drivers/acpx/agent-files-binding.js"; import { resolveRunnerProviderAssetsRoot } from "../drivers/acpx/provider-assets-root.js"; import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; - import { describe, expect, it, vi } from "vitest"; -import type { ControlPlanePort } from "../contracts/control-plane-port.js"; -import type { NativeExecutionInputV1 } from "../contracts/native-execution.js"; -import type { - NativeSession, - NativeSessionBackend, -} from "../contracts/native-session-backend.js"; -import type { - PrpEvent, - PrpStructuredRunResult, - PrpTerminalState, -} from "../protocol/replay-contract.js"; +import type { PrpEvent } from "../protocol/replay-contract.js"; import { PRP_PROTOCOL_VERSION } from "../protocol/replay-contract.js"; -import { executeNativeSession } from "../native-session-runtime.js"; import { redactCapabilityEvidenceData } from "./evidence-redaction.js"; import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import { parsePaperclipQuestionSet } from "../contracts/question-set.js"; -import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxCredentialBinding, createAcpxSidecarHostEnvironment } from "../drivers/acpx/environment.js"; +import { + ACPX_CREDENTIAL_BINDING_ENV, + createAcpxCredentialBinding, + createAcpxSidecarHostEnvironment, +} from "../drivers/acpx/environment.js"; import { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; import * as codexCommandRuntime from "../drivers/codex/codex-command.js"; - -import { - NATIVE_RUNTIME_ASSET_SCHEMA, - PAPERCLIP_EXECUTION_PROMPT, - PAPERCLIP_EXECUTION_PROMPT_REVISION, - canonicalNativeRuntimeContextDigest, - composeNativeSystemInstructions, - nativeRuntimePromptDigest, - type NativeRuntimeContextSnapshot, -} from "../contracts/runtime-context.js"; -import { - CODEX_SKILLLESS_BASE_INSTRUCTIONS, - createCodexTaskEnvelope, -} from "../contracts/codex.js"; -import { - PRP_BLOCK_TOOL_DESCRIPTION, - PRP_COMPLETION_TOOL_DESCRIPTION, -} from "../contracts/completion-result.js"; -import { - CodexAppServerDriver, - codexSemanticToolSpecs, -} from "../drivers/codex/codex-app-server-driver.js"; -import { releaseMaterializedNativeRuntimeSkills } from "../drivers/runtime-context-materializer.js"; +import { composeNativeSystemInstructions, type NativeRuntimeContextSnapshot } from "../contracts/runtime-context.js"; +import { CODEX_SKILLLESS_BASE_INSTRUCTIONS, createCodexTaskEnvelope } from "../contracts/codex.js"; +import { PRP_BLOCK_TOOL_DESCRIPTION, PRP_COMPLETION_TOOL_DESCRIPTION } from "../contracts/completion-result.js"; +import { CodexAppServerDriver, codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; import * as runtimeMaterializer from "../drivers/runtime-context-materializer.js"; import { RUNNERD_CANONICAL_ITEM } from "../drivers/codex/codex-driver-values.js"; - import { probeNativeRunnerEnvironment, NativeEnvironmentProbeCleanupError, @@ -83,7 +44,6 @@ import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, createRunnerdCodexAppServerArgs, - defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary, readRunnerdArtifactBinding, expandRunnerdCanonicalNotifications, latestRunnerdSessionReadiness, @@ -109,11 +69,13 @@ import { resolveRunnerdCodexSkillInputs, withCodexCollaborationRuntimeInstructions, } from "./runnerd-codex-transport.js"; - -// Explicit private-artifact test lane; production/default dist is never changed. -const defaultCapabilityRunnerdBinary = () => - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - qualifiedCapabilityRunnerdBinary(); +import { + defaultCapabilityRunnerdBinary, + expectTurnStarted, + fakeCodex, + fakeCodexArgs, + assignedRuntimeContext, +} from "../../test/support/runnerd-transport.js"; it("attributes opened candidate sessions to the harness biller instead of the model vendor", () => { const provider = runnerdLaunchProfileInternals.openedThreadModelProvider; @@ -130,21 +92,6 @@ it("attributes opened candidate sessions to the harness biller instead of the mo expect(provider("aws_agentcore", "codex", "global.anthropic.claude-sonnet-4-6")).toBe("aws"); }); -async function expectTurnStarted( - notifications: AsyncIterator<{ method: string }>, -) { - for (let index = 0; index < 32; index += 1) { - const next = await notifications.next(); - expect(next.done).not.toBe(true); - if (next.value.method === "turn/started") return; - // Startup ownership and capability facts can precede the active turn. - expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); - } - throw new Error( - "provider turn did not start within the bounded notification prefix", - ); -} - it("replaces an owned v1 runner with fresh v2 authorization before warm attachment", async () => { const directory = await mkdtemp(join(tmpdir(), "runnerd-v1-v2-replacement-")); const handles: durableControlPlane.RunnerProcessHandle[] = []; @@ -458,12 +405,6 @@ it("preserves the cause when a replacement runner's health cannot be verified", } }, 15_000); -// The retained-settlement maintenance family ("settles only retained control -// authority without starting another provider turn") lives in -// runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of -// sequential cases on a separate worker instead of serializing them behind -// this file's tests. - it.each(["alive", "pending_liveness", "pending_registration"] as const)( "bounds adopted runner authentication at its exact deadline with %s evidence", async (mode) => { @@ -2055,6 +1996,7 @@ it.each([false, true])("binds a vendored source server to its pnpm dependency ro await rm(workspace, { recursive: true, force: true }); } }); + it.each((["copilot", "pi", "antigravity"] as const).flatMap(agent => ["source", "scoped-npm", "deployment"].map(layout => ({ agent, layout }))))( "admits the controller-selected $agent package authority in a $layout layout", @@ -3498,11 +3440,6 @@ it("recovers provider readiness from an already-committed journal without replay ).toEqual(persistedReady); }); -const fakeCodex = resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", -); - it("keeps the Copilot authentication launch requirements free of Codex search policy", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-copilot-probe-policy-")); let captured: RunnerProviderLaunchRequirements | undefined; @@ -3641,14 +3578,6 @@ it.each(["stuck", "late-release", "late-release-error", "stuck-release", "cooper }, ); -function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { - return [ - "--state-file", - join(stateDirectory, "fake-codex-state.json"), - ...args, - ]; -} - it("joins owned credential preparation before close returns and never launches after close", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-owned-startup-")); const artifact = join(root, "unexecuted-runner-artifact"); @@ -3894,51 +3823,6 @@ it.each(["admission-only", "admission-only-mismatched-model", "complete", "api-k } finally { await rm(root, { recursive: true, force: true }); } }, 20_000); -function assignedRuntimeContext( - skillRoot: string, - instructionRoot: string, -): NativeRuntimeContextSnapshot { - const digest = "0".repeat(64); - const value = { - prompt: { - revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, - text: PAPERCLIP_EXECUTION_PROMPT, - digest: nativeRuntimePromptDigest(), - }, - instructions: { - entryPath: "AGENTS.md", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: instructionRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - skills: [ - { - key: "company/assigned", - runtimeName: "assigned", - versionId: "version-1", - bundle: { - schema: NATIVE_RUNTIME_ASSET_SCHEMA, - digest, - manifestDigest: digest, - rootPath: skillRoot, - fileCount: 1, - totalBytes: 1, - }, - }, - ], - mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, - } satisfies Omit; - return { - ...value, - aggregateDigest: canonicalNativeRuntimeContextDigest(value), - }; -} - it("unwraps a coalesced provider notification without losing its turn identity", () => { expect( unwrapRunnerdProviderNotification({ @@ -4817,528 +4701,6 @@ it.each([ 60_000, ); -it("rejects active work and buffered tools from a resumed stopped checkpoint", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-stopped-resume-active-"), - ); - const identity = { - runnerInstanceId: "runner-stopped-active", - environmentLeaseId: "lease-stopped-active", - runId: "run-stopped-first", - normalizedSessionId: "session-stopped-active", - turnId: "turn-stopped-first", - itemId: "item-stopped-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--linger-after-turn-start", - "--resume-unowned-turn-when-marked", - "--emit-tool-call-on-resume", - ), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - }; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: identity, - }); - let second: - ReturnType | undefined; - const semanticHandler = vi.fn(async () => ({ - success: true, - contentItems: [], - })); - first.transport.setServerRequestHandler(semanticHandler); - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the current task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - }); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - await expectTurnStarted( - first.transport.notifications()[Symbol.asyncIterator](), - ); - await first.transport.close(); - const providerPath = join( - stateDirectory, - "runner", - "codex-provider-state.json", - ); - expect(JSON.parse(await readFile(providerPath, "utf8"))).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - await writeFile(join(stateDirectory, "resume-unowned-turn"), "armed"); - const secondIdentity = { - ...identity, - runId: "run-stopped-second", - turnId: "turn-stopped-second", - itemId: "item-stopped-second", - }; - second = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: secondIdentity, - }); - second.transport.setServerRequestHandler(semanticHandler); - await expect(second.transport.request("thread/read", {})).rejects.toThrow( - "prepared provider checkpoint resumed unexpected active work", - ); - // Exercise the normal durable transfer before inspecting the rejection: - // provider pendingEvents is an acknowledged queue, not an event journal. - await vi.waitFor( - async () => { - const provider = JSON.parse(await readFile(providerPath, "utf8")); - expect(provider.pendingEvents).toEqual([]); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ) as { - committedEvents: Array<{ - eventType: string; - envelope: { payload: { payload: { code?: string } } }; - }>; - }; - const rejections = control.committedEvents.filter( - (event) => - event.eventType === "harness.diagnostic" && - event.envelope.payload.payload.code === - "prepared_provider_checkpoint_has_active_work", - ); - expect(rejections).toEqual([ - expect.objectContaining({ - logicalEffectCount: 1, - envelope: expect.objectContaining({ - ...secondIdentity, - payload: expect.objectContaining({ - payload: expect.objectContaining({ - code: "prepared_provider_checkpoint_has_active_work", - paperclipAccepted: false, - providerReportedActive: true, - }), - }), - }), - }), - ]); - }, - { timeout: 5_000 }, - ); - const closed = JSON.parse(await readFile(providerPath, "utf8")); - expect(closed).toMatchObject({ - lifecycle: "closed", - threadId: (opened.thread as Record).id, - activeProviderTurnId: null, - }); - expect(semanticHandler).not.toHaveBeenCalled(); - expect(closed.pendingEvents).toEqual([]); - } finally { - await Promise.allSettled([ - first.transport.close(), - second?.transport.close(), - ]); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); -it("does not retry a real memoized transport close whose suspension proof is unavailable", async () => { - const identity = { - runId: "run-recovery", - sessionId: "session-recovery", - companyId: "company-recovery", - issueId: "issue-recovery", - agentId: "agent-recovery", - }; - - const result: PrpStructuredRunResult = { - schema: "paperclip.run_result.v1", - reportedWorkDisposition: "done", - summary: "Recovered native work completed.", - completionClaim: { - contractRevision: "1", - objectiveSatisfied: true, - criteria: [ - { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, - ], - remainingWork: [], - }, - evidence: [], - verification: [{ commandOrCheck: "recovery", status: "passed" }], - attentionRequests: [], - artifacts: [], - }; - - const terminal: PrpTerminalState = { - schema: "paperclip.prp.terminal.v1", - turnTerminalState: "completed", - runTerminalState: "succeeded", - reportedWorkDisposition: "done", - }; - - const input: NativeExecutionInputV1 = { - schema: "paperclip.native-execution-input.v1", - binding: { - companyId: identity.companyId, - runId: identity.runId, - issueId: identity.issueId, - agentId: identity.agentId, - executionWorkspaceId: "workspace-recovery", - }, - task: { - identifier: "PAP-RECOVERY", - title: "Recover native work", - description: null, - prompt: "# PAP-RECOVERY: Recover native work", - workMode: "standard", - }, - workspace: { - cwd: "/workspace", - repoUrl: null, - repoRef: null, - branchName: null, - }, - session: { - normalizedSessionId: identity.sessionId, - driverKind: "codex_app_server", - protocolVersion: 1, - }, - provider: { kind: "codex", model: null }, - completionContract: { - id: "contract-recovery", - sha256: "contract-recovery-sha", - schemaVersion: "paperclip.completion-contract.v1", - contract: { - revision: "1", - objective: "Recover native work", - criteria: [{ id: "objective", requirement: "Complete after recovery" }], - }, - }, - interactionResponses: [], - credentialBindings: [], - }; - - function runnerEvent( - sourceSeq: number, - eventType: PrpEvent["eventType"], - payload: Record = {}, - ): PrpEvent { - return { - schema: "paperclip.prp.event.v1", - sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, - sourceSeq, - sourceInstanceId: "runner-recovery", - sourceKind: "runner", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-recovery", - eventType, - schemaVersion: 1, - priority: 0, - emittedAt: "2026-08-09T00:00:00.000Z", - payload, - }; - } - const stateDirectory = await mkdtemp( - join(tmpdir(), "native-close-quarantine-"), - ); - const readRunnerState = vi.fn(async () => ({ - schema: "paperclip.runner.durable.state.v1", - runnerInstanceId: "runner-close-quarantine", - environmentLeaseId: "lease-close-quarantine", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-close-quarantine", - itemId: "item-close-quarantine", - lifecycle: "ready", - })); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: resolve( - import.meta.dirname, - "../../runner/target/debug/fake-codex-app-server", - ), - codexArgs: ["--state-file", join(stateDirectory, "fake-codex-state.json")], - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: await readRunnerState(), - readRunnerState, - // A checkpoint owner requires durable suspension proof. A local transport - // without a checkpoint can simply terminate its process on close. - controlPlaneRegistration: async (authority) => { - await authority.start(); - return { - connectUrl: authority.connectUrl, - checkpoint: async () => {}, - release: async () => {}, - }; - }, - }); - try { - await bundle.transport.request("thread/start", { - cwd: stateDirectory, - dynamicTools: [], - }); - const failedClose = bundle.transport.close(); - const failure = await failedClose.catch((error: unknown) => error); - expect(failure).toMatchObject({ - code: "native_session_close_unrecoverable", - }); - expect(bundle.transport.close()).toBe(failedClose); - - vi.useFakeTimers(); - const close = vi.fn(({ reason }: { reason: string }) => - bundle.transport.close(reason), - ); - const capabilities = { - resume: true, - typedEvents: true, - steering: false, - interruption: true, - structuredResult: true, - }; - const session: NativeSession = { - identity: () => identity, - async capabilities() { - return capabilities; - }, - async *events() { - yield runnerEvent(1, "turn.completed"); - }, - async startTurn() { - return { turnId: "turn-recovery" }; - }, - async result() { - return { result, terminal, turnId: "turn-recovery" }; - }, - async snapshot() { - return { - backendKind: "mock", - sessionId: identity.sessionId, - identity, - providerSessionId: "provider-recovery", - cursor: null, - activeTurnId: null, - pendingRuntimeRequests: [], - lineage: [], - }; - }, - close, - }; - const openSession = vi.fn(async () => session); - const backend: NativeSessionBackend = { - async descriptor() { - return { - kind: "mock", - name: "real-memoized-close-quarantine", - version: "1", - capabilities, - }; - }, - openSession, - }; - const port: ControlPlanePort = { - async openRun() {}, - async checkpointSession() {}, - async appendEvent() { - return { - cursor: 1, - highestContiguousSourceSeq: 1, - disposition: "committed", - }; - }, - async replayEvents() { - return { events: [], highestContiguousSourceSeq: 0 }; - }, - async completeRun() {}, - }; - const execute = () => - executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - requireSessionCloseBeforeReturn: true, - }); - await expect(execute()).rejects.toBe(failure); - const readsAfterClose = readRunnerState.mock.calls.length; - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - recovery: "operator_required", - }); - await vi.advanceTimersByTimeAsync(600_000); - await expect(execute()).rejects.toMatchObject({ - code: "native_session_cleanup_quarantined", - }); - expect(openSession).toHaveBeenCalledOnce(); - expect(close).toHaveBeenCalledOnce(); - expect(readRunnerState).toHaveBeenCalledTimes(readsAfterClose); - } finally { - vi.useRealTimers(); - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 10_000); - -it.each(["not_suspended", "wrong_identity"] as const)( - "does not report local runner close healthy with %s durable evidence", - async (mode) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-local-close-unproven-"), - ); - const identity = { - runnerInstanceId: "runner-close-unproven", - environmentLeaseId: "lease-close-unproven", - runId: "run-close-unproven", - normalizedSessionId: "session-close-unproven", - turnId: "turn-close-unproven", - itemId: "item-close-unproven", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - closeGraceMs: 400, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - readRunnerState: async () => ({ - schema: "paperclip.runner.durable.state.v1", - ...identity, - ...(mode === "wrong_identity" ? { runId: "some-other-run" } : {}), - lifecycle: mode === "not_suspended" ? "ready" : "suspended", - }), - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await expect(bundle.transport.close()).rejects.toThrow( - "runner did not durably suspend before checkpoint", - ); - const control = JSON.parse( - await readFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - "utf8", - ), - ); - expect(control.identity).toEqual(identity); - expect(await readdir(stateDirectory)).toContain("runner"); - } finally { - await bundle.transport.close().catch(() => undefined); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 10_000, -); - -it.each([ - { - label: "interrupted", - args: ["--hold-turn"], - terminal: "turn.interrupted", - disposition: "needs_review", - }, - { - label: "failed", - args: ["--fail-turn-immediately"], - terminal: "turn.failed", - disposition: "needs_review", - }, - { - label: "completed", - args: [], - terminal: "turn.completed", - disposition: "done", - }, -])("retains a $label runner's result before closing its provider turn", async ({ - label, args, terminal, disposition, -}) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-stop-result-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...args), - stateDirectory, - }); - const driver = new CodexAppServerDriver({ - taskEnvelope: createCodexTaskEnvelope({ - objective: "Stop and retain unfinished work.", - }), - environment: { - PATH: process.env.PATH, - HOME: join(tmpdir(), "runnerd-stop-host-home"), - PAPERCLIP_WORKSPACE_CWD: stateDirectory, - }, - approvalPolicy: "never", - transportFactory: () => bundle.transport, - }); - const session = await driver.openSession({ - runId: "run-stop-result", - normalizedSessionId: "session-stop-result", - workingDirectory: stateDirectory, - }); - try { - const turn = await session.startTurn({ - message: { role: "user", text: "Keep working until interrupted." }, - }); - if (label === "interrupted") { - await session.interrupt({ - turnId: turn.turnId, - reason: "Stopped by the user", - }); - } - const events: PrpEvent[] = []; - for await (const event of session.events()) { - events.push(event); - if (event.eventType === "session.failed" || event.eventType === terminal) { - break; - } - } - // Let already committed durable suffix events reach the facade as well. - await bundle.transport.request("thread/read", {}); - const snapshot = await session.snapshot(); - expect(events.some((event) => event.eventType === "session.failed")).toBe(false); - expect(events.map((event) => event.eventType)).toContain("run.result.proposed"); - expect(events.at(-1)?.eventType).toBe(terminal); - expect(snapshot).toMatchObject({ - activeTurnId: null, - semanticResult: { - turnId: turn.turnId, - result: { reportedWorkDisposition: disposition }, - }, - }); - } finally { - await session.close().catch(() => undefined); - await rm(stateDirectory, { - recursive: true, - force: true, - maxRetries: 5, - retryDelay: 25, - }); - } -}, 30_000); - it("binds an immediately failed durable turn before exposing its terminal", async () => { const stateDirectory = await mkdtemp( join(tmpdir(), "runnerd-fast-terminal-"), @@ -5908,3406 +5270,6 @@ it("steers the active provider turn through the durable PRP command path", async } }, 30_000); -it.each([ - ["held-ack", "normal"], - ["lost-ack", "normal"], - ["rejected-attach", "normal"], - ["lost-ack", "after-activation"], -] as const)( - "preserves old warm-attach authority and event ownership across %s (%s observer)", - async (mode, observer) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-ack-")); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const effects = new Map(); - const handles: ReturnType[] = []; - let armed = false; - let heldEvent: PrpEvent | null = null; - let releaseCommit!: () => void; - let enteredCommit!: () => void; - const commitGate = new Promise((resolveCommit) => { - releaseCommit = resolveCommit; - }); - const commitEntered = new Promise((resolveEntered) => { - enteredCommit = resolveEntered; - }); - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore({ - ...options, - onCommittedEvent: async (event) => { - await options.onCommittedEvent?.(event); - const prior = effects.get(event.sourceEventId); - if (prior) { - expect(event).toEqual(prior.event); - prior.deliveries += 1; - } else { - effects.set(event.sourceEventId, { - event: structuredClone(event), - deliveries: 1, - }); - } - if ( - armed && - mode !== "rejected-attach" && - heldEvent === null && - event.eventType === "run.attached" - ) { - heldEvent = structuredClone(event); - enteredCommit(); - await commitGate; - if (mode === "lost-ack") { - // The external durable effect exists, but this connection - // disappears before its local cursor/ACK can be published. - throw new Error( - "fixture lost the old authority ACK after commit", - ); - } - } - }, - }); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolveWait, rejectWait) => { - timer = setTimeout( - () => rejectWait(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - }); - const readRunner = async () => - JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ) as { - runId: string; - ackedSourceSeq: number; - processedCommands: Record; - outbox: { envelope: { payload: PrpEvent } }[]; - }; - let providerPid: number | null = null; - let primaryError: unknown; - let cleanupProven = false; - try { - const opened = await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ) as { thread: { id: string } }; - const core = cores[0]!; - expect(cores).toHaveLength(1); - const oldIdentity = structuredClone(core.store.state.identity); - const runnerPid = bundle.evidence().runnerPid; - providerPid = bundle.evidence().codexPid; - const rotations: (typeof core.store.state)[] = []; - const preparedStates: (typeof core.store.state)[] = []; - const commit = core.store.commit.bind(core.store); - vi.spyOn(core.store, "commit").mockImplementation((candidate) => { - commit(candidate); - // Capture the durable old epoch at publication. The authenticated - // successor may activate before attachRun observes the completed command. - if ( - candidate.identity.runId === oldIdentity.runId && - candidate.warmTransition?.phase === "prepared" - ) { - preparedStates.push(structuredClone(core.store.state)); - } - }); - if (observer === "after-activation") { - const getCommand = core.getCommand.bind(core); - vi.spyOn(core, "getCommand").mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - command.status === "completed" && - core.store.state.completedWarmTransition === undefined - ) { - // Deterministically exercise an observer that misses the old epoch. - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - const rotate = core.rotateRunIdentity.bind(core); - vi.spyOn(core, "rotateRunIdentity").mockImplementation( - (identity, template) => { - rotations.push(structuredClone(core.store.state)); - return rotate(identity, template); - }, - ); - if (mode === "rejected-attach") { - const queue = core.queueCommand.bind(core); - vi.spyOn(core, "queueCommand").mockImplementation( - (type, payload = {}, id, immediate) => - queue( - type, - type === "run.attach" - ? { - ...payload, - provider: { - ...(payload.provider as Record), - model: "foreign-profile", - }, - } - : payload, - id, - immediate, - ), - ); - } - armed = true; - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-ack-next", - turnId: "turn-warm-ack-next", - itemId: "item-warm-ack-next", - }); - void attachment.catch(() => undefined); - if (mode === "rejected-attach") { - await expect(within("rejected attach", attachment)).rejects.toThrow( - "run.attach cannot change the durable Codex provider profile", - ); - expect(rotations).toHaveLength(0); - expect(core.store.state.identity).toEqual(oldIdentity); - expect((await readRunner()).runId).toBe(oldIdentity.runId); - const read = await within( - "read under unchanged authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("failed"); - } else { - await within("old authority commit barrier", commitEntered); - const retained = await readRunner(); - expect(retained.runId).toBe(oldIdentity.runId); - expect( - Object.values(retained.processedCommands).find((entry) => entry.commandType === "run.attach") - ?.status, - ).toBe("completed"); - expect( - retained.outbox.some( - (entry) => - entry.envelope.payload.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(true); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - expect(rotations).toHaveLength(0); - if (mode === "lost-ack") core.disconnectActiveRunner(); - releaseCommit(); - await within("warm attach after old ACK", attachment, 10_000); - expect(rotations).toHaveLength(1); - expect(preparedStates.length).toBeGreaterThan(0); - const retired = preparedStates[0]!; - if (observer === "after-activation") { - expect(rotations[0]!.identity.runId).toBe("run-warm-ack-next"); - expect( - rotations[0]!.committedEvents.some( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - ), - ).toBe(false); - } - const attachedEvent = retired.committedEvents.find( - (entry) => entry.sourceEventId === heldEvent!.sourceEventId, - )!; - expect(attachedEvent.logicalEffectCount).toBe(1); - expect(retired.ackedSourceSeq).toBeGreaterThanOrEqual( - attachedEvent.sourceSeq, - ); - expect( - retired.committedEvents.slice(-4).map((entry) => entry.eventType), - ).toEqual([ - "session.resumed", - "session.capabilities.updated", - "session.goal.snapshot", - "run.attached", - ]); - expect( - retired.committedEvents.every( - (entry) => entry.envelope.runId === oldIdentity.runId, - ), - ).toBe(true); - if (mode === "lost-ack") { - expect(retired.connectionCount).toBeGreaterThanOrEqual(2); - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(2); - } else { - expect(effects.get(heldEvent!.sourceEventId)?.deliveries).toBe(1); - } - await vi.waitFor(async () => - expect((await readRunner()).runId).toBe("run-warm-ack-next"), - ); - const read = await within( - "read under new authority", - bundle.transport.request("thread/read", {}), - ); - expect((read.thread as { id: string }).id).toBe(opened.thread.id); - } - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - } catch (error) { - primaryError = error; - throw error; - } finally { - releaseCommit(); - try { - await within( - "warm fixture close", - bundle.transport.close(), - 10_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact warm fixture runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } catch (error) { - if (primaryError === undefined) throw error; - console.error( - "Warm ACK fixture cleanup unproven; retaining its private state directory.", - ); - } finally { - try { - for (const core of cores) await core.stop().catch(() => undefined); - } finally { - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - } - }, - 30_000, -); - -it.each([false, true])( - "retains warm attach authority when its result is lost before controller persistence (held observer=%s)", - async (holdObserver) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-result-loss-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch(options); - handles.push(handle); - return handle; - }); - const within = async ( - label: string, - promise: Promise, - timeout = 5_000, - ) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error(`${label} timeout`)), - timeout, - ); - }), - ]); - } finally { - if (timer !== undefined) clearTimeout(timer); - } - }; - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: - process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - }); - let providerPid: number | null = null; - let cleanupProven = false; - let saveSpy: ReturnType | undefined; - let observerSpy: { mockRestore(): void } | undefined; - let attachment: Promise | undefined; - try { - await within( - "initial thread", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - expect(cores).toHaveLength(1); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - if (holdObserver) { - const getCommand = core.getCommand.bind(core); - observerSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((commandId) => { - const command = getCommand(commandId); - if ( - command?.type === "run.attach" && - core.store.state.completedWarmTransition?.command.commandId !== - commandId - ) { - return { ...command, status: "pending", result: null }; - } - return command; - }); - } - providerPid = bundle.evidence().codexPid; - const store = core.store as typeof core.store & { - commit(candidate: typeof core.store.state): void; - }; - const commit = store.commit.bind(store); - let lostResult = false; - let observeLoss!: () => void; - const loss = new Promise((resolveLoss) => { - observeLoss = resolveLoss; - }); - saveSpy = vi.spyOn(store, "commit").mockImplementation((candidate) => { - const attach = candidate.commands.find( - (entry) => entry.type === "run.attach", - ); - if (!lostResult && attach?.status === "completed") { - lostResult = true; - // The authenticated result reached the receiver, but the durable write - // did not. The clone has not been exposed in memory or on disk. - const persisted = JSON.parse(readFileSync(core.store.path, "utf8")); - expect(persisted.identity).toEqual(oldIdentity); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - ).status, - ).toBe("pending"); - core.disconnectActiveRunner(); - observeLoss(); - throw new Error( - "fixture lost attach result before durable controller commit", - ); - } - commit(candidate); - }); - attachment = bundle.transport.attachRun!({ - runId: "run-result-loss-next", - turnId: "turn-result-loss-next", - itemId: "item-result-loss-next", - }); - void attachment.catch(() => undefined); - await within("lost attach result", loss); - expect(lostResult).toBe(true); - expect(core.store.state.identity).toEqual(oldIdentity); - expect( - core.store.state.commands.find((entry) => entry.type === "run.attach") - ?.status, - ).toBe("pending"); - const outcome = await within( - "exact result replay after reconnect", - attachment.then( - () => ({ status: "completed" as const }), - (error: unknown) => ({ - status: "failed" as const, - message: String(error), - }), - ), - 10_000, - ); - const runner = JSON.parse( - await readFile( - join(stateDirectory, "runner/runner-state.json"), - "utf8", - ), - ); - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - expect( - { - outcome, - runnerRunId: runner.runId, - controllerRunId: core.store.state.identity.runId, - }, - "lost attach result must replay without leaving the two durable authorities split", - ).toEqual({ - outcome: { status: "completed" }, - runnerRunId: "run-result-loss-next", - controllerRunId: "run-result-loss-next", - }); - if (holdObserver) - expect( - core.store.state.completedWarmTransition?.receipt.newIdentity.runId, - ).toBe("run-result-loss-next"); - } finally { - saveSpy?.mockRestore(); - observerSpy?.mockRestore(); - try { - await within( - "result-loss fixture close", - bundle.transport.close(), - 5_000, - ).catch(() => undefined); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within("exact result-loss runner exit", handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - if (providerPid && !dead(-providerPid)) - process.kill(-providerPid, "SIGKILL"); - if (providerPid) - await vi.waitFor(() => expect(dead(-providerPid!)).toBe(true)); - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - launchSpy.mockRestore(); - coreSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it.each([ - ...[ - "before-result", - "after-result", - "after-activation", - "before-confirmation", - "after-confirmation", - ].flatMap((lossPoint) => - [false, true].map((routed) => ({ - lossPoint, - routed, - recoveryFault: "none", - })), - ), - ...["endpoint", "missing-capability", "listen", "malformed-core"].map( - (recoveryFault) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault, - }), - ), - ...["before_bootstrap", "before_spawn", "before_authentication"].map( - (stage) => ({ - lossPoint: "after-result", - routed: true, - recoveryFault: `authorize_${stage}`, - }), - ), - ...[ - "attach-wait", - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? ["attach-capability"] - : []), - ].map((recoveryFault) => ({ - lossPoint: "before-result", - routed: true, - recoveryFault, - })), - ...(process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER - ? [ - { - lossPoint: "after-result", - routed: false, - recoveryFault: "legacy-parser", - }, - ] - : []), - ...["before-result", "after-result", "after-activation", "after-confirmation"].map(lossPoint => ({ - lossPoint, routed: true, recoveryFault: "none", executionPrepared: true, - })), - { - lossPoint: "after-result", - routed: true, - recoveryFault: "none", - externalArtifact: true, - }, - { - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup: true, - }, - ...[ - "missing_snapshot", - "rejected_snapshot", - "wrong_thread", - "callback_failure", - "callback_async_failure", - ].map((ordinaryFollowup) => ({ - lossPoint: "after-confirmation", - routed: true, - recoveryFault: "none", - ordinaryFollowup, - })), -])( - "recovers a warm attachment with a fresh controller and runner ($lossPoint, routed=$routed, fault=$recoveryFault, followup=$ordinaryFollowup, external=$externalArtifact, prepared=$executionPrepared)", - async (testCase) => { - const { lossPoint, routed, recoveryFault } = testCase; - const externalArtifact = "externalArtifact" in testCase && testCase.externalArtifact; - const executionPrepared = "executionPrepared" in testCase && testCase.executionPrepared; - const configurations = new Map(); - const ordinaryFollowup = - "ordinaryFollowup" in testCase && testCase.ordinaryFollowup; - const snapshotFault = - typeof ordinaryFollowup === "string" && - !ordinaryFollowup.startsWith("callback_"); - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-attach-restart-"), - ); - const callsPath = join(stateDirectory, "calls.log"); - const cores: DurablePrpControlPlane[] = []; - const handles: ReturnType[] = []; - const routes = new Map< - string, - { core: DurablePrpControlPlane; generation: symbol } - >(); - const routeCalls: string[] = []; - let recovering = false; - let recoveryClaimCurrent = true; - let recoveryFenceActive = true; - const completionSnapshotIds: string[] = []; - let rejectHeldCompletion: ((error: Error) => void) | undefined; - let markHeldCompletionEntered!: () => void; - const heldCompletionEntered = new Promise((resolve) => { - markHeldCompletionEntered = resolve; - }); - let snapshotObserverSpy: ReturnType | undefined; - const routeServer = createServer((_request, response) => - response.writeHead(404).end(), - ); - routeServer.on("upgrade", (request, socket, head) => { - const entry = routes.get(request.url ?? ""); - if (!entry) { - socket.destroy(); - return; - } - entry.core.handleUpgrade(request, socket, request.url!, head); - }); - if (routed) - await new Promise((resolveListen) => - routeServer.listen(0, "127.0.0.1", resolveListen), - ); - const routeAddress = routeServer.address(); - const routePort = - routeAddress && typeof routeAddress === "object" ? routeAddress.port : 0; - const registration = async ( - core: DurablePrpControlPlane, - identity = core.store.state.identity, - ) => { - if ( - recovering && - ordinaryFollowup && - recoveryFenceActive && - !recoveryClaimCurrent - ) { - throw new Error("fixture old recovery claim is no longer current"); - } - const path = `/api/runner/v1/connect/${identity.runId}`; - const generation = Symbol(); - routes.set(path, { core, generation }); - routeCalls.push(identity.runId); - return { - connection: - recovering && recoveryFault === "listen" - ? { - mode: "listen" as const, - listenAddress: "0.0.0.0" as const, - listenPort: routePort, - listenPath: path, - } - : { - mode: "connect" as const, - connectUrl: `ws://127.0.0.1:${routePort}${path}${recovering && recoveryFault === "endpoint" ? "/changed" : ""}`, - }, - release: () => { - if (routes.get(path)?.generation === generation) routes.delete(path); - }, - }; - }; - const OriginalCore = durableControlPlane.DurablePrpControlPlane; - const coreSpy = vi - .spyOn(durableControlPlane, "DurablePrpControlPlane") - .mockImplementation(function ( - options: ConstructorParameters[0], - ) { - const core = new OriginalCore(options); - cores.push(core); - if (recovering && snapshotFault) { - const getCommand = core.getCommand.bind(core); - snapshotObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - const command = getCommand(id); - if ( - command?.type !== "session.snapshot" || - command.status !== "completed" - ) - return command; - const observed = structuredClone(command); - if (ordinaryFollowup === "rejected_snapshot") { - observed.status = "failed"; - observed.result = { - result: { message: "fixture snapshot observation rejected" }, - }; - } else if (ordinaryFollowup === "wrong_thread") { - observed.result = { - ...observed.result, - result: { - ...(observed.result?.result as Record), - driverSessionId: "foreign-thread", - providerSessionId: "foreign-thread", - }, - }; - } else observed.result = { result: { status: "session_open" } }; - return observed; - }); - } - return core; - } as unknown as typeof OriginalCore); - const launch = durableControlPlane.spawnRunner; - const launchSpy = vi - .spyOn(durableControlPlane, "spawnRunner") - .mockImplementation((options) => { - const handle = launch({ ...options, - ...(executionPrepared ? { processLauncher: undefined } : {}), - ...(externalArtifact && recovering ? { runnerBinaryPath: defaultCapabilityRunnerdBinary() } : {}), - }); - handles.push(handle); - return handle; - }); - const dead = (pid: number) => { - try { - process.kill(pid, 0); - return false; - } catch (error) { - return (error as NodeJS.ErrnoException).code === "ESRCH"; - } - }; - const stopOwnedProvider = async (pid: number) => { - // Exact runner completion can race the OS reaping its already-signalled - // provider. Absence, not the outcome of a redundant signal, is required. - try { - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 500, - interval: 10, - }); - } catch { - try { - process.kill(-pid, "SIGKILL"); - } catch (error) { - if ( - !["ESRCH", "EPERM"].includes( - String((error as NodeJS.ErrnoException).code), - ) - ) - throw error; - } - await vi.waitFor(() => expect(dead(-pid)).toBe(true), { - timeout: 2_000, - interval: 10, - }); - } - expect(dead(pid)).toBe(true); - }; - const fingerprintTree = async (root: string): Promise => { - const rows: unknown[] = []; - const visit = async (path: string, relative: string) => { - const metadata = await lstat(path); - rows.push({ - path: relative, - inode: metadata.ino, - mode: metadata.mode, - mtimeMs: metadata.mtimeMs, - digest: metadata.isFile() - ? createHash("sha256") - .update(await readFile(path)) - .digest("hex") - : metadata.isSymbolicLink() - ? createHash("sha256") - .update(await readlink(path)) - .digest("hex") - : null, - }); - if (metadata.isDirectory()) - for (const child of (await readdir(path)).sort()) - await visit(join(path, child), `${relative}/${child}`); - }; - await visit(root, "."); - return rows; - }; - const within = async (promise: Promise, timeout = 5_000) => { - let timer: NodeJS.Timeout | undefined; - try { - return await Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout( - () => reject(new Error("warm restart fixture timed out")), - timeout, - ); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } - }; - const options = { - runnerBinary: - recoveryFault === "attach-capability" - ? process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER! - : (process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? - defaultCapabilityRunnerdBinary()), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--call-log", - callsPath, - "--record-process-start", - ), - stateDirectory, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 2_000, - prpIdentity: (() => { - const runId = randomUUID(); - return { - runnerInstanceId: randomUUID(), - environmentLeaseId: - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_SCOPE === - "transient" - ? runId - : randomUUID(), - runId, - normalizedSessionId: randomUUID(), - turnId: `turn-${runId}`, - itemId: `item-${runId}`, - }; - })(), - ...(routed - ? { - controlPlaneRegistration: registration, - warmTransitionRegistrationMode: "routed_connect" as const, - } - : {}), - ...(executionPrepared ? { - runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - readRunnerState: async () => JSON.parse(await readFile(join(stateDirectory, "runner/runner-state.json"), "utf8")), - // The fixture's spawn spy supplies the real local process owner. - runnerProcessLauncher: () => { throw new Error("fixture process owner was bypassed"); }, - prepareLaunch: async (requirements: RunnerProviderLaunchRequirements, recovery?: PreparedRunnerProviderLaunch) => { - const existing = configurations.get(requirements.identity.runId); - if (existing) { - expect(requirements).toEqual(existing.requirements); - if (recovering) expect(recovery).toEqual(existing.prepared); - return existing.prepared; - } - const prepared = { configurationId: `prepared-${requirements.identity.runId}`, - requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { - options: { codexArgs: fakeCodexArgs(stateDirectory, "--call-log", callsPath, "--record-process-start") }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "", - }), - }; - configurations.set(requirements.identity.runId, { requirements: structuredClone(requirements), prepared }); - return prepared; - }, - } : {}), - }; - const first = createCapabilityRunnerdCodexTransport(options); - let resumed: - ReturnType | undefined; - const providerPids = new Set(); - let commitSpy: ReturnType | undefined; - let commandObserverSpy: ReturnType | undefined; - let detached: Promise | undefined; - let cleanupProven = false; - const legacyProbeDirectories: string[] = []; - try { - const opened = await within( - first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }), - ); - const openedThread = opened.thread as { id: string; sessionId: string }; - const thread = { id: openedThread.id, sessionId: openedThread.sessionId }; - const firstThreadSnapshot = await within( - first.transport.request("thread/read", {}), - ); - const firstEvidence = first.evidence(); - const cleanRunnerStateBytes = await readFile( - join(stateDirectory, "runner/runner-state.json"), - ); - const runnerProcessStartedAt = - process.platform === "darwin" - ? new Date( - execFileSync( - "ps", - ["-o", "lstart=", "-p", String(handles[0]!.child.pid)], - { encoding: "utf8", timeout: 1_500 }, - ).trim(), - ).toISOString() - : null; - if ((first.evidence().codexPid ?? 0) > 0) - providerPids.add(first.evidence().codexPid!); - const core = cores[0]!; - const oldIdentity = structuredClone(core.store.state.identity); - const nextRunId = randomUUID(); - const desired = { - ...oldIdentity, - runId: nextRunId, - turnId: `turn-${nextRunId}`, - itemId: `item-${nextRunId}`, - }; - if ( - recoveryFault === "attach-capability" || - recoveryFault === "attach-wait" - ) { - if (recoveryFault === "attach-wait") { - const getCommand = core.getCommand.bind(core); - commandObserverSpy = vi - .spyOn(core, "getCommand") - .mockImplementation((id) => { - if ( - core.store.state.commands.find( - (entry) => entry.commandId === id, - )?.type === "run.attach" - ) { - throw new Error("fixture result observer unavailable"); - } - return getCommand(id); - }); - } - const bootstrapCount = core.store.state.freshBootstraps; - await expect( - first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }), - ).rejects.toThrow( - recoveryFault === "attach-capability" - ? "capability is required" - : "result observer unavailable", - ); - expect(routeCalls).toEqual([oldIdentity.runId, desired.runId]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${oldIdentity.runId}`, - ]); - expect(core.store.state.freshBootstraps).toBe(bootstrapCount); - expect(handles).toHaveLength(1); - if (recoveryFault === "attach-capability") - expect( - core.store.state.commands.some( - (entry) => entry.type === "run.attach", - ), - ).toBe(false); - return; - } - const commit = core.store.commit.bind(core.store); - let lossObserved = false; - let signalLoss!: () => void; - const loss = new Promise((resolveLoss) => { - signalLoss = resolveLoss; - }); - commitSpy = vi - .spyOn(core.store, "commit") - .mockImplementation((candidate) => { - // The selected process-crash boundary stays unavailable until the - // exact owned runner is joined. A later replay must not silently - // settle this fixture before its retained pair is exported. - if (lossObserved) - throw new Error( - "fixture controller persistence unavailable after loss", - ); - const phase = candidate.warmTransition?.phase; - const target = - lossPoint === "after-activation" ? "activated" : "prepared"; - const atBoundary = lossPoint.includes("confirmation") - ? candidate.completedWarmTransition !== undefined && - candidate.warmTransition === undefined - : phase === target; - if (!lossObserved && atBoundary) { - lossObserved = true; - if ( - lossPoint !== "before-result" && - lossPoint !== "before-confirmation" - ) - commit(candidate); - // A dead controller cannot accept a reconnect through the second - // route installed for the in-flight authority rotation. Detaching - // only the current route leaves that overlapping fixture listener - // alive; completed-receipt replay needs no further store commit. - for (const [path, entry] of routes) { - if (entry.core === core) routes.delete(path); - } - core.disconnectActiveRunner(); - detached = first.transport.detachControllerForRestart!(); - signalLoss(); - // A committed confirmation is still before its ACK. Returning from - // this hook lets the current frame handler send that ACK even after - // close begins, collapsing the intended crash window on fast peers. - throw new Error("fixture interrupted exact result commit"); - } - commit(candidate); - }); - const attachment = first.transport.attachRun!({ - runId: desired.runId, - turnId: desired.turnId, - itemId: desired.itemId, - }); - void attachment.catch(() => undefined); - await within(loss); - await within(detached!); - expect(lossObserved).toBe(true); - const runnerPath = join(stateDirectory, "runner/runner-state.json"); - const runner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(runner.warmTransition.phase).toBe( - lossPoint === "after-activation" || lossPoint.includes("confirmation") - ? "activating" - : "prepared", - ); - const persisted = JSON.parse(await readFile(core.store.path, "utf8")); - expect( - persisted.commands.find( - (entry: { type: string }) => entry.type === "run.attach", - )?.status, - ).toBe( - lossPoint === "before-result" - ? "pending" - : lossPoint === "after-result" - ? "completed" - : undefined, - ); - // Stop only this fixture's exact owned process handles. No stored receipt - // or PID absence is used as authority to terminate an unknown owner. - await durableControlPlane - .waitForProcess(handles[0]!, 100) - .catch(() => undefined); - await within(handles[0]!.completion); - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - await within(attachment.catch(() => undefined)); - const joinedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - const joinedCore = JSON.parse(await readFile(core.store.path, "utf8")); - expect(joinedRunner.warmTransition).toEqual(runner.warmTransition); - expect(joinedCore.schema).toBe(persisted.schema); - expect(joinedCore.warmTransition).toEqual(persisted.warmTransition); - expect(joinedCore.completedWarmTransition).toEqual( - persisted.completedWarmTransition, - ); - commitSpy.mockRestore(); - const fixtureOutput = - process.env.PAPERCLIP_ATTACH_TRANSITION_FIXTURE_DIRECTORY; - if (fixtureOutput && recoveryFault === "none") { - const retainedArtifact = join(fixtureOutput, "paperclip-runnerd"); - await cp(options.runnerBinary, retainedArtifact, { force: false }); - expect( - createHash("sha256") - .update(await readFile(retainedArtifact)) - .digest("hex"), - ).toBe( - createHash("sha256") - .update(await readFile(options.runnerBinary)) - .digest("hex"), - ); - const retained = join( - fixtureOutput, - `${lossPoint}-${routed ? "routed" : "local"}`, - ); - await cp(stateDirectory, retained, { - recursive: true, - errorOnExist: true, - force: false, - }); - await writeFile( - join(retained, "transition-fixture-metadata.json"), - JSON.stringify( - { - schema: "paperclip.test.warm-transition-fixture.v1", - oldIdentity, - newIdentity: desired, - lossPoint, - routed, - runner: { - pid: handles[0]!.child.pid, - processGroupId: handles[0]!.processGroupId, - startedAt: runnerProcessStartedAt, - spawnObservedAt: handles[0]!.startedAt, - completion: await handles[0]!.completion, - processAbsent: dead(handles[0]!.child.pid!), - groupAbsent: dead(-handles[0]!.processGroupId!), - }, - providers: [...providerPids].map((pid) => ({ - pid, - processGroupId: pid, - startedAt: firstEvidence.providerProcessStartedAt, - processAbsent: dead(pid), - groupAbsent: dead(-pid), - })), - artifact: { - path: retainedArtifact, - version: runner.warmTransition.receipt.runnerVersion, - digest: runner.warmTransition.receipt.runnerDigest, - }, - thread, - firstThreadSnapshot, - firstEvidence, - }, - null, - 2, - ), - { mode: 0o600 }, - ); - } - routes.clear(); - const routeCallCount = routeCalls.length; - const beforeCalls = (await readFile(callsPath, "utf8")) - .trim() - .split(/\r?\n/); - expect( - beforeCalls.filter((call) => call === "process-start"), - ).toHaveLength(1); - if (recoveryFault === "legacy-parser") { - const legacyBinary = - process.env.PAPERCLIP_ATTACH_TRANSITION_LEGACY_RUNNER!; - const legacyDigest = `sha256:${createHash("sha256") - .update(await readFile(legacyBinary)) - .digest("hex")}`; - for (const [mode, bytes] of [ - ["pending", await readFile(runnerPath)], - ["clean", cleanRunnerStateBytes], - ] as const) { - const probe = await mkdtemp( - join(tmpdir(), "runnerd-legacy-schema-probe-"), - ); - legacyProbeDirectories.push(probe); - await writeFile(join(probe, "runner-state.json"), bytes, { - mode: 0o600, - }); - const handle = durableControlPlane.spawnRunner({ - connection: runner.warmTransition.receipt.connection, - stateDirectory: probe, - identity: oldIdentity, - runnerBinaryPath: legacyBinary, - runnerVersion: "0.3.0", - runnerDigest: legacyDigest, - ticket: "bootstrap_legacy_parser_probe", - maxOutboxBytes: runner.maxOutboxBytes, - p0ReserveBytes: runner.p0ReserveBytes, - maxRuntimeMs: 200, - reconnectGraceMs: 200, - }); - const result = await within(handle.completion, 5_000); - expect(result.code).not.toBe(0); - if (mode === "pending") { - expect(result.stderr).toContain( - "durable state binding does not match", - ); - expect(await readFile(join(probe, "runner-state.json"))).toEqual( - bytes, - ); - } else { - expect(result.stderr).not.toContain( - "durable state binding does not match", - ); - expect( - JSON.parse( - await readFile(join(probe, "runner-state.json"), "utf8"), - ).nextSourceSeq, - ).toBeGreaterThan(JSON.parse(bytes.toString("utf8")).nextSourceSeq); - } - } - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - return; - } - recovering = true; - if (recoveryFault === "malformed-core") - await writeFile(core.store.path, "{", { mode: 0o600 }); - const beforeRecoveryBytes = await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ); - const beforeRecoveryTree = - recoveryFault === "none" ? null : await fingerprintTree(stateDirectory); - const authorizationStages: string[] = []; - let releaseHeldAuthorization: (() => void) | undefined; - let markHeldAuthorizationEntered!: () => void; - const heldAuthorizationEntered = new Promise((resolve) => { - markHeldAuthorizationEntered = resolve; - }); - const authorizationFault = recoveryFault.startsWith("authorize_"); - const failedAuthorizationStage = recoveryFault.slice("authorize_".length); - const beforeProviderBytes = await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ); - resumed = createCapabilityRunnerdCodexTransport({ - ...options, - ...(externalArtifact ? { - runnerBinary: "/controller-missing-runner", - runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - } : {}), - ...(ordinaryFollowup - ? { - authorizeWarmTransitionRecovery: async () => { - if (!recoveryClaimCurrent) - throw new Error( - "fixture old recovery claim is no longer current", - ); - }, - onWarmTransitionRecoveryCompleted: (completion: { - transitionId: string; - }) => { - expect(completion.transitionId).toBe( - runner.warmTransition.receipt.transitionId, - ); - const completedSnapshot = cores[1]!.store.state.commands - .filter((command) => command.type === "session.snapshot") - .at(-1)!; - expect(completedSnapshot.status).toBe("completed"); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - completionSnapshotIds.push(completedSnapshot.commandId); - if ( - ordinaryFollowup === "callback_failure" && - completionSnapshotIds.length === 1 - ) { - throw new Error( - "fixture recovery completion callback failed", - ); - } - if ( - ordinaryFollowup === "callback_async_failure" && - completionSnapshotIds.length === 1 - ) { - return new Promise((_resolve, reject) => { - rejectHeldCompletion = reject; - markHeldCompletionEntered(); - }); - } - recoveryFenceActive = false; - }, - } - : {}), - ...(authorizationFault - ? { - authorizeWarmTransitionRecovery: async (stage: string) => { - authorizationStages.push(stage); - if ( - stage === "before_bootstrap" && - failedAuthorizationStage === "before_bootstrap" - ) { - await new Promise((resolveGate) => { - releaseHeldAuthorization = resolveGate; - markHeldAuthorizationEntered(); - }); - } - if (stage === failedAuthorizationStage) - throw new Error("fixture recovery authority revoked"); - }, - } - : {}), - ...(recoveryFault === "missing-capability" - ? { warmTransitionRegistrationMode: undefined } - : {}), - ...(recoveryFault !== "none" - ? { - environment: { - CODEX_API_KEY: "synthetic-refused-route-credential", - }, - } - : {}), - prpIdentity: desired, - resumeProviderSession: { - driverSessionId: thread.id, - providerSessionId: thread.sessionId, - }, - }); - if (recoveryFault !== "none") { - const recoveryRequest = within( - resumed.transport.request("thread/read", {}), - ); - void recoveryRequest.catch(() => undefined); - if ( - authorizationFault && - failedAuthorizationStage === "before_bootstrap" - ) { - await within(Promise.race([ - heldAuthorizationEntered, - recoveryRequest.then(() => { - throw new Error("warm restart authorization callback was not entered"); - }), - ])); - expect(releaseHeldAuthorization).toBeTypeOf("function"); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - try { - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not queue before bootstrap" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.resolveRuntimeRequest!({ - requestId: "must-not-resolve", - turnId: desired.turnId, - resolution: { action: "cancel" }, - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - expect(handles).toHaveLength(1); - } finally { - releaseHeldAuthorization!(); - } - } - const failedRecovery = expect(recoveryRequest).rejects; - if (authorizationFault) { - await failedRecovery.toThrow( - failedAuthorizationStage === "before_authentication" - ? "native_runner_warm_transition_recovery_pending" - : "fixture recovery authority revoked", - ); - const expectedStages = [ - "before_bootstrap", - "before_spawn", - "before_authentication", - ]; - expect([...new Set(authorizationStages)]).toEqual( - expectedStages.slice( - 0, - expectedStages.indexOf(failedAuthorizationStage) + 1, - ), - ); - expect(handles).toHaveLength( - failedAuthorizationStage === "before_authentication" ? 2 : 1, - ); - expect( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - ), - ).toEqual(beforeProviderBytes); - const refusedRunner = JSON.parse(await readFile(runnerPath, "utf8")); - expect(refusedRunner.warmTransition).toEqual(runner.warmTransition); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - if (failedAuthorizationStage !== "before_authentication") - expect(await readFile(runnerPath)).toEqual(beforeRecoveryBytes[1]); - if (failedAuthorizationStage === "before_bootstrap") - expect(await readFile(core.store.path)).toEqual( - beforeRecoveryBytes[0], - ); - await within(resumed.transport.close()).catch(() => undefined); - expect([...routes.keys()]).toEqual([]); - return; - } - if (recoveryFault === "malformed-core") await failedRecovery.toThrow(); - else - await failedRecovery.toThrow( - recoveryFault === "missing-capability" - ? "requires_exact_owned_endpoint" - : "registered_endpoint_mismatch", - ); - expect(handles).toHaveLength(1); - expect( - await Promise.all( - [core.store.path, runnerPath].map((path) => readFile(path)), - ), - ).toEqual(beforeRecoveryBytes); - expect(await fingerprintTree(stateDirectory)).toEqual( - beforeRecoveryTree, - ); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual(beforeCalls); - expect([...routes.keys()]).toEqual([]); - return; - } - if ( - snapshotFault || - ordinaryFollowup === "callback_failure" || - ordinaryFollowup === "callback_async_failure" - ) { - const firstRead = within(resumed.transport.request("thread/read", {})); - void firstRead.catch(() => undefined); - if (ordinaryFollowup === "callback_async_failure") { - await within(Promise.race([ - heldCompletionEntered, - firstRead.then(() => { - throw new Error("warm restart completion callback was not entered"); - }), - ])); - expect(rejectHeldCompletion).toBeTypeOf("function"); - try { - expect(recoveryFenceActive).toBe(true); - const queuedBefore = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must not pass held completion" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map( - (command) => command.commandId, - ), - ).toEqual(queuedBefore); - } finally { - rejectHeldCompletion!( - new Error("fixture recovery completion callback failed"), - ); - } - } - await expect(firstRead).rejects.toThrow( - snapshotFault - ? ordinaryFollowup === "rejected_snapshot" - ? "snapshot observation rejected" - : "completion_unproven" - : "recovery completion callback failed", - ); - expect(recoveryFenceActive).toBe(true); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 0 : 1); - const beforeDeniedWork = cores[1]!.store.state.commands.map( - (command) => command.commandId, - ); - const beforeDeniedProviderCalls = await readFile(callsPath, "utf8"); - await expect( - resumed.transport.request("turn/start", { - input: [{ text: "must remain fenced" }], - }), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.request("thread/resume", {}), - ).rejects.toThrow("warm_transition_completion_pending"); - await expect( - resumed.transport.attachRun!({ - runId: "must-not-attach", - turnId: "must-not-attach", - itemId: "must-not-attach", - }), - ).rejects.toThrow("warm_transition_completion_pending"); - expect( - cores[1]!.store.state.commands.map((command) => command.commandId), - ).toEqual(beforeDeniedWork); - expect(await readFile(callsPath, "utf8")).toBe( - beforeDeniedProviderCalls, - ); - snapshotObserverSpy?.mockRestore(); - const providerCallsBeforeRetry = await readFile(callsPath, "utf8"); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - expect(recoveryFenceActive).toBe(false); - expect(completionSnapshotIds).toHaveLength(snapshotFault ? 1 : 2); - expect(new Set(completionSnapshotIds).size).toBe( - completionSnapshotIds.length, - ); - expect(await readFile(callsPath, "utf8")).toBe( - providerCallsBeforeRetry, - ); - return; - } - const read = await within( - resumed.transport.request("thread/read", {}), - 10_000, - ); - if ((resumed.evidence().codexPid ?? 0) > 0) - providerPids.add(resumed.evidence().codexPid!); - expect(read.thread).toMatchObject(thread); - expect(cores).toHaveLength(2); - expect(cores[1]!.store.state.identity).toEqual(desired); - expect(cores[1]!.store.state.warmTransition).toBeUndefined(); - expect( - cores[1]!.store.state.completedWarmTransition?.receipt.transitionId, - ).toBe(runner.warmTransition.receipt.transitionId); - if (routed) { - expect(routeCalls.slice(routeCallCount)).toEqual([ - oldIdentity.runId, - desired.runId, - ]); - expect([...routes.keys()]).toEqual([ - `/api/runner/v1/connect/${desired.runId}`, - ]); - } - const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); - expect(calls.filter((call) => call === "process-start")).toHaveLength(2); - expect(calls.filter((call) => call === "thread/start")).toHaveLength(1); - expect(calls.filter((call) => call === "thread/resume")).toHaveLength(1); - expect(calls.filter((call) => call === "turn/start")).toHaveLength(0); - if (ordinaryFollowup) { - recoveryClaimCurrent = false; - const resumedCore = cores[1]!; - resumedCore.disconnectActiveRunner(); - await vi.waitFor( - () => expect(resumedCore.activeRunnerConnectionCount()).toBe(1), - { timeout: 2_000 }, - ); - expect(recoveryFenceActive).toBe(false); - expect( - (await within(resumed.transport.request("thread/read", {}))).thread, - ).toMatchObject(thread); - const thirdRunId = randomUUID(); - await within( - resumed.transport.attachRun!({ - runId: thirdRunId, - turnId: `turn-${thirdRunId}`, - itemId: `item-${thirdRunId}`, - }), - ); - expect(resumedCore.store.state.identity.runId).toBe(thirdRunId); - expect( - (await readFile(callsPath, "utf8")).trim().split(/\r?\n/), - ).toEqual([...calls, "thread/goal/get"]); - } - } finally { - snapshotObserverSpy?.mockRestore(); - commitSpy?.mockRestore(); - commandObserverSpy?.mockRestore(); - try { - await within(resumed?.transport.close() ?? Promise.resolve()).catch( - () => undefined, - ); - await within(first.transport.detachControllerForRestart!()).catch( - () => undefined, - ); - for (const handle of handles) { - await durableControlPlane - .waitForProcess(handle, 250) - .catch(() => undefined); - await within(handle.completion); - if (handle.processGroupId && !dead(-handle.processGroupId)) - process.kill(-handle.processGroupId, "SIGKILL"); - await vi.waitFor(() => { - expect(handle.child.pid && dead(handle.child.pid)).toBe(true); - expect(handle.processGroupId && dead(-handle.processGroupId)).toBe( - true, - ); - }); - } - for (const pid of providerPids) { - await stopOwnedProvider(pid); - } - cleanupProven = true; - } finally { - for (const core of cores) await core.stop().catch(() => undefined); - if (routed) - await new Promise((resolveClose) => - routeServer.close(() => resolveClose()), - ); - coreSpy.mockRestore(); - launchSpy.mockRestore(); - if (cleanupProven) - await rm(stateDirectory, { recursive: true, force: true }); - if (cleanupProven) - for (const directory of legacyProbeDirectories) - await rm(directory, { recursive: true, force: true }); - } - } - }, - 30_000, -); - -it("rotates PRP authority in place for a warm cross-run attachment", async () => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-warm-attach-")); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - const within = async (label: string, promise: Promise) => - await Promise.race([ - promise, - new Promise((_, reject) => - setTimeout(() => reject(new Error(`${label} timeout`)), 5_000), - ), - ]); - try { - await within("initialize", bundle.transport.request("initialize", {})); - await within( - "thread start", - bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ], - completionContract: { - revision: "sha256:warm-three-turn-contract", - criterionIds: ["objective"], - }, - }), - ); - const runnerPid = bundle.evidence().runnerPid; - const providerPid = bundle.evidence().codexPid; - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - const waitForCompletion = async (label: string) => { - const deadline = Date.now() + 5_000; - while (Date.now() < deadline) { - const next = await Promise.race([ - notifications.next(), - new Promise((_, reject) => - setTimeout( - () => reject(new Error(`${label} notification timeout`)), - 1_000, - ), - ), - ]); - if (next.value?.method === "turn/completed") return; - } - throw new Error(`${label} completion timeout`); - }; - await within( - "first turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "first run" }], - }), - ); - await waitForCompletion("first run"); - - await within( - "warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-second", - turnId: "turn-warm-second", - itemId: "item-warm-second", - }), - ); - await within( - "second turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "second run" }], - }), - ); - await waitForCompletion("second run"); - - await within( - "second warm attach", - bundle.transport.attachRun!({ - runId: "run-warm-third", - turnId: "turn-warm-third", - itemId: "item-warm-third", - }), - ); - await within( - "third turn start", - bundle.transport.request("turn/start", { - input: [{ type: "text", text: "third run" }], - }), - ); - await waitForCompletion("third run"); - - expect(bundle.evidence()).toMatchObject({ - runnerPid, - codexPid: providerPid, - runnerExited: false, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("recovers an active second warm business run after deleting its controller context assets", async () => { - const root = await mkdtemp(join(tmpdir(), "runnerd-warm-second-active-")); - const firstRoot = join(root, "first-controller"), nextRoot = join(root, "next-controller"); - const assetRoot = join(root, "retired-controller-assets"); - const skillRoot = join(assetRoot, "skill"), instructionRoot = join(assetRoot, "instructions"); - await mkdir(skillRoot, { recursive: true }); await mkdir(instructionRoot, { recursive: true }); - await writeFile(join(skillRoot, "SKILL.md"), "# Pinned skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Pinned instructions\n"); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null, generation = 0; - let checkpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - const registered = new WeakSet(); - server.on("upgrade", (request, socket, head) => authority - ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); - await new Promise(done => server.listen(0, "127.0.0.1", done)); - const address = server.address(); - if (!address || typeof address === "string") throw new Error("Missing owned Runner route"); - const registration = async (core: DurablePrpControlPlane) => { - if (!registered.has(core)) { - core.setCheckpointPersistence(async value => { checkpoint = value; }); - registered.add(core); - } - authority = core; const current = ++generation; - return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: async () => { if (generation === current) authority = null; } }; - }; - const initialIdentity = { runnerInstanceId: "runner-warm-second-active", environmentLeaseId: "lease-warm-second-active", - runId: "run-warm-first-healthy", normalizedSessionId: "session-warm-second-active", turnId: "turn-first", itemId: "item-first" }; - const options = { runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(root, "--emit-tool-call", "--durable-tool-ids"), stateDirectory: firstRoot, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - prpIdentity: initialIdentity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, controlPlaneRegistration: registration }; - const first = createCapabilityRunnerdCodexTransport(options); - const within = async (label: string, promise: Promise) => { - let timer: ReturnType | undefined; - try { return await Promise.race([promise, new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`warm active recovery stage timed out: ${label}`)), 5_000); })]); } - finally { clearTimeout(timer); } - }; - const waitComplete = async (transport: typeof first.transport) => { - for await (const notification of transport.notifications()) if (notification.method === "turn/completed") return; - }; - let calls = 0, runnerPid: number | null = null, providerPid: number | null = null; - let recovered: ReturnType | undefined; - const result = { ok: true, task: { id: "task-1" } }; - first.transport.setServerRequestHandler(async () => ++calls === 1 - ? { success: true, contentItems: [{ type: "inputText", text: JSON.stringify(result) }] } - : new Promise>(() => undefined)); - try { - const opened = await within("first thread/start", first.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [{ name: "get_task_context", description: "Read current task", - inputSchema: { type: "object", properties: {}, additionalProperties: false } }] })); - runnerPid = first.evidence().runnerPid; - providerPid = first.evidence().codexPid; - expect(providerPid).toEqual(expect.any(Number)); - const providerStart = execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim(); - const startedAt = first.transport.processInfo().startedAt; - const firstCompleted = waitComplete(first.transport); - await within("first turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "first healthy business run" }] })); - await within("healthy first turn completion", firstCompleted); - const attachment = { runId: "run-warm-second-active", turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; - await within("warm second business attachment", first.transport.attachRun!(attachment)); - const active = await within("second active turn/start", first.transport.request("turn/start", { input: [{ type: "text", text: "second business run held at tool" }] })); - await vi.waitFor(() => expect(calls).toBe(2), { timeout: 5_000 }); - const activeCore = authority!; - const savedIdentity = { ...activeCore.store.state.identity }; - expect(savedIdentity).toEqual({ ...initialIdentity, ...attachment }); - await activeCore.flushCheckpoint(); - await within("controller detach", first.detachControllerForRestart()); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - await rm(assetRoot, { recursive: true, force: true }); - expect(existsSync(assetRoot)).toBe(false); - checkpoint!.restoreSessionDirectory(nextRoot); - const noSpawn = vi.fn(() => { throw new Error("Duplicate Runner launch"); }); - const openedThread = opened.thread as Record; - const activeTurn = active.turn as Record; - expect(openedThread.id).toEqual(expect.any(String)); - expect(activeTurn.id).toEqual(expect.any(String)); - const shared = { ...options, stateDirectory: nextRoot, - resumeProviderSession: { driverSessionId: String(openedThread.id), providerSessionId: String(openedThread.sessionId) }, - resumeActiveTurnId: String(activeTurn.id), runnerArtifact: readRunnerdArtifactBinding(options.runnerBinary), - runnerStateDirectory: join(firstRoot, "runner"), readRunnerState: async () => JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8")), - prepareExternalRunnerState: async () => {}, archiveExternalRunnerState: async () => { throw new Error("Active authority must not rotate"); }, - runnerProcessLauncher: noSpawn, adoptExistingRunner: { pid: runnerPid!, processGroupId: runnerPid, startedAt, - isAlive: () => { try { process.kill(runnerPid!, 0); return true; } catch { return false; } }, - signal: (signal: NodeJS.Signals) => { try { process.kill(runnerPid!, signal); return true; } catch { return false; } } } }; - // The old server's synthetic pair must continue failing the strict Runner guard. - const wrong = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: { ...savedIdentity, - turnId: `turn-${attachment.runId}`, itemId: `item-${attachment.runId}` } }); - await expect(wrong.transport.request("thread/read", {})).rejects.toThrow("PRP recovery identity does not match the durable session binding"); - await wrong.detachControllerForRestart(); - // Leave the pending tool outcome unknown. Reattachment must authenticate - // and preserve it without fabricating a receipt or redispatching its handler. - recovered = createCapabilityRunnerdCodexTransport({ ...shared, prpIdentity: savedIdentity }); - await within("replacement thread/read", recovered.transport.request("thread/read", {})); - expect(recovered.evidence()).toMatchObject({ runnerPid, runnerExited: false }); - expect(authority!.activeRunnerConnectionCount()).toBe(1); - expect(() => process.kill(providerPid!, 0)).not.toThrow(); - expect(execFileSync("ps", ["-p", String(providerPid), "-o", "lstart="], { encoding: "utf8" }).trim()).toBe(providerStart); - expect(calls).toBe(2); - expect(authority!.store.state.commands.filter(command => command.type === "semantic_tool.result")).toHaveLength(0); - expect(recovered.transport.processInfo().startedAt).toBe(startedAt); - expect(authority!.store.state.identity).toEqual(savedIdentity); - expect(authority!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(noSpawn).not.toHaveBeenCalled(); - } catch (error) { - const saved = JSON.parse(await readFile(join(firstRoot, "runner/runner-state.json"), "utf8").catch(() => "{}")); - const stderr = await readFile(join(firstRoot, "diagnostics/runnerd.stderr.log"), "utf8").catch(() => ""); - throw new Error(`${String(error)}; owned protocol evidence ${JSON.stringify({ - transport: recovered?.evidence(), connections: authority?.activeRunnerConnectionCount(), - identity: authority?.store.state.identity, commands: authority?.store.state.commands.map(c => ({type:c.type,status:c.status})), - events: authority?.store.state.committedEvents.map(e => e.eventType).slice(-8), - runner: { lifecycle:saved.lifecycle, runId:saved.runId, turnId:saved.turnId, itemId:saved.itemId, - outbox:saved.outbox?.length }, stderr:stderr.slice(-1500), - })}`); - } finally { - await (recovered ? within("recovered cleanup", recovered.transport.close()).catch(() => undefined) : undefined); - await within("first cleanup", first.transport.close()).catch(() => undefined); - if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } - await (authority as DurablePrpControlPlane | null)?.stop(); - if (runnerPid) { try { process.kill(-runnerPid, "SIGKILL"); } catch {} } - server.closeAllConnections(); - if (server.listening) await new Promise(done => server.close(() => done())); - for (const controller of [firstRoot, nextRoot]) await releaseMaterializedNativeRuntimeSkills(join(controller, "codex-home", "skills")); - await rm(root, { recursive: true, force: true }); - } -}, 30_000); - -it("waits for a warm runner to re-authenticate before probing attachment readiness", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-reattach-before-probe-"), - ); - const server = createServer(); - const authorities = new Map(); - let blockFirstAuthorityReconnect = false; - let resolveRejectedReconnect!: () => void; - const rejectedReconnect = new Promise((resolvePromise) => { - resolveRejectedReconnect = resolvePromise; - }); - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - if (route === "/runner-1" && blockFirstAuthorityReconnect) { - resolveRejectedReconnect(); - socket.destroy(); - return; - } - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm reconnect test listener"); - } - let registrationCount = 0; - const diagnostics: string[] = []; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - runnerReconnectGraceMs: 5_000, - onDiagnostic: (message) => diagnostics.push(message), - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - release: () => { - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - const firstAuthority = authorities.get("/runner-1"); - if (!firstAuthority) throw new Error("Missing first warm authority"); - const priorSnapshotCount = firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - blockFirstAuthorityReconnect = true; - firstAuthority.disconnectActiveRunner(); - const attachment = bundle.transport.attachRun!({ - runId: "run-warm-after-reconnect", - turnId: "turn-warm-after-reconnect", - itemId: "item-warm-after-reconnect", - }); - await Promise.race([ - rejectedReconnect, - new Promise((_, reject) => - setTimeout( - () => reject(new Error("runner did not attempt to reconnect")), - 5_000, - ), - ), - ]); - - // No command may be queued while its sole authenticated consumer is - // absent. The generic 30-second command timeout used to turn this state - // into same-run recovery and replace the healthy warm runner process. - expect( - firstAuthority.store.state.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshotCount); - - blockFirstAuthorityReconnect = false; - await Promise.race([ - attachment, - new Promise((_, reject) => - setTimeout(() => reject(new Error("warm attachment timeout")), 10_000), - ), - ]); - expect(bundle.evidence()).toMatchObject({ - runnerPid, - runnerExited: false, - }); - expect(diagnostics).toContain( - "warm runner connection interrupted; waiting for re-authentication before authority rotation", - ); - expect(diagnostics).toContain( - "warm runner re-authenticated before authority rotation", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it("releases both PRP authorities when warm rotation activation fails", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-warm-attach-activation-failure-"), - ); - const server = createServer(); - const authorities = new Map(); - const released: string[] = []; - server.on("upgrade", (request, socket, head) => { - const route = request.url ?? ""; - const authority = authorities.get(route); - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, route, head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("Expected warm activation failure test listener"); - } - let registrationCount = 0; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, - controlPlaneRegistration: async (authority) => { - registrationCount += 1; - const route = `/runner-${registrationCount}`; - authorities.set(route, authority); - return { - connectUrl: `ws://127.0.0.1:${address.port}${route}`, - ...(registrationCount === 1 - ? {} - : { - activate: () => { - throw new Error("rotation activation failed"); - }, - }), - release: () => { - released.push(route); - if (authorities.get(route) === authority) authorities.delete(route); - }, - }; - }, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let runnerPid: number | null = null; - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: codexSemanticToolSpecs(), - }); - runnerPid = bundle.evidence().runnerPid; - - await expect( - bundle.transport.attachRun!({ - runId: "run-warm-activation-failure", - turnId: "turn-warm-activation-failure", - itemId: "item-warm-activation-failure", - }), - ).rejects.toThrow("rotation activation failed"); - expect(new Set(released)).toEqual(new Set(["/runner-1", "/runner-2"])); - expect(authorities.size).toBe(0); - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "rotation activation failed", - ); - } finally { - await bundle.transport.close().catch(() => undefined); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // A successful durable close already stopped the runner process group. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([ - { - binding: "runner instance", - priorRunnerInstanceId: "runner-other", - priorEnvironmentLeaseId: "lease-current", - }, - { - binding: "environment lease", - priorRunnerInstanceId: "runner-current", - priorEnvironmentLeaseId: "lease-other", - }, -])( - "quarantines a mismatched $binding instead of reusing its provider session", - async ({ priorRunnerInstanceId, priorEnvironmentLeaseId }) => { - const container = await mkdtemp( - join(tmpdir(), "runnerd-mismatched-authority-"), - ); - const stateDirectory = join(container, "state"); - await mkdir(join(stateDirectory, "control-plane"), { recursive: true }); - await writeFile( - join(stateDirectory, "control-plane", "control-plane-state.json"), - JSON.stringify({ - identity: { - runnerInstanceId: priorRunnerInstanceId, - environmentLeaseId: priorEnvironmentLeaseId, - runId: "run-old", - normalizedSessionId: "session-current", - turnId: "turn-old", - itemId: "item-old", - }, - }), - { mode: 0o600 }, - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - stateDirectory, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: { - runnerInstanceId: "runner-current", - environmentLeaseId: "lease-current", - runId: "run-new", - normalizedSessionId: "session-current", - turnId: "turn-new", - itemId: "item-new", - }, - }); - try { - await expect(bundle.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_state_quarantined", - ); - expect(await readdir(stateDirectory)).toEqual([]); - expect( - (await readdir(container)).some((entry) => - entry.startsWith("state.quarantine-"), - ), - ).toBe(true); - } finally { - await bundle.transport.close(); - await rm(container, { recursive: true, force: true }); - } - }, -); - -it.each([0, 193 * 1024 * 1024])( - "probes an exact-authority resume with %i extra journal bytes and confirms its live provider identity", - async (extraJournalBytes) => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-exact-authority-resume-"), - ); - const identity = { - runnerInstanceId: "runner-exact-resume", - environmentLeaseId: "lease-exact-resume", - runId: "run-exact-resume", - normalizedSessionId: "session-exact-resume", - turnId: "turn-exact-resume", - itemId: "item-exact-resume", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--durable-turn-ids"), - stateDirectory, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - prpIdentity: identity, - }; - const first = createCapabilityRunnerdCodexTransport(options); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let providerThread: { id: string; sessionId: string } | null = null; - try { - const opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const thread = opened.thread as Record; - providerThread = { - id: String(thread.id), - sessionId: String(thread.sessionId), - }; - } finally { - await first.transport.close(); - } - if (providerThread === null) { - throw new Error( - "exact-authority fixture did not return a provider thread", - ); - } - - const statePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - if (extraJournalBytes > 0) { - const padding = Buffer.alloc(1024 * 1024, 0x20); - const stateHandle = await open(statePath, "a"); - try { - for (let remaining = extraJournalBytes; remaining > 0;) { - const bytesToWrite = Math.min(remaining, padding.length); - await stateHandle.write(padding, 0, bytesToWrite); - remaining -= bytesToWrite; - } - } finally { - await stateHandle.close(); - } - expect((await stat(statePath)).size).toBeGreaterThan( - 64 * 1024 * 1024, - ); - } - const beforeResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ type: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect( - beforeResume.commands.some((command) => command.type === "run.attach"), - ).toBe(false); - const priorResumeEvents = beforeResume.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ).length; - const priorSnapshots = beforeResume.commands.filter( - (command) => command.type === "session.snapshot", - ).length; - - const resumed = createCapabilityRunnerdCodexTransport({ - ...options, - resumeProviderSession: { - driverSessionId: providerThread.id, - providerSessionId: providerThread.sessionId, - }, - }); - resumed.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await resumed.transport.request("thread/read", {}); - expect(read.thread).toMatchObject(providerThread); - const afterResume = JSON.parse(await readFile(statePath, "utf8")) as { - commands: Array<{ commandId: string; type: string; status: string }>; - committedEvents: Array<{ eventType: string }>; - }; - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - commandId: expect.stringMatching(/^command_resume_probe_/), - type: "runner.drain", - status: "completed", - }), - ); - expect(afterResume.commands).toContainEqual( - expect.objectContaining({ - type: "session.snapshot", - status: "completed", - }), - ); - expect( - afterResume.commands.filter( - (command) => command.type === "session.snapshot", - ), - ).toHaveLength(priorSnapshots + 2); - // The authenticated snapshot above proves the live provider identity. - // Control-first dispatch may deliver that command before the independent - // session event is ingested. Still require exactly one durable event; - // don't mistake an immediate file read for an event-delivery barrier. - await vi.waitFor( - async () => { - const delivered = JSON.parse(await readFile(statePath, "utf8")) as { - committedEvents: Array<{ eventType: string }>; - }; - expect( - delivered.committedEvents.filter( - (event) => event.eventType === "session.resumed", - ), - ).toHaveLength(priorResumeEvents + 1); - }, - { timeout: 3_000, interval: 25 }, - ); - } finally { - await resumed.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } - }, - 30_000, -); - -it("still fails closed when a real close grace period cannot fit a durable suspension round trip", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-close-grace-too-small-"), - ); - const identity = { - runnerInstanceId: "runner-close-grace-too-small", - environmentLeaseId: "lease-close-grace-too-small", - runId: "run-close-grace-too-small", - normalizedSessionId: "session-close-grace-too-small", - turnId: "turn-close-grace-too-small", - itemId: "item-close-grace-too-small", - }; - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory), - stateDirectory, - // No real durable command round trip can complete this fast. A wider - // budget for the provider-drain proof must not turn this barrier into - // one that always passes; it still needs the actual proof to arrive. - closeGraceMs: 1, - lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, - prpIdentity: identity, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - const closed = bundle.transport.close(); - await expect(closed).rejects.toMatchObject({ - code: "native_session_close_unrecoverable", - semanticToolsSettled: true, - }); - // Retirement preserves the original rejected promise and its proof. - expect(bundle.transport.close()).toBe(closed); - } finally { - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -it.each([false, true, "restart-before-bootstrap", "unknown-semantic", "accounting-rejected", "credentials-cleaned"] as const)("cold-restores a suspended provider session under its durable run binding (pending tail: %s)", async (pendingTail) => { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-cold-attach-")); - const tracePath = join(stateDirectory, "provider-trace.ndjson"); - const skillRoot = join(stateDirectory, "runtime-skill"); - const instructionRoot = join(stateDirectory, "runtime-instructions"); - await Promise.all([mkdir(skillRoot), mkdir(instructionRoot)]); - await writeFile(join(skillRoot, "SKILL.md"), "# Assigned runtime skill\n"); - await writeFile(join(instructionRoot, "AGENTS.md"), "Runtime instructions\n"); - const credentialsCleaned = pendingTail === "credentials-cleaned"; - const sourceCodexHome = join(stateDirectory, "selected-account-first"); - if (credentialsCleaned) { - await mkdir(sourceCodexHome); - await writeFile(join(sourceCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - const baseIdentity = { - runnerInstanceId: "runner-cold-attach", - environmentLeaseId: "lease-cold-attach", - runId: "run-cold-first", - normalizedSessionId: "session-cold-attach", - turnId: "turn-cold-first", - itemId: "item-cold-first", - }; - const options = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs( - stateDirectory, - "--include-skill-instructions", - "--durable-turn-ids", - ...(credentialsCleaned ? ["--require-codex-home-auth"] : []), - "-c", - 'shell_environment_policy.set={PATH="/run/A"}', - ), - stateDirectory, - ...(credentialsCleaned ? { sourceCodexHome } : {}), - environment: { - PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-A-capability", - PAPERCLIP_PROVIDER_TRACE_PATH: tracePath, - PAPERCLIP_PROVIDER_TRACE_MAX_BYTES: String(64 * 1024 * 1024), - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_NAME: "assigned", - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/first", - PAPERCLIP_NATIVE_MCP_TOKEN: "first-assigned-gateway-credential", - } : {}), - }, - lifecyclePolicy: { mode: "per_turn" as const, idleTimeoutMs: null }, - runtimeContext: assignedRuntimeContext(skillRoot, instructionRoot), - }; - const dynamicTools = [ - { - name: "get_task_context", - description: "Read the active task.", - inputSchema: { - type: "object", - properties: {}, - additionalProperties: false, - }, - }, - ]; - const first = createCapabilityRunnerdCodexTransport({ - ...options, - prpIdentity: baseIdentity, - }); - first.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - let firstProviderThread: { id: string; sessionId: string } | null = null; - try { - const started = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [...dynamicTools, ...codexSemanticToolSpecs()], - completionContract: { - revision: "contract-first", - criterionIds: ["criterion-first"], - }, - }); - const startedThread = started.thread as Record; - firstProviderThread = { - id: String(startedThread.id), - sessionId: String(startedThread.sessionId), - }; - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - await first.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned first process" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/completed") break; - } - } finally { - await first.transport.close(); - } - if (!firstProviderThread) { - throw new Error("cold attach fixture did not return a provider thread"); - } - const successorCodexHome = join(stateDirectory, "selected-account-next"); - if (credentialsCleaned) { - await rm(join(stateDirectory, "codex-home", "auth.json")); - await rm(sourceCodexHome, { recursive: true }); - await mkdir(successorCodexHome); - await writeFile(join(successorCodexHome, "auth.json"), JSON.stringify({ OPENAI_API_KEY: "selected-account-fixture" }), { mode: 0o600 }); - } - - // Model a retained provider-owned suffix: rotation must deliver and durably - // accept it before attempting run.attach, rather than dropping or ACKing it. - const retainedDelta = "cold-recovery-retained-tail"; - if (pendingTail) { - const providerFile = join(stateDirectory, "runner", "codex-provider-state.json"); - const provider = JSON.parse(await readFile(providerFile, "utf8")); - expect(provider.pendingEvents).toEqual([]); - provider.pendingEvents.push({ - executorEventId: `codex_provider_${String(provider.nextProviderEventSeq++).padStart(16, "0")}`, - eventType: pendingTail === "unknown-semantic" ? "semantic_tool.input" : "usage.reported", - priority: "p0", - payload: pendingTail === "unknown-semantic" ? { semantic_tool: { - schema: "paperclip.prp.semantic_tool.v1", schemaVersion: 1, phase: "input", - callId: "unknown-source-call", operationId: "get_task_context", input: {}, idempotencyKey: null, - correlation: { runId: baseIdentity.runId, normalizedSessionId: baseIdentity.normalizedSessionId, - turnId: baseIdentity.turnId, itemId: baseIdentity.itemId }, - content: { digest: `sha256:${createHash("sha256").update("{}").digest("hex")}`, redactionDisposition: "digest_only", references: [] }, - } } : { provider: "codex", kind: "usage", delta: retainedDelta, - providerTurnId: provider.completed_provider_turn_id, runDelta: { inputTokens: 20, outputTokens: 2 }, - runDeltaAvailable: true, runDeltaBasis: "run" }, - }); - await writeFile(providerFile, JSON.stringify(provider)); - } - const retainedEvents: PrpEvent[] = []; - let releaseRetained!: () => void, retainedEntered!: () => void; - const retainedAccepted = new Promise(done => { releaseRetained = done; }); - const retainedAdmission = new Promise(done => { retainedEntered = done; }); - const secondIdentity = { - ...baseIdentity, - runId: "run-cold-second", - turnId: "turn-cold-second", - itemId: "item-cold-second", - }; - let stagedCheckpoint: durableControlPlane.DurablePrpCheckpoint | undefined; - let rejectStagedCommit = pendingTail === "restart-before-bootstrap"; - const rotatedOptions = { - ...options, - ...(credentialsCleaned ? { sourceCodexHome: successorCodexHome } : {}), - onStagedRunAttachment: pendingTail === "restart-before-bootstrap" ? async (authority: DurablePrpControlPlane) => { - authority.setCheckpointPersistence(async checkpoint => { - stagedCheckpoint = checkpoint; - if (rejectStagedCommit) throw new Error("fixture lost controller after durable intent"); - }); - } : undefined, - onRecoveryAccountingEvent: pendingTail ? async (event: PrpEvent) => { - if (event.payload.delta === retainedDelta) { - retainedEvents.push(structuredClone(event)); - retainedEntered(); - if (pendingTail === "accounting-rejected") throw new Error("source accounting receipt unavailable"); - await retainedAccepted; - } - } : undefined, - environment: { ...options.environment, PAPERCLIP_GITHUB_BROKER_TOKEN: "test-run-B-capability", - ...(credentialsCleaned ? { - PAPERCLIP_NATIVE_MCP_URL: "https://paperclip.example/mcp/current", - PAPERCLIP_NATIVE_MCP_TOKEN: "current-assigned-gateway-credential", - } : {}), - }, - codexArgs: options.codexArgs.map((arg) => arg.replace('/run/A', '/run/B')), - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-second", - criterionIds: ["criterion-second"], - }, - prpIdentity: secondIdentity, - }; - let rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - if (pendingTail === "restart-before-bootstrap") { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("fixture lost controller after durable intent"); - expect(rotated.evidence().runnerPid).toBeNull(); - expect(stagedCheckpoint?.runId).toBe(secondIdentity.runId); - expect([...stagedCheckpoint!.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value).type)).not.toContain("run.attach"); - await rotated.detachControllerForRestart().catch(() => undefined); - rejectStagedCommit = false; - await rm(join(stateDirectory, "control-plane"), { recursive: true, force: true }); - stagedCheckpoint!.restoreSessionDirectory(stateDirectory); - rotated = createCapabilityRunnerdCodexTransport(rotatedOptions); - } - const currentHandler = vi.fn(async () => ({ success: true, contentItems: [] })); - rotated.transport.setServerRequestHandler(currentHandler); - if (pendingTail === "accounting-rejected") { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - try { - await Promise.race([retainedAdmission, reading]); - const source = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(source.identity).toEqual(baseIdentity); - expect(source.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(source.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - expect(currentHandler).not.toHaveBeenCalled(); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await expect(reading).rejects.toThrow(); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - if (pendingTail === "unknown-semantic") { - try { - await expect(rotated.transport.request("thread/read", {})).rejects.toThrow("native_runner_source_semantic_reconciliation_required"); - expect(currentHandler).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retained.identity).toEqual(baseIdentity); - expect(retained.stagedRunAttachment.targetIdentity).toEqual(secondIdentity); - expect(retained.commands.some((command: { type: string }) => ["run.attach", "semantic_tool.result"].includes(command.type))).toBe(false); - expect(retained.committedEvents.some((event: PrpEvent) => event.eventType === "semantic_tool.input")).toBe(true); - } finally { - await expect(rotated.transport.close()).rejects.toMatchObject({ code: "native_session_close_unrecoverable" }); - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - } - return; - } - let recoveryError: unknown; - try { - const reading = rotated.transport.request("thread/read", {}); - void reading.catch(() => undefined); - if (pendingTail) { - await Promise.race([retainedAdmission, reading]); - const retainedRunner = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(retainedRunner.outbox.some((entry: { envelope: { payload: PrpEvent } }) => - entry.envelope.payload.sourceEventId === retainedEvents[0]!.sourceEventId)).toBe(true); - const retainedCore = stagedCheckpoint ? { - identity: JSON.parse(stagedCheckpoint.records.get("header")!).state.identity, - commands: [...stagedCheckpoint.records].filter(([key]) => key.startsWith("command:")).map(([, value]) => JSON.parse(value)), - } : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(retainedCore.identity).toEqual(baseIdentity); - expect(retainedCore.commands.some((command: { type: string }) => command.type === "run.attach")).toBe(false); - if (credentialsCleaned) { - const config = await readFile(join(stateDirectory, "codex-home", "config.toml"), "utf8"); - expect(config).toContain("https://paperclip.example/mcp/current"); - expect(config).toContain("current-assigned-gateway-credential"); - expect(config).not.toContain("first-assigned-gateway-credential"); - expect(currentHandler).not.toHaveBeenCalled(); - } - } - releaseRetained(); - const read = await reading; - expect(retainedEvents).toHaveLength(pendingTail ? 1 : 0); - if (pendingTail) expect(retainedEvents[0]).toMatchObject({ - runId: baseIdentity.runId, turnId: baseIdentity.turnId, itemId: baseIdentity.itemId, - }); - const persistedProvider = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(persistedProvider.pendingEvents).toEqual([]); - expect(persistedProvider.config.args.join("\n")).toContain('/run/B'); - expect(JSON.stringify(persistedProvider)).not.toContain("test-run-B-capability"); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - await rotated.transport.request("turn/start", { - input: [{ type: "text", text: "$assigned second authority epoch" }, { type: "skill", name: "assigned", path: join(skillRoot, "SKILL.md") }], - }); - for await (const event of rotated.transport.notifications()) { - if (event.method === "paperclip/runResult") break; - } - expect(rotated.evidence().diagnostics).toContain( - "runnerd attached the durable provider session to a fresh PRP run authority", - ); - if (pendingTail) { - const target = stagedCheckpoint ? JSON.parse(stagedCheckpoint.records.get("header")!).state - : JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8")); - expect(target.stagedRunAttachment).toBeUndefined(); - expect(target.completedWarmTransition.receipt).toMatchObject({ oldIdentity: baseIdentity, newIdentity: secondIdentity }); - } else expect(await stat(join(stateDirectory, "authority-epochs"))).toBeDefined(); - } catch (error) { - recoveryError = error; - throw error; - } finally { - releaseRetained(); - await rotated.transport.close().catch(error => { - if (recoveryError) throw new AggregateError([recoveryError, error], "Provider recovery and cleanup failed"); - throw error; - }); - } - const requestFrames = (await readFile(tracePath, "utf8")) - .trim() - .split("\n") - .map((line) => JSON.parse(line) as Record) - .filter( - (entry) => - entry.kind === "frame" && entry.direction === "client_to_provider", - ) - .map( - (entry) => - JSON.parse( - Buffer.from(String(entry.rawBase64), "base64").toString("utf8"), - ) as Record, - ) - ; - const resumeFrames = requestFrames.filter((frame) => frame.method === "thread/resume"); - for (const frame of requestFrames.filter((f) => ["thread/start", "thread/resume"].includes(String(f.method)))) { - expect(frame.params).toMatchObject({ config: { "skills.include_instructions": true } }); - } - const turnFrames = requestFrames.filter((f) => f.method === "turn/start"); - expect(turnFrames).toHaveLength(2); - for (const frame of turnFrames) { - expect(frame.params).toMatchObject({ input: expect.arrayContaining([{ type: "skill", name: "assigned", path: join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md") }]) }); - } - expect(resumeFrames.length).toBeGreaterThanOrEqual(1); - for (const frame of resumeFrames) { - expect(frame).toEqual( - expect.objectContaining({ - method: "thread/resume", - params: expect.objectContaining({ threadId: firstProviderThread.id }), - }), - ); - } - - if (pendingTail) { - await releaseMaterializedNativeRuntimeSkills(join(stateDirectory, "codex-home", "skills")); - await rm(stateDirectory, { recursive: true, force: true }); - return; - } - - // A remote process owner keeps runner-state outside the controller's local - // session root. Resume must defer to its explicit state reader instead of - // rejecting recovery before the remote checkpoint can be made available. - const externallyOwnedRunnerStateDirectory = join( - stateDirectory, - "externally-owned-runner", - ); - await rename( - join(stateDirectory, "runner"), - externallyOwnedRunnerStateDirectory, - ); - const readRunnerState = async () => - JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - "utf8", - ), - ) as Record; - const mismatched = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: { - ...secondIdentity, - runId: "run-cold-other", - turnId: "turn-cold-other", - itemId: "item-cold-other", - }, - }); - await expect(mismatched.transport.request("thread/read", {})).rejects.toThrow( - "native_runner_prp_run_rotation_unavailable", - ); - await mismatched.transport.close(); - - const externalIdentity = { - ...secondIdentity, - runId: "run-cold-external", - turnId: "turn-cold-external", - itemId: "item-cold-external", - }; - const rejectedExternalRotationSteps: string[] = []; - let externalArchiveDirectory: string | null = null; - const rejectedExternalRotation = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - rejectedExternalRotationSteps.push("prepared"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - await expect( - stat(join(stateDirectory, "control-plane")), - ).rejects.toMatchObject({ code: "ENOENT" }); - expect( - await stat( - join( - stateDirectory, - "authority-epochs", - `epoch-${archiveKey}`, - "control-plane", - ), - ), - ).toBeDefined(); - externalArchiveDirectory = join( - stateDirectory, - "external-authority-epochs", - archiveKey, - ); - await mkdir(externalArchiveDirectory, { recursive: true }); - await rename( - join(externallyOwnedRunnerStateDirectory, "runner-state.json"), - join(externalArchiveDirectory, "runner-state.json"), - ); - rejectedExternalRotationSteps.push("remote-archived"); - throw new Error("controller crashed after external archive"); - }, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - await expect( - rejectedExternalRotation.transport.request("thread/read", {}), - ).rejects.toThrow("controller crashed after external archive"); - await rejectedExternalRotation.transport.close(); - expect(rejectedExternalRotationSteps).toEqual([ - "prepared", - "remote-archived", - ]); - await expect(stat(join(stateDirectory, "control-plane"))).rejects.toThrow(); - await expect( - stat(join(externallyOwnedRunnerStateDirectory, "runner-state.json")), - ).rejects.toThrow(); - - const externalRotationSteps: string[] = []; - const externallyRotated = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - prepareExternalRunnerState: async () => { - throw new Error("retry must not prepare a new external runner"); - }, - archiveExternalRunnerState: async ({ archiveKey }) => { - expect(externalArchiveDirectory).toBe( - join(stateDirectory, "external-authority-epochs", archiveKey), - ); - externalRotationSteps.push("archived"); - return JSON.parse( - await readFile( - join(externalArchiveDirectory!, "runner-state.json"), - "utf8", - ), - ) as Record; - }, - resumeDynamicTools: dynamicTools, - resumeCompletionContract: { - revision: "contract-external", - criterionIds: ["criterion-external"], - }, - prpIdentity: externalIdentity, - }); - externallyRotated.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await externallyRotated.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: firstProviderThread.id, - sessionId: firstProviderThread.sessionId, - cwd: tmpdir(), - }); - expect(externalRotationSteps).toEqual(["archived"]); - } finally { - await externallyRotated.transport.close(); - } - - const restored = createCapabilityRunnerdCodexTransport({ - ...options, - runnerStateDirectory: externallyOwnedRunnerStateDirectory, - readRunnerState, - resumeDynamicTools: dynamicTools, - prpIdentity: externalIdentity, - }); - restored.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - const read = await restored.transport.request("thread/read", {}); - expect(read.thread).toMatchObject({ - id: "codex-thread-1", - cwd: tmpdir(), - }); - expect( - (await stat(join(stateDirectory, "codex-home", "skills", "assigned"))) - .mode & 0o222, - ).toBe(0); - expect( - ( - await stat( - join(stateDirectory, "codex-home", "skills", "assigned", "SKILL.md"), - ) - ).mode & 0o222, - ).toBe(0); - const providerState = JSON.parse( - await readFile( - join(externallyOwnedRunnerStateDirectory, "codex-provider-state.json"), - "utf8", - ), - ) as { toolBridge?: { authorized?: Record } }; - expect(Object.keys(providerState.toolBridge?.authorized ?? {})).toEqual([ - "get_task_context", - "paperclip_block", - "paperclip_finish", - ]); - await restored.transport.request("turn/start", { - input: [{ type: "text", text: "restored process" }], - }); - for await (const event of restored.transport.notifications()) { - if (event.method === "turn/completed") break; - } - expect(restored.evidence()).toMatchObject({ - runnerExited: false, - codexPid: expect.any(Number), - }); - } finally { - await restored.transport.close(); - await releaseMaterializedNativeRuntimeSkills( - join(stateDirectory, "codex-home", "skills"), - ); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - -async function verifyLiveRunnerAdoption( - mismatchedCheckpoint: boolean, - mismatchedArtifact = false, - goalMidTurn = false, - detachBeforeCleanup = false, - startWithoutCheckpoint = false, - preparedTask = false, - adoptionAuthorization: "allow" | "deny" | "legacy" | "legacy-missing" | "missing-binding" = "allow", - toolHandlerOrder?: "before" | "after" | "completion-before" | "completion-after", - uncertainAtClose = false, - semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", - stopExitProofFault?: "missing" | "false", -) { - const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); - const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; - const completionPath = join(stateDirectory, "pending-completion.json"); - if (pendingCompletion) await writeFile(completionPath, JSON.stringify({ - schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed the synthetic work", - completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [], remainingWork: [] }, - evidence: [], verification: [], attentionRequests: [], artifacts: [], - })); - const server = createServer(); - let authority: DurablePrpControlPlane | null = null; - const checkpoint = vi.fn(async () => undefined); - server.on("upgrade", (request, socket, head) => { - if (!authority) { - socket.destroy(); - return; - } - authority.handleUpgrade(request, socket, "/runner", head); - }); - await new Promise((resolveListen) => - server.listen(0, "127.0.0.1", resolveListen), - ); - const address = server.address(); - if (!address || typeof address === "string") - throw new Error("Expected adoption test listener"); - const registration = vi.fn(async (next: DurablePrpControlPlane) => { - authority = next; - return { - connectUrl: `ws://127.0.0.1:${address.port}/runner`, - ...(mismatchedArtifact || uncertainAtClose || semanticRecoveryMode === "cleanup-only" ? { checkpoint } : {}), - release: async () => { - if (authority === next) authority = null; - }, - }; - }); - const identity = { - runnerInstanceId: "runner-live-adopt", - environmentLeaseId: "lease-live-adopt", - runId: "run-live-adopt", - normalizedSessionId: "session-live-adopt", - turnId: "turn-live-adopt", - itemId: "item-live-adopt", - }; - const sharedOptions = { - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, ...(pendingCompletion ? ["--terminal-result-file", completionPath] : toolHandlerOrder ? ["--emit-tool-call"] : []), ...(goalMidTurn ? ["--goal-autostart", "--goal-item-trigger", join(stateDirectory, "emit-goal-item")] : [])), - stateDirectory, - prpIdentity: identity, - lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, - controlPlaneRegistration: registration, - }; - const prepareLaunch = vi.fn(async (requirements: RunnerProviderLaunchRequirements) => ({ - configurationId: "adopted-preparation", requirementsDigest: runnerLaunchRequirementsDigest(requirements), - provider: createRunnerProviderLaunch(requirements, { options: { codexArgs: sharedOptions.codexArgs }, - root: stateDirectory, cwd: tmpdir(), codexHome: join(stateDirectory, "codex-home"), - codexCommand: fakeCodex, providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: "" }), - })); - const options = { ...sharedOptions, ...(preparedTask ? { - prepareLaunch, runnerArtifact: readRunnerdArtifactBinding(defaultCapabilityRunnerdBinary()), - runnerProcessLauncher: (spec: durableControlPlane.RunnerProcessLaunchSpec) => { - const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: { ...process.env, ...spec.environment }, - detached: process.platform !== "win32", stdio: "ignore" }); - return { child, processGroupId: child.pid ?? null, completion: new Promise<{ code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }>((resolveExit, reject) => { - child.once("error", reject); - child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); - }) }; - }, - } : {}) }; - const first = createCapabilityRunnerdCodexTransport(options); - const pendingTool = vi.fn(() => new Promise>(() => undefined)); - if (toolHandlerOrder) first.transport.setServerRequestHandler(pendingTool); - let releaseRecovery!: () => void; - const recoveryGate = new Promise(resolve => { releaseRecovery = resolve; }); - const recoveryDenied = new Error("execution authority revoked after Stop"); - const recoveredTool = vi.fn(async () => { - if (semanticRecoveryMode) { - // Model server authority denying replay after Stop cleared the task lock. - // The gate holds that denial until reattachment has returned its session, - // before the caller invokes the eventual onSession cancellation callback. - await recoveryGate; - throw recoveryDenied; - } - return { result: { receipt: "known-outcome" }, isError: false }; - }); - const redispatchedTool = vi.fn(async () => ({ success: true, contentItems: [] })); - let runnerPid: number | null = null; - let runnerStartedAt: string | null = null; - let adopted: ReturnType | null = - null; - let restoreIntervalSpies: () => void = () => undefined; - try { - let opened: Record; - try { - opened = await first.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: toolHandlerOrder && !pendingCompletion ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : codexSemanticToolSpecs(), - }); - } catch (error) { - const stderr = await readFile( - join(stateDirectory, "diagnostics", "runnerd.stderr.log"), - "utf8", - ).catch(() => ""); - throw new Error( - `${String(error)}\n${JSON.stringify(first.evidence())}${stderr ? `\n${stderr}` : ""}`, - ); - } - runnerPid = first.evidence().runnerPid; - runnerStartedAt = first.transport.processInfo().startedAt; - expect(runnerPid).toEqual(expect.any(Number)); - - if (goalMidTurn) { - await first.transport.request("thread/goal/set", { objective: "Recover a live goal", status: "active" }); - for await (const event of first.transport.notifications()) { - if (event.method === "turn/started") break; - } - } - - if (toolHandlerOrder) { - await first.transport.request("turn/start", { input: [{ type: "text", text: "Read current task" }] }); - await vi.waitFor(() => expect(pendingTool).toHaveBeenCalledOnce(), { timeout: 5_000 }); - } - await first.detachControllerForRestart(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - if (goalMidTurn) { - await writeFile(join(stateDirectory, "emit-goal-item"), "emit"); - // runnerd need not poll the provider into its PRP outbox while disconnected. - // Wait for flushed provider output, not a platform-dependent final poll - // racing the disconnect. Adoption must still bind that buffered item. - await vi.waitFor(async () => { - expect(await readFile(join(stateDirectory, "emit-goal-item.sent"), "utf8")).toBe("sent"); - }, { timeout: 5_000 }); - } - - const controlPlaneStatePath = join( - stateDirectory, - "control-plane", - "control-plane-state.json", - ); - const compactProviderIdentityEvents = async () => { - const controlPlaneState = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { committedEvents: Array<{ eventType: string }> }; - controlPlaneState.committedEvents = - controlPlaneState.committedEvents.filter( - (event) => - event.eventType !== "harness.ready" && - event.eventType !== "session.started" && - event.eventType !== "session.resumed", - ); - await writeFile( - controlPlaneStatePath, - `${JSON.stringify(controlPlaneState, null, 2)}\n`, - { mode: 0o600 }, - ); - }; - if (!startWithoutCheckpoint) await compactProviderIdentityEvents(); - - if (preparedTask) { - if (adoptionAuthorization !== "legacy") await rm(join(stateDirectory, "launch-preparation.json")); - prepareLaunch.mockClear(); - registration.mockClear(); - if (adoptionAuthorization === "missing-binding") { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - delete retained.preparedLaunchConfigurations; - await writeFile(controlPlaneStatePath, JSON.stringify(retained), { mode: 0o600 }); - } - } - const authorizeRunnerAdoption = vi.fn(async () => { - expect(registration).not.toHaveBeenCalled(); - if (adoptionAuthorization === "deny") throw new Error("project grant revoked"); - }); - const duplicateLauncher = vi.fn(() => { - throw new Error("duplicate runner spawn attempted"); - }); - const openedThread = opened.thread as Record; - const signal = vi.fn(() => true); - const probeError = new Error("fixture remote process identity is unavailable"); - let rejectLiveness = false; - const intervals = uncertainAtClose ? vi.spyOn(globalThis, "setInterval") : null; - const clearedIntervals = uncertainAtClose ? vi.spyOn(globalThis, "clearInterval") : null; - restoreIntervalSpies = () => { intervals?.mockRestore(); clearedIntervals?.mockRestore(); }; - adopted = createCapabilityRunnerdCodexTransport({ - ...options, - ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), - ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), - ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), - // Hash different stable bytes without replacing the real runner artifact - // used by concurrent tests. Adoption must never execute this path. - ...(mismatchedArtifact - ? { - runnerBinary: resolve(import.meta.dirname, "../../package.json"), - runnerReconnectGraceMs: 150, - } - : {}), - ...(toolHandlerOrder && !pendingCompletion ? { recoverSemanticToolInput: recoveredTool } : {}), - resumeDynamicTools: pendingCompletion ? codexSemanticToolSpecs() : toolHandlerOrder ? [{ name: "get_task_context", description: "Read current task", inputSchema: { type: "object", properties: {}, additionalProperties: false } }] : [], - resumeProviderSession: startWithoutCheckpoint ? undefined : { - driverSessionId: String(openedThread.id), - providerSessionId: mismatchedCheckpoint - ? "wrong-provider-session" - : String(openedThread.sessionId), - }, - runnerProcessLauncher: duplicateLauncher, - adoptExistingRunner: { - pid: runnerPid!, - processGroupId: runnerPid, - startedAt: runnerStartedAt, - signal, - isAlive: () => { - if (rejectLiveness) return Promise.reject(probeError); - try { - process.kill(runnerPid!, 0); - return true; - } catch { - return false; - } - }, - }, - }); - if (preparedTask && ["deny", "legacy-missing", "missing-binding"].includes(adoptionAuthorization)) { - await expect(adopted.transport.request("thread/read", {})).rejects.toThrow( - adoptionAuthorization === "deny" ? "project grant revoked" : adoptionAuthorization === "legacy-missing" - ? "runner_launch_recovery_requirements_unavailable" : "runner_launch_recovery_configuration_unavailable"); - expect(registration).not.toHaveBeenCalled(); - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(authorizeRunnerAdoption).toHaveBeenCalledTimes(adoptionAuthorization === "deny" ? 1 : 0); - return; - } - if (mismatchedArtifact) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - expect(authority?.activeRunnerConnectionCount()).toBe(0); - await expect( - adopted.transport.request("turn/start", { - input: [{ type: "text", text: "must not be dispatched" }], - }), - ).rejects.toThrow("native_adopted_runner_authentication_timeout"); - await adopted.transport.close(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse( - await readFile(controlPlaneStatePath, "utf8"), - ) as { - identity: unknown; - commands: Array<{ type: string; status: string }>; - }; - expect(retained.identity).toEqual(identity); - expect( - retained.commands.some( - (command) => - command.type === "runner.drain" && command.status === "pending", - ), - ).toBe(true); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["runner.suspend"]), - ); - expect(retained.commands.map((command) => command.type)).not.toEqual( - expect.arrayContaining(["turn.stop"]), - ); - return; - } - if (mismatchedCheckpoint) { - await expect( - adopted.transport.request("thread/read", {}), - ).rejects.toThrow("native_adopted_provider_identity_mismatch"); - expect(duplicateLauncher).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - return; - } - if (toolHandlerOrder?.endsWith("before")) adopted.transport.setServerRequestHandler(redispatchedTool); - await expect(adopted.transport.request(startWithoutCheckpoint ? "thread/start" : "thread/read", {})).resolves.toEqual( - expect.objectContaining({ - thread: expect.objectContaining({ id: "codex-thread-1" }), - }), - ); - if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); - if (semanticRecoveryMode) { - const retained = authority as DurablePrpControlPlane | null; - expect(retained).not.toBeNull(); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - if (semanticRecoveryMode === "denied-before-cancel") { - await vi.waitFor(() => expect(recoveredTool).toHaveBeenCalledOnce()); - // This is a real Core replay callback, not an artificial transport error. - // Deliver its denial before giving the caller its cancellation turn. - releaseRecovery(); - await vi.waitFor(async () => { - await expect(adopted!.transport.request("thread/read", {})).rejects.toBe(recoveryDenied); - }); - await expect(adopted.transport.request("turn/interrupt", {})).rejects.toBe(recoveryDenied); - expect(retained!.store.state.commands.some(command => command.type === "turn.interrupt")).toBe(false); - } else { - // Let all startup/handler microtasks and a real reconciliation read run. - // Cleanup must not enter replay even while its authority would deny it. - await adopted.transport.request("thread/read", {}); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - releaseRecovery(); - await expect(adopted.transport.request("turn/interrupt", {})).resolves.toEqual({}); - expect(retained!.store.state.commands.filter(command => command.type === "turn.interrupt")).toMatchObject([{ status: "completed" }]); - } - expect(redispatchedTool).not.toHaveBeenCalled(); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.commands.filter(command => command.type === "turn.start")).toHaveLength(1); - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (semanticRecoveryMode === "cleanup-only") { - const faultedStopProof = vi.fn(); - if (stopExitProofFault) { - const queue = retained!.queueCommand.bind(retained!); - vi.spyOn(retained!, "queueCommand").mockImplementation((type, ...args) => { - const command = queue(type, ...args); - if (type === "turn.stop") { - let result = command.result; - Object.defineProperty(command, "result", { - enumerable: true, - configurable: true, - get: () => result, - set: value => { - const receipt = value as { result?: Record } | null; - if (receipt?.result && typeof receipt.result === "object") { - faultedStopProof(receipt.result.providerExitConfirmed); - const stopped = { ...receipt.result }; - if (stopExitProofFault === "missing") delete stopped.providerExitConfirmed; - else stopped.providerExitConfirmed = false; - result = { ...receipt, result: stopped }; - } else result = value; - }, - }); - } - return command; - }); - await expect(adopted.transport.close()).rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); - expect(faultedStopProof).toHaveBeenCalledWith(true); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("unsettled"); - } else { - await expect(adopted.transport.close()).resolves.toBeUndefined(); - expect(checkpoint).toHaveBeenCalledExactlyOnceWith("settled"); - expect(retained!.store.state.commands.filter(command => command.type === "turn.stop")).toMatchObject([{ - status: "completed", result: { result: { providerExitConfirmed: true } }, - }]); - } - expect(retained!.semanticToolResultsSettled()).toBe(false); - expect(retained!.store.state.commands.some(command => command.type === "semantic_tool.result")).toBe(false); - expect(retained!.store.state.committedEvents.some(event => event.eventType === "semantic_tool.input")).toBe(true); - if (!stopExitProofFault) { - expect(retained!.store.state.commands.filter(command => command.type === "runner.suspend")).toMatchObject([{ status: "completed" }]); - const runnerState = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8")); - expect(runnerState).toMatchObject({ ...identity, lifecycle: "suspended" }); - } - const providerState = JSON.parse(await readFile(join(stateDirectory, "runner", "codex-provider-state.json"), "utf8")); - expect(providerState).toMatchObject({ activeProviderTurnId: null, pendingEvents: [], queuedEvents: [] }); - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).not.toHaveBeenCalled(); - } - return; - } - if (toolHandlerOrder) { - await vi.waitFor(() => { - expect(pendingCompletion ? redispatchedTool : recoveredTool).toHaveBeenCalledOnce(); - const result = authority!.store.state.commands.find(command => command.type === "semantic_tool.result"); - expect(result?.status).toBe("completed"); - }, { timeout: 5_000 }); - if (pendingCompletion) { - expect(recoveredTool).not.toHaveBeenCalled(); - expect(redispatchedTool).toHaveBeenCalledWith(expect.objectContaining({ params: expect.objectContaining({ tool: "paperclip_finish" }) })); - } else expect(redispatchedTool).not.toHaveBeenCalled(); - } - expect(adopted.evidence().runnerPid).toBe(runnerPid); - expect(adopted.transport.processInfo().startedAt).toBe(runnerStartedAt); - if (startWithoutCheckpoint) { - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - for (const type of ["run.prepare", "session.open"]) { - expect(retained.commands.filter((command: { type: string }) => command.type === type)).toHaveLength(1); - } - } - if (goalMidTurn) { - const observed = await Promise.race([ - (async () => { - for await (const notification of adopted!.transport.notifications()) { - if (notification.method === "item/started") return notification; - } - throw new Error("recovered goal item was lost"); - })(), - new Promise((_, reject) => setTimeout(() => reject(new Error("recovered item timed out")), 5_000)), - ]); - expect(observed.params).toMatchObject({ threadId: "codex-thread-1", turnId: "provider-goal-turn-1" }); - } - expect(duplicateLauncher).not.toHaveBeenCalled(); - if (preparedTask) { - if (adoptionAuthorization === "legacy") { - expect(prepareLaunch).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ identity }), expect.objectContaining({ configurationId: "adopted-preparation" })); - expect(authorizeRunnerAdoption).not.toHaveBeenCalled(); - } else { - expect(prepareLaunch).not.toHaveBeenCalled(); - expect(authorizeRunnerAdoption).toHaveBeenCalledExactlyOnceWith("adopted-preparation", identity); - } - expect((await adopted.transport.request("thread/read", {})).thread).toMatchObject({ cwd: tmpdir() }); - } - if (uncertainAtClose) { - const before = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const retainedAuthority = authority as DurablePrpControlPlane | null; - rejectLiveness = true; - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(retainedAuthority?.activeRunnerConnectionCount()).toBe(0); - expect(intervals!.mock.results.length).toBeGreaterThan(0); - for (const interval of intervals!.mock.results) if (interval.type === "return") expect(clearedIntervals).toHaveBeenCalledWith(interval.value); - const notificationsClosed = (async () => { for await (const _event of adopted!.transport.notifications()) {} return true; })(); - let queueDeadline: ReturnType | undefined; - try { - expect(await Promise.race([notificationsClosed, new Promise(resolve => { queueDeadline = setTimeout(() => resolve(false), 250); })])).toBe(true); - } finally { clearTimeout(queueDeadline); } - await expect(adopted.transport.close()).rejects.toBe(probeError); - expect(adopted.transport.processInfo()).toMatchObject({ exited: false }); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - expect(signal).not.toHaveBeenCalled(); - expect(checkpoint).not.toHaveBeenCalled(); - expect(duplicateLauncher).not.toHaveBeenCalled(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - expect(retained.identity).toEqual(before.identity); - expect(retained.commands).toEqual(before.commands); - return; - } - expect(adopted.evidence().diagnostics).toContain( - `adopted runner ${runnerPid} authenticated to its durable PRP authority`, - ); - if (!startWithoutCheckpoint) { - expect(adopted.evidence().diagnostics).toContain( - "restored adopted provider identity from the exact durable checkpoint after PRP event compaction; awaiting live confirmation", - ); - expect(adopted.evidence().diagnostics).toContain( - "confirmed adopted provider identity against authenticated recovery session.snapshot", - ); - } - if (detachBeforeCleanup) { - await adopted.detachControllerForRestart(); - await adopted.transport.close("old controller finalizer"); - expect(signal).not.toHaveBeenCalled(); - expect(() => process.kill(runnerPid!, 0)).not.toThrow(); - const retained = JSON.parse(await readFile(controlPlaneStatePath, "utf8")); - const commandTypes = retained.commands.map((command: { type: string }) => command.type); - expect(commandTypes).not.toContain("turn.stop"); - expect(commandTypes).not.toContain("runner.suspend"); - } - } finally { - releaseRecovery(); - await adopted?.transport.close().catch(() => undefined); - if (uncertainAtClose) { - const providerPid = adopted?.evidence().codexPid; - if (providerPid) { - try { process.kill(providerPid, "SIGKILL"); } - catch (error) { if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; } - } - await (authority as DurablePrpControlPlane | null)?.stop(); - } - restoreIntervalSpies(); - if (runnerPid) { - try { - process.kill(-runnerPid, "SIGKILL"); - } catch { - // The adopted runner normally exits after its durable suspend command. - } - } - server.closeAllConnections(); - if (server.listening) { - await new Promise((resolveClose) => - server.close(() => resolveClose()), - ); - } - await rm(stateDirectory, { recursive: true, force: true }); - } -} - -it( - "adopts a live runner on the same durable authority without spawning a duplicate", - () => verifyLiveRunnerAdoption(false), - 30_000, -); - -it("adopts a prepared runner after losing its controller launch cache without preparing resources again", - () => verifyLiveRunnerAdoption(false, false, false, false, false, true), 30_000); - -it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", - mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); -it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); - -it( - "blocks adopted runner artifact drift without duplicate launch, checkpoint replacement, or process signals", - () => verifyLiveRunnerAdoption(false, true), - 15_000, -); - -it( - "rejects a live runner whose provider identity mismatches the compacted checkpoint", - () => verifyLiveRunnerAdoption(true), - 30_000, -); - -it("binds buffered mid-goal items only after the authenticated recovery snapshot", () => verifyLiveRunnerAdoption(false, false, true), 30_000); - -it("keeps an adopted runner alive when the detached controller finalizer closes", () => verifyLiveRunnerAdoption(false, false, true, true), 30_000); - -it("adopts an opening session without a checkpoint instead of bootstrapping a duplicate provider", () => verifyLiveRunnerAdoption(false, false, false, false, true), 30_000); - it("cancels startup readiness before releasing the route that settles its ready callback", async () => { const root = await mkdtemp(join(tmpdir(), "runnerd-ready-release-cycle-")); let releaseReady!: () => void; @@ -9470,52 +5432,6 @@ it("rejects the notification stream promptly when runnerd exits after accepting } }, 30_000); -it("persists an active provider as settled before bounded suspension", async () => { - const stateDirectory = await mkdtemp( - join(tmpdir(), "runnerd-active-suspension-"), - ); - const bundle = createCapabilityRunnerdCodexTransport({ - runnerBinary: defaultCapabilityRunnerdBinary(), - codexCommand: fakeCodex, - codexArgs: fakeCodexArgs(stateDirectory, "--linger-after-turn-start"), - stateDirectory, - }); - bundle.transport.setServerRequestHandler(async () => ({ - success: true, - contentItems: [], - })); - try { - await bundle.transport.request("initialize", {}); - await bundle.transport.request("thread/start", { - cwd: tmpdir(), - dynamicTools: [], - }); - await bundle.transport.request("turn/start", { - input: [{ type: "text", text: "Wait for another instruction." }], - }); - const notifications = bundle.transport - .notifications() - [Symbol.asyncIterator](); - await expectTurnStarted(notifications); - await bundle.transport.close(); - - const providerState = JSON.parse( - await readFile( - join(stateDirectory, "runner", "codex-provider-state.json"), - "utf8", - ), - ) as Record; - expect(providerState).toMatchObject({ - lifecycle: "prepared", - activeProviderTurnId: null, - }); - } finally { - await bundle.transport.close(); - await rm(stateDirectory, { recursive: true, force: true }); - } -}, 30_000); - - it.each(["claude", "codex"] as const)("keeps the explicitly assigned gateway in the %s runner environment", (agent) => { const gateway = { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", @@ -9550,7 +5466,6 @@ it("resolves explicit skills to the remote provider home and rejects unassigned expect(() => resolveRunnerdCodexSkillInputs([skill], null, "/runner/codex-home")).toThrow("assigned runtime skill"); }); - it("admits only exact live Pi turn controls", () => { expect(parseAcpxTurnControlCapabilities(undefined, "pi")).toEqual({ steering: false, queuedFollowUp: false }); expect(parseAcpxTurnControlCapabilities({ steering: true, queuedFollowUp: true }, "pi")).toEqual({ steering: true, queuedFollowUp: true }); @@ -9843,7 +5758,6 @@ it.each([ }); }, 30_000); - it.each([ { provider: "codex", acpxAgent: "cursor", acpxMode: "plan" }, { provider: "acpx", acpxAgent: "copilot", acpxMode: "" }, @@ -9856,11 +5770,11 @@ it.each([ it.each([undefined, "medium", "minimal", "xhigh", null])("rejects non-exact Pi transport thinking level %s before spawn", piThinkingLevel => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "acpx", acpxAgent: "pi", piThinkingLevel: piThinkingLevel as never })).toThrow(/thinking/); }); + it("rejects Pi thinking settings on a different transport provider", () => { expect(() => createCapabilityRunnerdCodexTransport({ provider: "codex", piThinkingLevel: "low" })).toThrow(/only supported/); }); - describe("cold process admission budget", () => { it("bounds Pi cold startup and recovery at 60 seconds", () => { expect(coldAdmissionTimeoutMs("acpx", "pi", true)).toBe(60_000); @@ -9929,7 +5843,6 @@ it("reports a generic failure when an ACPX rejection does not match Runner's clo expect((failure as Error).message).not.toContain("Private provider explanation."); }); - it("shares Pi's absolute cold deadline across recovery barriers and rejects a late ACK", async () => { vi.useFakeTimers(); try { @@ -9978,6 +5891,7 @@ it("aborts startup promptly on close while allowing the owned cleanup command to await drain; } finally { vi.useRealTimers(); } }); + it("launches and closes native Codex without materializing or editing the preconfigured home", async () => { const root = await mkdtemp(join(tmpdir(), "runner-preconfigured-launch-")); const home = join(root, "user-home"); @@ -10005,19 +5919,3 @@ it("launches and closes native Codex without materializing or editing the precon await rm(root, { force: true, recursive: true }); } }, 30_000); - - -it.each(["before", "after"] as const)("settles a saved tool outcome with the handler installed %s recovery startup", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it.each(["completion-before", "completion-after"] as const)("recovers a pending finish without an ordinary-tool lookup (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order), 30_000); - -it("exposes semantic recovery before cancellation when the authority has been revoked", () => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "denied-before-cancel"), 30_000); - -it.each(["before", "after", "completion-before", "completion-after"] as const)("allows cleanup-only adoption to cancel without replaying a pending semantic input (%s)", order => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", order, false, "cleanup-only"), 30_000); - -it.each(["missing", "false"] as const)("refuses cleanup settlement when the exact stop receipt has %s provider exit proof", fault => - verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", "before", false, "cleanup-only", fault), 30_000); diff --git a/packages/paperclip-runner/test/support/runnerd-transport.ts b/packages/paperclip-runner/test/support/runnerd-transport.ts new file mode 100644 index 00000000000..171414f2415 --- /dev/null +++ b/packages/paperclip-runner/test/support/runnerd-transport.ts @@ -0,0 +1,88 @@ +import { join, resolve } from "node:path"; +import { expect } from "vitest"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, + type NativeRuntimeContextSnapshot, +} from "../../src/contracts/runtime-context.js"; +import { defaultCapabilityRunnerdBinary as qualifiedCapabilityRunnerdBinary } from "../../src/live/runnerd-codex-transport.js"; + +export const defaultCapabilityRunnerdBinary = () => + process.env.PAPERCLIP_ATTACH_TRANSITION_RUNNER ?? + qualifiedCapabilityRunnerdBinary(); + +export async function expectTurnStarted( + notifications: AsyncIterator<{ method: string }>, +) { + for (let index = 0; index < 32; index += 1) { + const next = await notifications.next(); + expect(next.done).not.toBe(true); + if (next.value.method === "turn/started") return; + // Startup ownership and capability facts can precede the active turn. + expect(next.value.method).toBe("paperclip/canonicalProviderEvent"); + } + throw new Error( + "provider turn did not start within the bounded notification prefix", + ); +} + +export const fakeCodex = resolve( + import.meta.dirname, + "../../runner/target/debug/fake-codex-app-server", +); + +export function fakeCodexArgs(stateDirectory: string, ...args: string[]): string[] { + return [ + "--state-file", + join(stateDirectory, "fake-codex-state.json"), + ...args, + ]; +} + +export function assignedRuntimeContext( + skillRoot: string, + instructionRoot: string, +): NativeRuntimeContextSnapshot { + const digest = "0".repeat(64); + const value = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath: "AGENTS.md", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: instructionRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + skills: [ + { + key: "company/assigned", + runtimeName: "assigned", + versionId: "version-1", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: skillRoot, + fileCount: 1, + totalBytes: 1, + }, + }, + ], + mcp: { assignmentSetId: "assigned", digest, bindingId: "binding" }, + } satisfies Omit; + return { + ...value, + aggregateDigest: canonicalNativeRuntimeContextDigest(value), + }; +} diff --git a/packages/paperclip-runner/vitest.config.ts b/packages/paperclip-runner/vitest.config.ts index 3514fa1ea01..4c5817aec31 100644 --- a/packages/paperclip-runner/vitest.config.ts +++ b/packages/paperclip-runner/vitest.config.ts @@ -1,7 +1,9 @@ +import { availableParallelism } from "node:os"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { configDefaults, defineConfig } from "vitest/config"; +import { RunnerSequencer } from "./scripts/vitest-sequencer.mjs"; const packageRoot = fileURLToPath(new URL(".", import.meta.url)); @@ -17,6 +19,8 @@ export default defineConfig({ ? { reporters: [...configDefaults.reporters, resolve(packageRoot, "../../scripts/ci-profile/vitest-reporter.mjs")] } : {}), environment: "node", + ...(process.env.GITHUB_ACTIONS === "true" ? { maxWorkers: Math.min(4, availableParallelism()) } : {}), + sequence: { sequencer: RunnerSequencer }, include: [ "src/**/*.test.ts", "src/**/*.test.tsx", @@ -27,6 +31,7 @@ export default defineConfig({ "examples/scenario-explorer/src/**/*.test.tsx", "scripts/local-runner-browser-server.test.mjs", "scripts/live-console-browser-server.test.mjs", + "scripts/vitest-sequencer.test.mjs", ], }, }); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 3ca0bc25840..a507f3034f8 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -966,7 +966,9 @@ test('native dependency caches are target-bound and unmerged master callers cann const cache = producer.split(' - name: Cache native Rust dependencies\n')[1].split('\n - ')[0]; assert.match(cache, /Swatinem\/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6/); assert.match(cache, /workspaces: \$\{\{ steps.native_rust_workspace.outputs.path \}\} -> target/); - assert.match(cache, /shared-key: release-native-v2-\$\{\{ matrix.host \}\}-\$\{\{ matrix.target \}\}-\$\{\{ hashFiles\('packages\/paperclip-runner\/protocol\/\*\*', 'packages\/paperclip-runner\/src\/\*\*', 'packages\/paperclip-runner\/\*.json'\) \}\}/); + const key = cache.match(/^\s*shared-key: (.+)$/m)?.[1]; + assert.ok(key?.startsWith("release-native-v2-${{ hashFiles('packages/paperclip-runner/runner/Cargo.toml', ")); + assert.ok(key?.endsWith("-${{ matrix.host }}-${{ matrix.target }}-${{ hashFiles('packages/paperclip-runner/protocol/**', 'packages/paperclip-runner/src/**', 'packages/paperclip-runner/*.json') }}")); for (const option of ['cache-workspace-crates', 'cache-bin', 'cache-on-failure']) { assert.match(cache, new RegExp(option + ': false')); } diff --git a/scripts/ci-dependency-cache.mjs b/scripts/ci-dependency-cache.mjs index b447c01d362..cca636cfb45 100644 --- a/scripts/ci-dependency-cache.mjs +++ b/scripts/ci-dependency-cache.mjs @@ -7,11 +7,19 @@ import { readPluginsUnder } from "./link-plugin-dev-sdk.mjs"; const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const postinstallHooks = new Map([ + [".", "node scripts/link-plugin-dev-sdk.mjs"], + ["server", "node scripts/install-copilot-assets.mjs"], + ["packages/paperclip-runner", "node scripts/install-copilot-assets.mjs"], +]); + export function cachePlan({ files, read, scope, packageDirs }) { const dependencyInputs = files.filter((file) => /(^|\/)package\.json$/.test(file) || ["pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc", ".pnpmfile.cjs", "pnpmfile.cjs"].includes(file) || file.startsWith("scripts/") + || file.startsWith("server/scripts/") + || file.startsWith("packages/paperclip-runner/scripts/") || file.startsWith("patches/")); const digest = (inputs, salt) => { const hash = createHash("sha256").update(JSON.stringify(salt)); @@ -28,8 +36,8 @@ export function cachePlan({ files, read, scope, packageDirs }) { if (manifest === null) return false; const scripts = JSON.parse(manifest).scripts ?? {}; return ['preinstall', 'install', 'postinstall', 'prepare'].every((hook) => - scripts[hook] === undefined || (directory === '.' && hook === 'postinstall' - && scripts[hook] === 'node scripts/link-plugin-dev-sdk.mjs')); + scripts[hook] === undefined || (hook === 'postinstall' + && scripts[hook] === postinstallHooks.get(directory))); }); const buildInputs = files.filter((file) => file.startsWith("packages/shared/") || file.startsWith("packages/plugins/sdk/")