From c5e4b46d3fb047e14c2aea520338237131088687 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:29:36 -0600 Subject: [PATCH] fix(acpx): shut down the per-port ACPX runtime when its session closes (#21151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The experimental Runner starts ACP providers through ACPX, and each runtime port owns one ACPX runtime > - When the native provider dies during a turn, the port retires the turn at once, but the turn stays queued inside the ACPX runtime > - `port.close()` closed only the session handle, so that queued turn wrote its final session record after close had returned > - Any caller that removes the state directory after close races that write; CI showed it as `ENOTEMPTY ... state/sessions` in an unrelated test on 5 branches > - This pull request makes close shut down the port's own ACPX runtime after the session closes, inside the existing close bound > - The benefit is that a closed port no longer writes into its state directory, and the flaky test is fixed at its cause ## Linked Issues or Issue Description **What happened?** `packages/paperclip-runner` test `native-distribution-integrity.test.ts > native ACPX execution closure > expires a real ACPX permission on native child death without replaying its mutation` fails intermittently with `Error: ENOTEMPTY: directory not empty, rmdir '/tmp/native-acpx-test-XXXX/state/sessions'`. It failed 5 times on 5 unrelated branches in 48 hours. **Steps to reproduce** 1. Run the test 30 times while two CPU-bound shell loops run. 2. 2 of 30 runs fail with ENOTEMPTY in `afterEach`. 3. Instrumenting the session store shows a `saveSessionOwner` write from `finalizeRuntimeTurn` that starts after `port.close()` and `afterEach` have begun. **Paperclip version or commit** `master` at the base of this branch (CI on unrelated pull requests, 2026-11-10 to 2026-11-12). **Expected behavior** After `port.close()` resolves, the ACPX runtime owned by that port does no more work and writes no more state. **Actual behavior** The turn that native provider death retired is still queued in the ACPX runtime manager. It finalizes after close and writes the session file while the caller removes the directory. ## What Changed - `acpx-runtime-adapter.ts`: after the final successful close, the port calls `runtime.shutdown()` within `runtimeCloseTimeoutMs`. ACPX shutdown waits for its pending turn tasks, so the abandoned turn finishes its writes first. - A close that succeeds late, after its bound, also starts the shutdown. A concurrent or repeated `close()` waits for the same shutdown and gets its failure too. A shutdown failure is reported in the existing `ACPX runtime and provider cleanup failed` error. The shutdown attempt stays in the retained cleanup owners. - `acpx-runtime-adapter.test.ts`: regression tests. Close does not resolve before runtime shutdown settles. A shutdown failure is reported to every closer. A late close success starts the shutdown. ## Verification - Red/green: both new tests fail on master (`expected "vi.fn()" to be called once, but got 0 times`; `promise resolved "undefined" instead of rejecting`) and pass with this change. - Loop of the flaky test, 30 runs each, with 2 extra CPU-bound shell loops on an already loaded 18-core Mac (load average about 30): - Before: 28 pass, 2 fail (both `ENOTEMPTY ... state/sessions`). - After: 30 pass, 0 fail. - `npx vitest run src/drivers/acpx` in `packages/paperclip-runner`: 92 files, 1743 tests pass. - `tsc --noEmit -p packages/paperclip-runner`: clean. ## Risks - Low risk. The runtime is created per port and is not used after a successful final close. Close can now take up to one more close bound if the runtime's pending work is slow. A late reconciliation close never runs after this point, because shutdown only starts when the port is finally closed. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, used through Claude Code with tool use (shell, file edits) and extended reasoning. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../drivers/acpx/acpx-runtime-adapter.test.ts | 50 +++++++++++++++++++ .../src/drivers/acpx/acpx-runtime-adapter.ts | 31 ++++++++++-- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.test.ts index 2cde8e51642..e228bb1dc77 100644 --- a/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.test.ts @@ -2048,6 +2048,9 @@ describe("Codex ACPX runtime adapter", () => { resolveRetainedClose = resolve; }); vi.mocked(runtime.close).mockReturnValueOnce(retainedClose); + let finishPendingWork!: () => void; + const pendingWork = new Promise((resolve) => { finishPendingWork = resolve; }); + runtime.shutdown = vi.fn(() => pendingWork); const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { createRegistry: () => registry(), createStore: () => store(), @@ -2063,17 +2066,64 @@ describe("Codex ACPX runtime adapter", () => { await close; expect(runtime.close).toHaveBeenCalledOnce(); + expect(runtime.shutdown).not.toHaveBeenCalled(); + resolveRetainedClose(); await vi.advanceTimersByTimeAsync(0); + expect(runtime.shutdown).toHaveBeenCalledOnce(); + finishPendingWork(); await expect( port.close({ reason: "observe late success" }), ).resolves.toBeUndefined(); expect(runtime.close).toHaveBeenCalledOnce(); + expect(runtime.shutdown).toHaveBeenCalledOnce(); } finally { vi.useRealTimers(); } }); + it("does not finish closing until the runtime's abandoned turn work has stopped writing state", async () => { + const runtime = fakeRuntime(); + vi.mocked(runtime.close).mockResolvedValue(undefined); + let finishPendingWork!: () => void; + const pendingWork = new Promise((resolve) => { finishPendingWork = resolve; }); + runtime.shutdown = vi.fn(() => pendingWork); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + }); + + let closed = false; + let rejoined = false; + const close = port.close({ reason: "provider died mid-turn" }).then(() => { closed = true; }); + await vi.waitFor(() => expect(runtime.shutdown).toHaveBeenCalledOnce()); + const rejoin = port.close({ reason: "second caller" }).then(() => { rejoined = true; }); + await new Promise((resolve) => setImmediate(resolve)); + expect([closed, rejoined]).toEqual([false, false]); + + finishPendingWork(); + await Promise.all([close, rejoin]); + expect(runtime.close).toHaveBeenCalledOnce(); + expect(runtime.shutdown).toHaveBeenCalledOnce(); + }); + + it("reports a runtime shutdown failure after the session itself closed", async () => { + const runtime = fakeRuntime(); + vi.mocked(runtime.close).mockResolvedValue(undefined); + runtime.shutdown = vi.fn().mockRejectedValue(new Error("owned client close failed")); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + }); + + await expect(port.close({ reason: "close" })).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await expect(port.close({ reason: "repeat" })).rejects.toThrow("ACPX runtime and provider cleanup failed"); + expect(runtime.close).toHaveBeenCalledOnce(); + expect(runtime.shutdown).toHaveBeenCalledOnce(); + }); + it("retains cleanup after guardian exit until provider exit is proven", async () => { const runtime = fakeRuntime(); const child = fakeChild(); diff --git a/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.ts index 3680089046c..8823d182312 100644 --- a/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/acpx-runtime-adapter.ts @@ -1327,6 +1327,7 @@ function runtimePort( pendingExternalIntent: boolean; }; let runtimeClosed = false; + let runtimeShutdown: Promise | undefined; let providerProcessLost = false; let runtimeCloseAttempt: RuntimeCloseAttempt | undefined; let lateReconciliationOwner: Promise | undefined; @@ -1431,6 +1432,7 @@ function runtimePort( attempt.reconciliationGeneration, ); runtimeClosed = !hasUnreconciledLateFailure(); + if (runtimeClosed) void shutdownClosedRuntime(); } scheduleLateFailureReconciliation(); return; @@ -1451,6 +1453,11 @@ function runtimePort( }); }; + const shutdownClosedRuntime = (): Promise => { + runtimeShutdown ??= boundedCloseOutcome(ownedRuntimeShutdownOutcome(runtime), runtimeCloseTimeoutMs); + return runtimeShutdown; + }; + async function closeRuntime(input: { reason: string; reconciliation?: { @@ -1459,7 +1466,13 @@ function runtimePort( }; }): Promise { extensionBoundary.active?.controller.abort(new Error("ACPX runtime is closing")); - if (runtimeClosed) return; + if (runtimeClosed) { + const shutdownError = await shutdownClosedRuntime(); + if (shutdownError !== null) { + throw new AggregateError([shutdownError], "ACPX runtime and provider cleanup failed"); + } + return; + } if ( runtimeCloseAttempt?.origin.kind === "reconciliation" && input.reconciliation === undefined @@ -1526,9 +1539,10 @@ function runtimePort( } else { runtimeClosed = false; } + const shutdownError = runtimeClosed ? await shutdownClosedRuntime() : null; scheduleLateFailureReconciliation(); - if (closeError !== null || processErrors.length > 0) { - const errors = [closeError, ...processErrors].filter( + if (closeError !== null || processErrors.length > 0 || shutdownError !== null) { + const errors = [closeError, ...processErrors, shutdownError].filter( (error): error is unknown => error !== null, ); throw new AggregateError( @@ -1940,6 +1954,17 @@ function ownedRuntimeCloseOutcome( return retainRuntimeCleanupOwner(cleanup); } +function ownedRuntimeShutdownOutcome(runtime: AcpRuntime): Promise { + return retainRuntimeCleanupOwner( + Promise.resolve() + .then(() => runtime.shutdown?.()) + .then( + () => null, + (error: unknown) => error, + ), + ); +} + function retainRuntimeCleanupOwner(cleanup: Promise): Promise { activeRuntimeCleanupOwners.add(cleanup); void cleanup