From 14f961b6989d9d88d60f98ee0dff3ae2f799be7e Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Wed, 14 Oct 2026 08:26:32 -0700 Subject: [PATCH] fix(runner): report Runner disconnect age when a Dot bridge exits (#16296) DurablePrpControlPlane records when the last authenticated Runner connection closes and exposes runnerDisconnectedForMs(). RunnerdDotDriver appends disconnectedMs= to dot_runner_process_exited_recovery_required errors. Diagnostic only; authority, classification and recovery are unchanged. Co-authored-by: Devin Foley <139239+devinfoley@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- .../durable-prp-control-plane.test.ts | 36 +++++++++++++++++++ .../durable-prp-control-plane.ts | 19 +++++++++- .../drivers/dot/runnerd-dot-driver.test.ts | 3 +- .../src/drivers/dot/runnerd-dot-driver.ts | 11 ++++-- 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 94f9b79505b..fa87884d8c5 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -112,6 +112,42 @@ it("renews one authenticated connection for three weeks without replacing its au } }, 30_000); +it("reports how long the authenticated Runner has been disconnected", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-disconnect-age-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + try { + await core.start(); + expect(core.runnerDisconnectedForMs()).toBeNull(); + // An unauthenticated peer closing is not a Runner disconnect. + let closeUnauthenticated = () => {}; + const unauthenticated = core.attachWireConnection({ + onJson: () => {}, + onClose: (listener) => { + closeUnauthenticated = () => listener({ message: "closed" }); + }, + close: () => closeUnauthenticated(), + sendJson: () => {}, + }); + expect(unauthenticated.isAuthenticated()).toBe(false); + closeUnauthenticated(); + expect(core.runnerDisconnectedForMs()).toBeNull(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + expect(core.runnerDisconnectedForMs()).toBeNull(); + const before = Date.now(); + client.socket.destroy(); + await vi.waitFor(() => expect(core.activeRunnerConnectionCount()).toBe(0), { timeout: 1_000 }); + const after = Date.now(); + expect(core.runnerDisconnectedForMs(after + 300_000)).toBeGreaterThanOrEqual(300_000); + expect(core.runnerDisconnectedForMs(after + 300_000)).toBeLessThanOrEqual(300_000 + after - before); + const resumed = (await authenticate(core, client.leaseToken!))!; + expect(core.runnerDisconnectedForMs()).toBeNull(); + resumed.socket.destroy(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } +}, 30_000); + it.each(["expired", "revoked", "wrong-run", "wrong-connection", "wrong-epoch", "future-expiry"])( "cannot renew a lease with %s authority", async (fault) => { diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 4b444eaaa65..614ae6402c5 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -1492,6 +1492,9 @@ export class DurablePrpControlPlane { #expectedRunnerDigest: string; #server: Server | null = null; #connections = new Set(); + // Set when the last authenticated Runner connection closes; cleared on the + // next authentication. Diagnostic only: it never gates authority. + #runnerDisconnectedAt: number | null = null; #connectionProcessing = new Map>(); #pendingSemanticCalls = new Set(); #semanticCallbacksRetired = false; @@ -1654,6 +1657,15 @@ export class DurablePrpControlPlane { ).length; } + /** + * Milliseconds since the last authenticated Runner connection closed, or + * null when a Runner is connected or has never authenticated. + */ + runnerDisconnectedForMs(now = Date.now()): number | null { + if (this.#runnerDisconnectedAt === null || this.activeRunnerConnectionCount() > 0) return null; + return Math.max(0, now - this.#runnerDisconnectedAt); + } + /** A reusable close requires every admitted callback's exact durable result. */ semanticToolResultsSettled(): boolean { return ( @@ -2084,7 +2096,11 @@ export class DurablePrpControlPlane { }; void tail.then(release, release); }, - onClose: () => this.#connections.delete(connection), + onClose: () => { + this.#connections.delete(connection); + if (connection.secureChannel !== null && this.activeRunnerConnectionCount() === 0) + this.#runnerDisconnectedAt ??= Date.now(); + }, }); this.#connections.add(connection); return { @@ -2746,6 +2762,7 @@ export class DurablePrpControlPlane { pending.serverProof, clientProof, ); + this.#runnerDisconnectedAt = null; for (const active of this.#connections) { if (active !== connection && active.secureChannel !== null) active.close(); diff --git a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts index 62a2608ea69..26141d2bae3 100644 --- a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts @@ -120,7 +120,8 @@ it.each([ } else { handle!.child.kill("SIGTERM"); await handle!.completion; - await expect(session.read!()).rejects.toThrow("dot_runner_process_exited_recovery_required"); + await expect(session.read!()).rejects.toThrow( + /^dot_runner_process_exited_recovery_required: code=\S+ signal=\S+ disconnectedMs=(?:none|\d+)$/); } } finally { commandSpy.mockRestore(); spawnSpy.mockRestore(); diff --git a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.ts b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.ts index 0e9bc59c405..dd897bddf95 100644 --- a/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.ts +++ b/packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.ts @@ -92,6 +92,11 @@ async function readProviderCheckpoint(options: RunnerdDotDriverOptions): Promise return JSON.parse(readFileSync(path, "utf8")); } +export function runnerDisconnectDiagnostic(core: Pick): string { + const disconnectedMs = core.runnerDisconnectedForMs(); + return `disconnectedMs=${disconnectedMs ?? "none"}`; +} + class RunnerdDotSession implements HarnessSession { #core: DurablePrpControlPlane | null = null; #process: RunnerProcessHandle | null = null; @@ -168,11 +173,13 @@ class RunnerdDotSession implements HarnessSession { // Remote monitors do not report an exit code. The authenticated, // persisted receipt is the evidence of shutdown, for either launcher. if (core.getCommand("dot_shutdown")?.status === "completed") return; - this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: code=${result.code} signal=${result.signal}`); + // Without an exit status, the controller-side disconnect age separates + // reconnect-grace expiry (the Runner was already away) from a crash. + this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: code=${result.code} signal=${result.signal} ${runnerDisconnectDiagnostic(core)}`); this.#wake(); }, () => { if (this.#closed || core.getCommand("dot_shutdown")?.status === "completed") return; - this.#failure ??= new Error("dot_runner_process_exited_recovery_required"); + this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: ${runnerDisconnectDiagnostic(core)}`); this.#wake(); }); } else if (!await o.adoptExistingRunner.isAlive()) { throw new Error("dot_runner_adoption_failed"); }