fix(runner): report Runner disconnect age when a Dot bridge exits (#16296)

DurablePrpControlPlane records when the last authenticated Runner connection closes and exposes runnerDisconnectedForMs(). RunnerdDotDriver appends disconnectedMs=<n|none> to dot_runner_process_exited_recovery_required errors. Diagnostic only; authority, classification and recovery are unchanged.

Co-authored-by: Devin Foley <139239+devinfoley@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts

@@ -112,6 +112,42 @@ it("renews one authenticated connection for three weeks without replacing its au
   }
 }, 30_000);
 
+it("reports how long the authenticated Runner has been disconnected", async () => {
+  const root = mkdtempSync(resolve(tmpdir(), "runner-disconnect-age-"));
+  const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest });
+  try {
+    await core.start();
+    expect(core.runnerDisconnectedForMs()).toBeNull();
+    // An unauthenticated peer closing is not a Runner disconnect.
+    let closeUnauthenticated = () => {};
+    const unauthenticated = core.attachWireConnection({
+      onJson: () => {},
+      onClose: (listener) => {
+        closeUnauthenticated = () => listener({ message: "closed" });
+      },
+      close: () => closeUnauthenticated(),
+      sendJson: () => {},
+    });
+    expect(unauthenticated.isAuthenticated()).toBe(false);
+    closeUnauthenticated();
+    expect(core.runnerDisconnectedForMs()).toBeNull();
+    const client = (await authenticate(core, core.issueBootstrapTicket()))!;
+    expect(core.runnerDisconnectedForMs()).toBeNull();
+    const before = Date.now();
+    client.socket.destroy();
+    await vi.waitFor(() => expect(core.activeRunnerConnectionCount()).toBe(0), { timeout: 1_000 });
+    const after = Date.now();
+    expect(core.runnerDisconnectedForMs(after + 300_000)).toBeGreaterThanOrEqual(300_000);
+    expect(core.runnerDisconnectedForMs(after + 300_000)).toBeLessThanOrEqual(300_000 + after - before);
+    const resumed = (await authenticate(core, client.leaseToken!))!;
+    expect(core.runnerDisconnectedForMs()).toBeNull();
+    resumed.socket.destroy();
+  } finally {
+    await core.stop();
+    rmSync(root, { recursive: true, force: true });
+  }
+}, 30_000);
+
 it.each(["expired", "revoked", "wrong-run", "wrong-connection", "wrong-epoch", "future-expiry"])(
   "cannot renew a lease with %s authority",
   async (fault) => {

packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts

@@ -1492,6 +1492,9 @@ export class DurablePrpControlPlane {
   #expectedRunnerDigest: string;
   #server: Server | null = null;
   #connections = new Set<AuthorityConnection>();
+  // Set when the last authenticated Runner connection closes; cleared on the
+  // next authentication. Diagnostic only: it never gates authority.
+  #runnerDisconnectedAt: number | null = null;
   #connectionProcessing = new Map<AuthorityConnection, Promise<void>>();
   #pendingSemanticCalls = new Set<string>();
   #semanticCallbacksRetired = false;
@@ -1654,6 +1657,15 @@ export class DurablePrpControlPlane {
     ).length;
   }
 
+  /**
+   * Milliseconds since the last authenticated Runner connection closed, or
+   * null when a Runner is connected or has never authenticated.
+   */
+  runnerDisconnectedForMs(now = Date.now()): number | null {
+    if (this.#runnerDisconnectedAt === null || this.activeRunnerConnectionCount() > 0) return null;
+    return Math.max(0, now - this.#runnerDisconnectedAt);
+  }
+
   /** A reusable close requires every admitted callback's exact durable result. */
   semanticToolResultsSettled(): boolean {
     return (
@@ -2084,7 +2096,11 @@ export class DurablePrpControlPlane {
         };
         void tail.then(release, release);
       },
-      onClose: () => this.#connections.delete(connection),
+      onClose: () => {
+        this.#connections.delete(connection);
+        if (connection.secureChannel !== null && this.activeRunnerConnectionCount() === 0)
+          this.#runnerDisconnectedAt ??= Date.now();
+      },
     });
     this.#connections.add(connection);
     return {
@@ -2746,6 +2762,7 @@ export class DurablePrpControlPlane {
       pending.serverProof,
       clientProof,
     );
+    this.#runnerDisconnectedAt = null;
     for (const active of this.#connections) {
       if (active !== connection && active.secureChannel !== null)
         active.close();

packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts

@@ -120,7 +120,8 @@ it.each([
     } else {
       handle!.child.kill("SIGTERM");
       await handle!.completion;
-      await expect(session.read!()).rejects.toThrow("dot_runner_process_exited_recovery_required");
+      await expect(session.read!()).rejects.toThrow(
+        /^dot_runner_process_exited_recovery_required: code=\S+ signal=\S+ disconnectedMs=(?:none|\d+)$/);
     }
   } finally {
     commandSpy.mockRestore(); spawnSpy.mockRestore();

packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.ts

@@ -92,6 +92,11 @@ async function readProviderCheckpoint(options: RunnerdDotDriverOptions): Promise
   return JSON.parse(readFileSync(path, "utf8"));
 }
 
+export function runnerDisconnectDiagnostic(core: Pick<DurablePrpControlPlane, "runnerDisconnectedForMs">): string {
+  const disconnectedMs = core.runnerDisconnectedForMs();
+  return `disconnectedMs=${disconnectedMs ?? "none"}`;
+}
+
 class RunnerdDotSession implements HarnessSession {
   #core: DurablePrpControlPlane | null = null;
   #process: RunnerProcessHandle | null = null;
@@ -168,11 +173,13 @@ class RunnerdDotSession implements HarnessSession {
         // Remote monitors do not report an exit code. The authenticated,
         // persisted receipt is the evidence of shutdown, for either launcher.
         if (core.getCommand("dot_shutdown")?.status === "completed") return;
-        this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: code=${result.code} signal=${result.signal}`);
+        // Without an exit status, the controller-side disconnect age separates
+        // reconnect-grace expiry (the Runner was already away) from a crash.
+        this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: code=${result.code} signal=${result.signal} ${runnerDisconnectDiagnostic(core)}`);
         this.#wake();
       }, () => {
         if (this.#closed || core.getCommand("dot_shutdown")?.status === "completed") return;
-        this.#failure ??= new Error("dot_runner_process_exited_recovery_required");
+        this.#failure ??= new Error(`dot_runner_process_exited_recovery_required: ${runnerDisconnectDiagnostic(core)}`);
         this.#wake();
       });
     } else if (!await o.adoptExistingRunner.isAlive()) { throw new Error("dot_runner_adoption_failed"); }