fix(runner): close bootstrap auth race

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { createDecipheriv, createHash, createHmac } from "node:crypto";
 import {
   chmodSync,
   lstatSync,
@@ -12,6 +13,7 @@ import {
 import { connect, type Socket } from "node:net";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
 
 import { describe, expect, it } from "vitest";
 
@@ -59,10 +61,23 @@ async function upgradeSocket(url: string): Promise<Socket> {
   });
 }
 
-async function receiveServerJson(socket: Socket): Promise<Record<string, unknown>> {
+async function receiveServerOutcome(socket: Socket): Promise<Record<string, unknown> | null> {
   return new Promise((resolveValue, rejectValue) => {
     let buffer = Buffer.alloc(0);
-    const timer = setTimeout(() => rejectValue(new Error("Server frame timed out.")), 2_000);
+    const cleanup = (): void => {
+      clearTimeout(timer);
+      socket.off("data", onData);
+      socket.off("close", onClose);
+      socket.off("error", onError);
+    };
+    const onClose = (): void => {
+      cleanup();
+      resolveValue(null);
+    };
+    const onError = (error: Error): void => {
+      cleanup();
+      rejectValue(error);
+    };
     const onData = (chunk: Buffer): void => {
       buffer = Buffer.concat([buffer, chunk]);
       if (buffer.length < 2) return;
@@ -78,8 +93,7 @@ async function receiveServerJson(socket: Socket): Promise<Record<string, unknown
         cursor = 10;
       }
       if (buffer.length < cursor + length) return;
-      clearTimeout(timer);
-      socket.off("data", onData);
+      cleanup();
       resolveValue(
         JSON.parse(buffer.subarray(cursor, cursor + length).toString("utf8")) as Record<
           string,
@@ -87,11 +101,22 @@ async function receiveServerJson(socket: Socket): Promise<Record<string, unknown
         >,
       );
     };
+    const timer = setTimeout(() => {
+      cleanup();
+      rejectValue(new Error("Server frame timed out."));
+    }, 2_000);
     socket.on("data", onData);
-    socket.once("error", rejectValue);
+    socket.once("close", onClose);
+    socket.once("error", onError);
   });
 }
 
+async function receiveServerJson(socket: Socket): Promise<Record<string, unknown>> {
+  const value = await receiveServerOutcome(socket);
+  if (value === null) throw new Error("WebSocket closed before the server frame arrived.");
+  return value;
+}
+
 function sendMaskedJson(socket: Socket, value: unknown): void {
   const payload = Buffer.from(JSON.stringify(value));
   const mask = Buffer.from([0x11, 0x22, 0x33, 0x44]);
@@ -109,6 +134,7 @@ function sendMaskedJson(socket: Socket, value: unknown): void {
 }
 
 async function expectSocketClosed(socket: Socket): Promise<void> {
+  if (socket.destroyed) return;
   await new Promise<void>((resolveClose, rejectClose) => {
     const timer = setTimeout(() => rejectClose(new Error("WebSocket was not rejected.")), 2_000);
     socket.once("close", () => {
@@ -143,6 +169,139 @@ function authHello(
   };
 }
 
+function domainDigest(domain: string, parts: readonly Buffer[]): Buffer {
+  const digest = createHash("sha256").update(domain).update(Buffer.from([0]));
+  for (const part of parts) {
+    const length = Buffer.alloc(8);
+    length.writeBigUInt64BE(BigInt(part.length));
+    digest.update(length).update(part);
+  }
+  return digest.digest();
+}
+
+function domainHmac(key: Buffer, domain: string, parts: readonly Buffer[]): Buffer {
+  const digest = createHmac("sha256", key).update(domain).update(Buffer.from([0]));
+  for (const part of parts) {
+    const length = Buffer.alloc(8);
+    length.writeBigUInt64BE(BigInt(part.length));
+    digest.update(length).update(part);
+  }
+  return digest.digest();
+}
+
+interface AuthExchange {
+  response: Record<string, unknown>;
+  authKey: Buffer;
+  canonicalChallenge: string;
+  serverProof: string;
+  clientProof: string;
+}
+
+function validAuthResponse(
+  challenge: Record<string, unknown>,
+  token: string,
+): AuthExchange {
+  const challengePayload = challenge.payload as Record<string, unknown>;
+  const serverProof = challengePayload.serverProof;
+  if (typeof serverProof !== "string") throw new Error("Challenge omitted its server proof.");
+  const canonicalPayload = { ...challengePayload };
+  delete canonicalPayload.serverProof;
+  const canonicalChallenge = phase3Internals.canonicalJson(canonicalPayload);
+  const material = phase3Internals.credentialMaterial(token);
+  const expectedServerProof = domainHmac(
+    material.authKey,
+    "paperclip-runner-server-proof-v1",
+    [Buffer.from(canonicalChallenge)],
+  ).toString("hex");
+  if (serverProof !== expectedServerProof) throw new Error("Challenge server proof was invalid.");
+  const clientProof = domainHmac(
+    material.authKey,
+    "paperclip-runner-client-proof-v1",
+    [Buffer.from(canonicalChallenge), Buffer.from(serverProof)],
+  ).toString("hex");
+  return {
+    response: {
+      protocol: "paperclip.runner",
+      version: 1,
+      kind: "auth_response",
+      payload: {
+        credentialId: challengePayload.credentialId,
+        clientNonce: challengePayload.clientNonce,
+        serverNonce: challengePayload.serverNonce,
+        clientProof,
+      },
+    },
+    authKey: material.authKey,
+    canonicalChallenge,
+    serverProof,
+    clientProof,
+  };
+}
+
+function decryptCoreFrame(
+  frame: Record<string, unknown>,
+  exchange: AuthExchange,
+): Record<string, unknown> {
+  if (
+    frame.schema !== "paperclip.runner.secure-frame.v1" ||
+    typeof frame.counter !== "number" ||
+    typeof frame.ciphertext !== "string"
+  ) {
+    throw new Error("Expected an encrypted core frame.");
+  }
+  const counter = BigInt(frame.counter);
+  const binding = domainDigest("paperclip-runner-session-binding-v1", [
+    Buffer.from(exchange.canonicalChallenge),
+    Buffer.from(exchange.serverProof),
+    Buffer.from(exchange.clientProof),
+  ]);
+  const receiveKey = domainHmac(
+    exchange.authKey,
+    "paperclip-runner-core-to-client-key-v1",
+    [binding],
+  );
+  const nonce = Buffer.alloc(12);
+  nonce.write("P3S1", 0, "ascii");
+  nonce.writeBigUInt64BE(counter, 4);
+  const sessionId = `sha256:${binding.toString("hex")}`;
+  const aad = Buffer.from(
+    `paperclip.runner.secure-frame.v1\0${sessionId}\0core_to_client\0${counter}`,
+  );
+  const sealed = Buffer.from(frame.ciphertext, "hex");
+  const decipher = createDecipheriv("aes-256-gcm", receiveKey, nonce);
+  decipher.setAAD(aad);
+  decipher.setAuthTag(sealed.subarray(-16));
+  return JSON.parse(
+    Buffer.concat([decipher.update(sealed.subarray(0, -16)), decipher.final()]).toString(
+      "utf8",
+    ),
+  ) as Record<string, unknown>;
+}
+
+async function beginWireAuthentication(
+  core: Phase3MockCore,
+  token: string,
+): Promise<{ socket: Socket; exchange: AuthExchange }> {
+  const socket = await upgradeSocket(core.connectUrl);
+  const credentialId = phase3Internals.credentialMaterial(token).credentialId;
+  sendMaskedJson(socket, authHello(core.identity, credentialId));
+  const challenge = await receiveServerJson(socket);
+  expect(challenge.kind).toBe("auth_challenge");
+  return { socket, exchange: validAuthResponse(challenge, token) };
+}
+
+async function authenticateWire(
+  core: Phase3MockCore,
+  token: string,
+): Promise<{ socket: Socket; welcome: Record<string, unknown> }> {
+  const { socket, exchange } = await beginWireAuthentication(core, token);
+  const framePromise = receiveServerJson(socket);
+  sendMaskedJson(socket, exchange.response);
+  const welcome = decryptCoreFrame(await framePromise, exchange);
+  expect(welcome.kind).toBe("welcome");
+  return { socket, welcome };
+}
+
 async function runRunnerProcess(options: {
   connectUrl: string;
   stateDirectory: string;
@@ -262,6 +421,81 @@ describe.sequential("Phase 3 durable transport and recovery", () => {
     }
   });
 
+  it("authenticates only one of two valid proofs for the same bootstrap ticket", async () => {
+    const scratch =
+      process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+      process.env.PAPERCLIP_SCRATCH_DIR ??
+      tmpdir();
+    const root = mkdtempSync(resolve(scratch, "phase3-concurrent-ticket-test-"));
+    const core = new Phase3MockCore({
+      stateDirectory: root,
+      identity: phase3Internals.phase3Identity(),
+      fault: "none",
+    });
+    try {
+      await core.start();
+      const ticket = core.issueBootstrapTicket();
+      const credentialId = phase3Internals.credentialMaterial(ticket).credentialId;
+      const [first, second] = await Promise.all([
+        beginWireAuthentication(core, ticket),
+        beginWireAuthentication(core, ticket),
+      ]);
+      const firstOutcome = receiveServerOutcome(first.socket);
+      const secondOutcome = receiveServerOutcome(second.socket);
+
+      sendMaskedJson(first.socket, first.exchange.response);
+      sendMaskedJson(second.socket, second.exchange.response);
+      const outcomes = await Promise.all([firstOutcome, secondOutcome]);
+      const authenticated = outcomes.flatMap((outcome, index) =>
+        outcome === null
+          ? []
+          : [decryptCoreFrame(outcome, index === 0 ? first.exchange : second.exchange)],
+      );
+
+      expect(authenticated).toHaveLength(1);
+      expect(authenticated[0]?.kind).toBe("welcome");
+      expect(core.store.state.connectionCount).toBe(1);
+      expect(Object.values(core.store.state.leases)).toHaveLength(1);
+      expect(core.store.state.tickets[credentialId]?.usedAt).not.toBeNull();
+    } finally {
+      await core.stop();
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  it("rejects a valid bootstrap proof completed after ticket expiry", async () => {
+    const scratch =
+      process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+      process.env.PAPERCLIP_SCRATCH_DIR ??
+      tmpdir();
+    const root = mkdtempSync(resolve(scratch, "phase3-expired-challenge-test-"));
+    const core = new Phase3MockCore({
+      stateDirectory: root,
+      identity: phase3Internals.phase3Identity(),
+      fault: "none",
+    });
+    try {
+      await core.start();
+      const ticket = core.issueBootstrapTicket(500);
+      const credentialId = phase3Internals.credentialMaterial(ticket).credentialId;
+      const pending = await beginWireAuthentication(core, ticket);
+      const expiresAtUnixMs = core.store.state.tickets[credentialId]?.expiresAtUnixMs;
+      if (expiresAtUnixMs === undefined) throw new Error("Bootstrap ticket disappeared.");
+      await delay(Math.max(1, expiresAtUnixMs - Date.now() + 10));
+      const outcome = receiveServerOutcome(pending.socket);
+
+      sendMaskedJson(pending.socket, pending.exchange.response);
+
+      await expect(outcome).resolves.toBeNull();
+      expect(core.store.state.connectionCount).toBe(0);
+      expect(Object.values(core.store.state.leases)).toHaveLength(0);
+      expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull();
+    } finally {
+      await core.stop();
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
   it("requires proof before consuming a ticket and rejects mismatched session identity", async () => {
     const scratch =
       process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
@@ -303,12 +537,75 @@ describe.sequential("Phase 3 durable transport and recovery", () => {
       });
       await expectSocketClosed(forgedProofSocket);
       expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull();
+
+      const replacedRecord = await beginWireAuthentication(core, bootstrap);
+      const ticket = core.store.state.tickets[credentialId];
+      if (ticket === undefined) throw new Error("Bootstrap ticket disappeared.");
+      core.store.state.tickets[credentialId] = {
+        ...ticket,
+        recordId: "bootstrap_ticket_replacement",
+      };
+      core.store.save();
+      const replacementOutcome = receiveServerOutcome(replacedRecord.socket);
+      sendMaskedJson(replacedRecord.socket, replacedRecord.exchange.response);
+      await expect(replacementOutcome).resolves.toBeNull();
+      expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull();
+      expect(core.store.state.connectionCount).toBe(0);
+      expect(Object.values(core.store.state.leases)).toHaveLength(0);
     } finally {
       await core.stop();
       rmSync(root, { recursive: true, force: true });
     }
   });
 
+  it.each(["expired", "revoked"] as const)(
+    "rejects a valid proof after its challenged lease is %s",
+    async (invalidatedAs) => {
+      const scratch =
+        process.env.PAPERCLIP_RUN_SCRATCH_DIR ??
+        process.env.PAPERCLIP_SCRATCH_DIR ??
+        tmpdir();
+      const root = mkdtempSync(resolve(scratch, `phase3-${invalidatedAs}-lease-test-`));
+      const core = new Phase3MockCore({
+        stateDirectory: root,
+        identity: phase3Internals.phase3Identity(),
+        fault: "none",
+      });
+      try {
+        await core.start();
+        const bootstrap = core.issueBootstrapTicket();
+        const initial = await authenticateWire(core, bootstrap);
+        const leaseToken = (initial.welcome.payload as Record<string, unknown>)
+          .connectionLeaseToken;
+        if (typeof leaseToken !== "string") throw new Error("Welcome omitted its lease token.");
+        initial.socket.destroy();
+
+        const pending = await beginWireAuthentication(core, leaseToken);
+        const credentialId = phase3Internals.credentialMaterial(leaseToken).credentialId;
+        const lease = core.store.state.leases[credentialId];
+        if (lease === undefined) throw new Error("Connection lease disappeared.");
+        if (invalidatedAs === "expired") {
+          lease.expiresAtUnixMs = Date.now() - 1;
+          lease.expiresAt = new Date(lease.expiresAtUnixMs).toISOString();
+        } else {
+          lease.revokedAt = new Date().toISOString();
+          lease.revocationEpoch += 1;
+        }
+        core.store.save();
+        const outcome = receiveServerOutcome(pending.socket);
+
+        sendMaskedJson(pending.socket, pending.exchange.response);
+
+        await expect(outcome).resolves.toBeNull();
+        expect(core.store.state.connectionCount).toBe(1);
+        expect(Object.values(core.store.state.leases)).toHaveLength(1);
+      } finally {
+        await core.stop();
+        rmSync(root, { recursive: true, force: true });
+      }
+    },
+  );
+
   it.runIf(process.platform !== "win32")(
     "atomically stores mock-core state without following a preplanted predictable symlink",
     () => {

packages/paperclip-runner/src/mock-core/phase3-recovery.ts

@@ -62,8 +62,10 @@ const secureFrameSchema = "paperclip.runner.secure-frame.v1";
 const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11";
 const coreStateSchema = "paperclip.runner.phase3.mock-core-state.v1";
 const maxFrameBytes = 1024 * 1024;
+const authChallengeTtlMs = 5_000;
 
 interface BootstrapTicketRecord {
+  recordId: string;
   credentialId: string;
   authKeyDigest: string;
   identity: Phase3Identity;
@@ -75,6 +77,7 @@ interface BootstrapTicketRecord {
 }
 
 interface ConnectionLeaseRecord {
+  recordId: string;
   credentialId: string;
   authKeyDigest: string;
   leaseId: string;
@@ -105,6 +108,33 @@ interface StoredCoreState {
 }
 
 type PendingAuthorization =
+  | {
+      kind: "bootstrap";
+      recordId: string;
+      credentialId: string;
+      authKey: Buffer;
+      identity: Phase3Identity;
+      runnerVersion: string;
+      runnerDigest: string;
+      expiresAt: string;
+      expiresAtUnixMs: number;
+      recordSnapshot: string;
+    }
+  | {
+      kind: "lease";
+      recordId: string;
+      credentialId: string;
+      authKey: Buffer;
+      identity: Phase3Identity;
+      protocolVersion: number;
+      expiresAt: string;
+      expiresAtUnixMs: number;
+      leaseId: string;
+      revocationEpoch: number;
+      recordSnapshot: string;
+    };
+
+type LiveAuthorization =
   | {
       kind: "bootstrap";
       authKey: Buffer;
@@ -118,6 +148,7 @@ type PendingAuthorization =
 
 interface PendingChallenge {
   authorization: PendingAuthorization;
+  deadlineUnixMs: number;
   canonicalChallenge: string;
   serverProof: string;
   clientNonce: string;
@@ -651,6 +682,7 @@ export class Phase3MockCore {
     const material = credentialMaterial(ticket);
     const expiresAtUnixMs = Date.now() + ttlMs;
     this.store.state.tickets[material.credentialId] = {
+      recordId: `bootstrap_ticket_${randomUUID()}`,
       credentialId: material.credentialId,
       authKeyDigest: `sha256:${material.authKey.toString("hex")}`,
       identity: structuredClone(this.identity),
@@ -779,16 +811,44 @@ export class Phase3MockCore {
     const ticket = this.store.state.tickets[credentialId];
     const lease = this.store.state.leases[credentialId];
     const authorization: PendingAuthorization | null =
-      ticket !== undefined && ticket.usedAt === null && ticket.expiresAtUnixMs > Date.now()
-        ? { kind: "bootstrap", authKey: authKeyFromDigest(ticket.authKeyDigest), ticket }
+      ticket !== undefined &&
+      typeof ticket.recordId === "string" &&
+      ticket.credentialId === credentialId &&
+      ticket.usedAt === null &&
+      ticket.expiresAtUnixMs > Date.now()
+        ? {
+            kind: "bootstrap",
+            recordId: ticket.recordId,
+            credentialId: ticket.credentialId,
+            authKey: authKeyFromDigest(ticket.authKeyDigest),
+            identity: structuredClone(ticket.identity),
+            runnerVersion: ticket.runnerVersion,
+            runnerDigest: ticket.runnerDigest,
+            expiresAt: ticket.expiresAt,
+            expiresAtUnixMs: ticket.expiresAtUnixMs,
+            recordSnapshot: canonicalJson(ticket),
+          }
         : lease !== undefined &&
+            typeof lease.recordId === "string" &&
+            lease.credentialId === credentialId &&
             lease.revokedAt === null &&
             lease.expiresAtUnixMs > Date.now()
-          ? { kind: "lease", authKey: authKeyFromDigest(lease.authKeyDigest), lease }
+          ? {
+              kind: "lease",
+              recordId: lease.recordId,
+              credentialId: lease.credentialId,
+              authKey: authKeyFromDigest(lease.authKeyDigest),
+              identity: structuredClone(lease.identity),
+              protocolVersion: lease.protocolVersion,
+              expiresAt: lease.expiresAt,
+              expiresAtUnixMs: lease.expiresAtUnixMs,
+              leaseId: lease.leaseId,
+              revocationEpoch: lease.revocationEpoch,
+              recordSnapshot: canonicalJson(lease),
+            }
           : null;
     if (authorization === null) return null;
-    const binding = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease;
-    const identity = binding.identity;
+    const identity = authorization.identity;
     if (
       payload.runnerInstanceId !== identity.runnerInstanceId ||
       payload.environmentLeaseId !== identity.environmentLeaseId ||
@@ -800,13 +860,59 @@ export class Phase3MockCore {
       payload.runnerDigest !== this.#expectedRunnerDigest ||
       payload.protocolMin !== 1 ||
       payload.protocolMax !== 1 ||
-      (authorization.kind === "lease" && authorization.lease.protocolVersion !== protocolVersion)
+      (authorization.kind === "bootstrap" &&
+        (authorization.runnerVersion !== this.#expectedRunnerVersion ||
+          authorization.runnerDigest !== this.#expectedRunnerDigest)) ||
+      (authorization.kind === "lease" && authorization.protocolVersion !== protocolVersion)
     ) {
       return null;
     }
     return authorization;
   }
 
+  #reauthorizePendingChallenge(
+    pending: PendingChallenge,
+    now: number,
+  ): LiveAuthorization | null {
+    if (pending.deadlineUnixMs <= now) return null;
+    const expected = pending.authorization;
+    if (expected.kind === "bootstrap") {
+      const ticket = this.store.state.tickets[expected.credentialId];
+      if (
+        ticket === undefined ||
+        ticket.recordId !== expected.recordId ||
+        ticket.credentialId !== expected.credentialId ||
+        ticket.usedAt !== null ||
+        ticket.expiresAtUnixMs <= now ||
+        canonicalJson(ticket) !== expected.recordSnapshot
+      ) {
+        return null;
+      }
+      return {
+        kind: "bootstrap",
+        authKey: authKeyFromDigest(ticket.authKeyDigest),
+        ticket,
+      };
+    }
+
+    const lease = this.store.state.leases[expected.credentialId];
+    if (
+      lease === undefined ||
+      lease.recordId !== expected.recordId ||
+      lease.credentialId !== expected.credentialId ||
+      lease.revokedAt !== null ||
+      lease.expiresAtUnixMs <= now ||
+      canonicalJson(lease) !== expected.recordSnapshot
+    ) {
+      return null;
+    }
+    return {
+      kind: "lease",
+      authKey: authKeyFromDigest(lease.authKeyDigest),
+      lease,
+    };
+  }
+
   #authHello(connection: MockWebSocketConnection, envelope: Record<string, unknown>): void {
     if (connection.pendingChallenge !== null) {
       connection.close();
@@ -822,10 +928,9 @@ export class Phase3MockCore {
       connection.close();
       return;
     }
-    const credential = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease;
     const serverNonce = randomUUID();
     const challengePayload: Record<string, unknown> = {
-      credentialId: credential.credentialId,
+      credentialId: authorization.credentialId,
       credentialKind: authorization.kind,
       clientNonce: payload.clientNonce,
       serverNonce,
@@ -838,10 +943,10 @@ export class Phase3MockCore {
       runnerVersion: payload.runnerVersion,
       runnerDigest: payload.runnerDigest,
       selectedVersion: protocolVersion,
-      credentialLeaseId: authorization.kind === "lease" ? authorization.lease.leaseId : null,
-      credentialExpiresAt: credential.expiresAt,
-      credentialExpiresAtUnixMs: credential.expiresAtUnixMs,
-      revocationEpoch: authorization.kind === "lease" ? authorization.lease.revocationEpoch : 0,
+      credentialLeaseId: authorization.kind === "lease" ? authorization.leaseId : null,
+      credentialExpiresAt: authorization.expiresAt,
+      credentialExpiresAtUnixMs: authorization.expiresAtUnixMs,
+      revocationEpoch: authorization.kind === "lease" ? authorization.revocationEpoch : 0,
     };
     const canonicalChallenge = canonicalJson(challengePayload);
     const serverProof = domainHmac(
@@ -851,6 +956,10 @@ export class Phase3MockCore {
     ).toString("hex");
     connection.pendingChallenge = {
       authorization,
+      deadlineUnixMs: Math.min(
+        authorization.expiresAtUnixMs,
+        Date.now() + authChallengeTtlMs,
+      ),
       canonicalChallenge,
       serverProof,
       clientNonce: payload.clientNonce,
@@ -870,18 +979,23 @@ export class Phase3MockCore {
     if (
       pending === null ||
       payload === undefined ||
-      payload.credentialId !==
-        (pending.authorization.kind === "bootstrap"
-          ? pending.authorization.ticket.credentialId
-          : pending.authorization.lease.credentialId) ||
+      payload.credentialId !== pending.authorization.credentialId ||
       payload.clientNonce !== pending.clientNonce ||
       payload.serverNonce !== pending.serverNonce
     ) {
       connection.close();
       return;
     }
+    // WebSocket callbacks run synchronously on the mock core's event loop. Re-reading,
+    // validating, consuming, minting, and persisting here forms one state mutation
+    // boundary, so another proof cannot interleave with bootstrap consumption.
+    const authorization = this.#reauthorizePendingChallenge(pending, Date.now());
+    if (authorization === null) {
+      connection.close();
+      return;
+    }
     const expectedClientProof = domainHmac(
-      pending.authorization.authKey,
+      authorization.authKey,
       "paperclip-runner-client-proof-v1",
       [Buffer.from(pending.canonicalChallenge), Buffer.from(pending.serverProof)],
     );
@@ -892,13 +1006,14 @@ export class Phase3MockCore {
     const clientProof = expectedClientProof.toString("hex");
     let leaseToken: string | null = null;
     let lease: ConnectionLeaseRecord;
-    if (pending.authorization.kind === "bootstrap") {
-      pending.authorization.ticket.usedAt = new Date().toISOString();
+    if (authorization.kind === "bootstrap") {
+      authorization.ticket.usedAt = new Date().toISOString();
       leaseToken = `lease_${randomUUID()}`;
       const material = credentialMaterial(leaseToken);
       const ttlMs = this.fault === "lease-expiry" && !this.#faultTriggered ? 50 : 30_000;
       const expiresAtUnixMs = Date.now() + ttlMs;
       lease = {
+        recordId: `connection_lease_record_${randomUUID()}`,
         credentialId: material.credentialId,
         authKeyDigest: `sha256:${material.authKey.toString("hex")}`,
         leaseId: `connection_lease_${randomUUID()}`,
@@ -910,14 +1025,15 @@ export class Phase3MockCore {
         revokedAt: null,
       };
       this.store.state.leases[material.credentialId] = lease;
+      this.store.save();
     } else {
-      lease = pending.authorization.lease;
+      lease = authorization.lease;
     }
     connection.pendingChallenge = null;
     connection.lease = lease;
     connection.connectionId = `connection_${this.store.state.connectionCount + 1}`;
     connection.secureChannel = createSecureChannel(
-      pending.authorization.authKey,
+      authorization.authKey,
       pending.canonicalChallenge,
       pending.serverProof,
       clientProof,