diff --git a/packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts b/packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts index e79bdf8c208..395a90396f0 100644 --- a/packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts +++ b/packages/paperclip-runner/src/mock-core/phase3-recovery.test.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { createDecipheriv, createHash, createHmac } from "node:crypto"; import { chmodSync, lstatSync, @@ -12,6 +13,7 @@ import { import { connect, type Socket } from "node:net"; import { tmpdir } from "node:os"; import { resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { describe, expect, it } from "vitest"; @@ -59,10 +61,23 @@ async function upgradeSocket(url: string): Promise { }); } -async function receiveServerJson(socket: Socket): Promise> { +async function receiveServerOutcome(socket: Socket): Promise | null> { return new Promise((resolveValue, rejectValue) => { let buffer = Buffer.alloc(0); - const timer = setTimeout(() => rejectValue(new Error("Server frame timed out.")), 2_000); + const cleanup = (): void => { + clearTimeout(timer); + socket.off("data", onData); + socket.off("close", onClose); + socket.off("error", onError); + }; + const onClose = (): void => { + cleanup(); + resolveValue(null); + }; + const onError = (error: Error): void => { + cleanup(); + rejectValue(error); + }; const onData = (chunk: Buffer): void => { buffer = Buffer.concat([buffer, chunk]); if (buffer.length < 2) return; @@ -78,8 +93,7 @@ async function receiveServerJson(socket: Socket): Promise, ); }; + const timer = setTimeout(() => { + cleanup(); + rejectValue(new Error("Server frame timed out.")); + }, 2_000); socket.on("data", onData); - socket.once("error", rejectValue); + socket.once("close", onClose); + socket.once("error", onError); }); } +async function receiveServerJson(socket: Socket): Promise> { + const value = await receiveServerOutcome(socket); + if (value === null) throw new Error("WebSocket closed before the server frame arrived."); + return value; +} + function sendMaskedJson(socket: Socket, value: unknown): void { const payload = Buffer.from(JSON.stringify(value)); const mask = Buffer.from([0x11, 0x22, 0x33, 0x44]); @@ -109,6 +134,7 @@ function sendMaskedJson(socket: Socket, value: unknown): void { } async function expectSocketClosed(socket: Socket): Promise { + if (socket.destroyed) return; await new Promise((resolveClose, rejectClose) => { const timer = setTimeout(() => rejectClose(new Error("WebSocket was not rejected.")), 2_000); socket.once("close", () => { @@ -143,6 +169,139 @@ function authHello( }; } +function domainDigest(domain: string, parts: readonly Buffer[]): Buffer { + const digest = createHash("sha256").update(domain).update(Buffer.from([0])); + for (const part of parts) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(part.length)); + digest.update(length).update(part); + } + return digest.digest(); +} + +function domainHmac(key: Buffer, domain: string, parts: readonly Buffer[]): Buffer { + const digest = createHmac("sha256", key).update(domain).update(Buffer.from([0])); + for (const part of parts) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(part.length)); + digest.update(length).update(part); + } + return digest.digest(); +} + +interface AuthExchange { + response: Record; + authKey: Buffer; + canonicalChallenge: string; + serverProof: string; + clientProof: string; +} + +function validAuthResponse( + challenge: Record, + token: string, +): AuthExchange { + const challengePayload = challenge.payload as Record; + const serverProof = challengePayload.serverProof; + if (typeof serverProof !== "string") throw new Error("Challenge omitted its server proof."); + const canonicalPayload = { ...challengePayload }; + delete canonicalPayload.serverProof; + const canonicalChallenge = phase3Internals.canonicalJson(canonicalPayload); + const material = phase3Internals.credentialMaterial(token); + const expectedServerProof = domainHmac( + material.authKey, + "paperclip-runner-server-proof-v1", + [Buffer.from(canonicalChallenge)], + ).toString("hex"); + if (serverProof !== expectedServerProof) throw new Error("Challenge server proof was invalid."); + const clientProof = domainHmac( + material.authKey, + "paperclip-runner-client-proof-v1", + [Buffer.from(canonicalChallenge), Buffer.from(serverProof)], + ).toString("hex"); + return { + response: { + protocol: "paperclip.runner", + version: 1, + kind: "auth_response", + payload: { + credentialId: challengePayload.credentialId, + clientNonce: challengePayload.clientNonce, + serverNonce: challengePayload.serverNonce, + clientProof, + }, + }, + authKey: material.authKey, + canonicalChallenge, + serverProof, + clientProof, + }; +} + +function decryptCoreFrame( + frame: Record, + exchange: AuthExchange, +): Record { + if ( + frame.schema !== "paperclip.runner.secure-frame.v1" || + typeof frame.counter !== "number" || + typeof frame.ciphertext !== "string" + ) { + throw new Error("Expected an encrypted core frame."); + } + const counter = BigInt(frame.counter); + const binding = domainDigest("paperclip-runner-session-binding-v1", [ + Buffer.from(exchange.canonicalChallenge), + Buffer.from(exchange.serverProof), + Buffer.from(exchange.clientProof), + ]); + const receiveKey = domainHmac( + exchange.authKey, + "paperclip-runner-core-to-client-key-v1", + [binding], + ); + const nonce = Buffer.alloc(12); + nonce.write("P3S1", 0, "ascii"); + nonce.writeBigUInt64BE(counter, 4); + const sessionId = `sha256:${binding.toString("hex")}`; + const aad = Buffer.from( + `paperclip.runner.secure-frame.v1\0${sessionId}\0core_to_client\0${counter}`, + ); + const sealed = Buffer.from(frame.ciphertext, "hex"); + const decipher = createDecipheriv("aes-256-gcm", receiveKey, nonce); + decipher.setAAD(aad); + decipher.setAuthTag(sealed.subarray(-16)); + return JSON.parse( + Buffer.concat([decipher.update(sealed.subarray(0, -16)), decipher.final()]).toString( + "utf8", + ), + ) as Record; +} + +async function beginWireAuthentication( + core: Phase3MockCore, + token: string, +): Promise<{ socket: Socket; exchange: AuthExchange }> { + const socket = await upgradeSocket(core.connectUrl); + const credentialId = phase3Internals.credentialMaterial(token).credentialId; + sendMaskedJson(socket, authHello(core.identity, credentialId)); + const challenge = await receiveServerJson(socket); + expect(challenge.kind).toBe("auth_challenge"); + return { socket, exchange: validAuthResponse(challenge, token) }; +} + +async function authenticateWire( + core: Phase3MockCore, + token: string, +): Promise<{ socket: Socket; welcome: Record }> { + const { socket, exchange } = await beginWireAuthentication(core, token); + const framePromise = receiveServerJson(socket); + sendMaskedJson(socket, exchange.response); + const welcome = decryptCoreFrame(await framePromise, exchange); + expect(welcome.kind).toBe("welcome"); + return { socket, welcome }; +} + async function runRunnerProcess(options: { connectUrl: string; stateDirectory: string; @@ -262,6 +421,81 @@ describe.sequential("Phase 3 durable transport and recovery", () => { } }); + it("authenticates only one of two valid proofs for the same bootstrap ticket", async () => { + const scratch = + process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? + process.env.PAPERCLIP_SCRATCH_DIR ?? + tmpdir(); + const root = mkdtempSync(resolve(scratch, "phase3-concurrent-ticket-test-")); + const core = new Phase3MockCore({ + stateDirectory: root, + identity: phase3Internals.phase3Identity(), + fault: "none", + }); + try { + await core.start(); + const ticket = core.issueBootstrapTicket(); + const credentialId = phase3Internals.credentialMaterial(ticket).credentialId; + const [first, second] = await Promise.all([ + beginWireAuthentication(core, ticket), + beginWireAuthentication(core, ticket), + ]); + const firstOutcome = receiveServerOutcome(first.socket); + const secondOutcome = receiveServerOutcome(second.socket); + + sendMaskedJson(first.socket, first.exchange.response); + sendMaskedJson(second.socket, second.exchange.response); + const outcomes = await Promise.all([firstOutcome, secondOutcome]); + const authenticated = outcomes.flatMap((outcome, index) => + outcome === null + ? [] + : [decryptCoreFrame(outcome, index === 0 ? first.exchange : second.exchange)], + ); + + expect(authenticated).toHaveLength(1); + expect(authenticated[0]?.kind).toBe("welcome"); + expect(core.store.state.connectionCount).toBe(1); + expect(Object.values(core.store.state.leases)).toHaveLength(1); + expect(core.store.state.tickets[credentialId]?.usedAt).not.toBeNull(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects a valid bootstrap proof completed after ticket expiry", async () => { + const scratch = + process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? + process.env.PAPERCLIP_SCRATCH_DIR ?? + tmpdir(); + const root = mkdtempSync(resolve(scratch, "phase3-expired-challenge-test-")); + const core = new Phase3MockCore({ + stateDirectory: root, + identity: phase3Internals.phase3Identity(), + fault: "none", + }); + try { + await core.start(); + const ticket = core.issueBootstrapTicket(500); + const credentialId = phase3Internals.credentialMaterial(ticket).credentialId; + const pending = await beginWireAuthentication(core, ticket); + const expiresAtUnixMs = core.store.state.tickets[credentialId]?.expiresAtUnixMs; + if (expiresAtUnixMs === undefined) throw new Error("Bootstrap ticket disappeared."); + await delay(Math.max(1, expiresAtUnixMs - Date.now() + 10)); + const outcome = receiveServerOutcome(pending.socket); + + sendMaskedJson(pending.socket, pending.exchange.response); + + await expect(outcome).resolves.toBeNull(); + expect(core.store.state.connectionCount).toBe(0); + expect(Object.values(core.store.state.leases)).toHaveLength(0); + expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull(); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); + it("requires proof before consuming a ticket and rejects mismatched session identity", async () => { const scratch = process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? @@ -303,12 +537,75 @@ describe.sequential("Phase 3 durable transport and recovery", () => { }); await expectSocketClosed(forgedProofSocket); expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull(); + + const replacedRecord = await beginWireAuthentication(core, bootstrap); + const ticket = core.store.state.tickets[credentialId]; + if (ticket === undefined) throw new Error("Bootstrap ticket disappeared."); + core.store.state.tickets[credentialId] = { + ...ticket, + recordId: "bootstrap_ticket_replacement", + }; + core.store.save(); + const replacementOutcome = receiveServerOutcome(replacedRecord.socket); + sendMaskedJson(replacedRecord.socket, replacedRecord.exchange.response); + await expect(replacementOutcome).resolves.toBeNull(); + expect(core.store.state.tickets[credentialId]?.usedAt).toBeNull(); + expect(core.store.state.connectionCount).toBe(0); + expect(Object.values(core.store.state.leases)).toHaveLength(0); } finally { await core.stop(); rmSync(root, { recursive: true, force: true }); } }); + it.each(["expired", "revoked"] as const)( + "rejects a valid proof after its challenged lease is %s", + async (invalidatedAs) => { + const scratch = + process.env.PAPERCLIP_RUN_SCRATCH_DIR ?? + process.env.PAPERCLIP_SCRATCH_DIR ?? + tmpdir(); + const root = mkdtempSync(resolve(scratch, `phase3-${invalidatedAs}-lease-test-`)); + const core = new Phase3MockCore({ + stateDirectory: root, + identity: phase3Internals.phase3Identity(), + fault: "none", + }); + try { + await core.start(); + const bootstrap = core.issueBootstrapTicket(); + const initial = await authenticateWire(core, bootstrap); + const leaseToken = (initial.welcome.payload as Record) + .connectionLeaseToken; + if (typeof leaseToken !== "string") throw new Error("Welcome omitted its lease token."); + initial.socket.destroy(); + + const pending = await beginWireAuthentication(core, leaseToken); + const credentialId = phase3Internals.credentialMaterial(leaseToken).credentialId; + const lease = core.store.state.leases[credentialId]; + if (lease === undefined) throw new Error("Connection lease disappeared."); + if (invalidatedAs === "expired") { + lease.expiresAtUnixMs = Date.now() - 1; + lease.expiresAt = new Date(lease.expiresAtUnixMs).toISOString(); + } else { + lease.revokedAt = new Date().toISOString(); + lease.revocationEpoch += 1; + } + core.store.save(); + const outcome = receiveServerOutcome(pending.socket); + + sendMaskedJson(pending.socket, pending.exchange.response); + + await expect(outcome).resolves.toBeNull(); + expect(core.store.state.connectionCount).toBe(1); + expect(Object.values(core.store.state.leases)).toHaveLength(1); + } finally { + await core.stop(); + rmSync(root, { recursive: true, force: true }); + } + }, + ); + it.runIf(process.platform !== "win32")( "atomically stores mock-core state without following a preplanted predictable symlink", () => { diff --git a/packages/paperclip-runner/src/mock-core/phase3-recovery.ts b/packages/paperclip-runner/src/mock-core/phase3-recovery.ts index b5021b607f8..88641bc5609 100644 --- a/packages/paperclip-runner/src/mock-core/phase3-recovery.ts +++ b/packages/paperclip-runner/src/mock-core/phase3-recovery.ts @@ -62,8 +62,10 @@ const secureFrameSchema = "paperclip.runner.secure-frame.v1"; const websocketGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; const coreStateSchema = "paperclip.runner.phase3.mock-core-state.v1"; const maxFrameBytes = 1024 * 1024; +const authChallengeTtlMs = 5_000; interface BootstrapTicketRecord { + recordId: string; credentialId: string; authKeyDigest: string; identity: Phase3Identity; @@ -75,6 +77,7 @@ interface BootstrapTicketRecord { } interface ConnectionLeaseRecord { + recordId: string; credentialId: string; authKeyDigest: string; leaseId: string; @@ -105,6 +108,33 @@ interface StoredCoreState { } type PendingAuthorization = + | { + kind: "bootstrap"; + recordId: string; + credentialId: string; + authKey: Buffer; + identity: Phase3Identity; + runnerVersion: string; + runnerDigest: string; + expiresAt: string; + expiresAtUnixMs: number; + recordSnapshot: string; + } + | { + kind: "lease"; + recordId: string; + credentialId: string; + authKey: Buffer; + identity: Phase3Identity; + protocolVersion: number; + expiresAt: string; + expiresAtUnixMs: number; + leaseId: string; + revocationEpoch: number; + recordSnapshot: string; + }; + +type LiveAuthorization = | { kind: "bootstrap"; authKey: Buffer; @@ -118,6 +148,7 @@ type PendingAuthorization = interface PendingChallenge { authorization: PendingAuthorization; + deadlineUnixMs: number; canonicalChallenge: string; serverProof: string; clientNonce: string; @@ -651,6 +682,7 @@ export class Phase3MockCore { const material = credentialMaterial(ticket); const expiresAtUnixMs = Date.now() + ttlMs; this.store.state.tickets[material.credentialId] = { + recordId: `bootstrap_ticket_${randomUUID()}`, credentialId: material.credentialId, authKeyDigest: `sha256:${material.authKey.toString("hex")}`, identity: structuredClone(this.identity), @@ -779,16 +811,44 @@ export class Phase3MockCore { const ticket = this.store.state.tickets[credentialId]; const lease = this.store.state.leases[credentialId]; const authorization: PendingAuthorization | null = - ticket !== undefined && ticket.usedAt === null && ticket.expiresAtUnixMs > Date.now() - ? { kind: "bootstrap", authKey: authKeyFromDigest(ticket.authKeyDigest), ticket } + ticket !== undefined && + typeof ticket.recordId === "string" && + ticket.credentialId === credentialId && + ticket.usedAt === null && + ticket.expiresAtUnixMs > Date.now() + ? { + kind: "bootstrap", + recordId: ticket.recordId, + credentialId: ticket.credentialId, + authKey: authKeyFromDigest(ticket.authKeyDigest), + identity: structuredClone(ticket.identity), + runnerVersion: ticket.runnerVersion, + runnerDigest: ticket.runnerDigest, + expiresAt: ticket.expiresAt, + expiresAtUnixMs: ticket.expiresAtUnixMs, + recordSnapshot: canonicalJson(ticket), + } : lease !== undefined && + typeof lease.recordId === "string" && + lease.credentialId === credentialId && lease.revokedAt === null && lease.expiresAtUnixMs > Date.now() - ? { kind: "lease", authKey: authKeyFromDigest(lease.authKeyDigest), lease } + ? { + kind: "lease", + recordId: lease.recordId, + credentialId: lease.credentialId, + authKey: authKeyFromDigest(lease.authKeyDigest), + identity: structuredClone(lease.identity), + protocolVersion: lease.protocolVersion, + expiresAt: lease.expiresAt, + expiresAtUnixMs: lease.expiresAtUnixMs, + leaseId: lease.leaseId, + revocationEpoch: lease.revocationEpoch, + recordSnapshot: canonicalJson(lease), + } : null; if (authorization === null) return null; - const binding = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease; - const identity = binding.identity; + const identity = authorization.identity; if ( payload.runnerInstanceId !== identity.runnerInstanceId || payload.environmentLeaseId !== identity.environmentLeaseId || @@ -800,13 +860,59 @@ export class Phase3MockCore { payload.runnerDigest !== this.#expectedRunnerDigest || payload.protocolMin !== 1 || payload.protocolMax !== 1 || - (authorization.kind === "lease" && authorization.lease.protocolVersion !== protocolVersion) + (authorization.kind === "bootstrap" && + (authorization.runnerVersion !== this.#expectedRunnerVersion || + authorization.runnerDigest !== this.#expectedRunnerDigest)) || + (authorization.kind === "lease" && authorization.protocolVersion !== protocolVersion) ) { return null; } return authorization; } + #reauthorizePendingChallenge( + pending: PendingChallenge, + now: number, + ): LiveAuthorization | null { + if (pending.deadlineUnixMs <= now) return null; + const expected = pending.authorization; + if (expected.kind === "bootstrap") { + const ticket = this.store.state.tickets[expected.credentialId]; + if ( + ticket === undefined || + ticket.recordId !== expected.recordId || + ticket.credentialId !== expected.credentialId || + ticket.usedAt !== null || + ticket.expiresAtUnixMs <= now || + canonicalJson(ticket) !== expected.recordSnapshot + ) { + return null; + } + return { + kind: "bootstrap", + authKey: authKeyFromDigest(ticket.authKeyDigest), + ticket, + }; + } + + const lease = this.store.state.leases[expected.credentialId]; + if ( + lease === undefined || + lease.recordId !== expected.recordId || + lease.credentialId !== expected.credentialId || + lease.revokedAt !== null || + lease.expiresAtUnixMs <= now || + canonicalJson(lease) !== expected.recordSnapshot + ) { + return null; + } + return { + kind: "lease", + authKey: authKeyFromDigest(lease.authKeyDigest), + lease, + }; + } + #authHello(connection: MockWebSocketConnection, envelope: Record): void { if (connection.pendingChallenge !== null) { connection.close(); @@ -822,10 +928,9 @@ export class Phase3MockCore { connection.close(); return; } - const credential = authorization.kind === "bootstrap" ? authorization.ticket : authorization.lease; const serverNonce = randomUUID(); const challengePayload: Record = { - credentialId: credential.credentialId, + credentialId: authorization.credentialId, credentialKind: authorization.kind, clientNonce: payload.clientNonce, serverNonce, @@ -838,10 +943,10 @@ export class Phase3MockCore { runnerVersion: payload.runnerVersion, runnerDigest: payload.runnerDigest, selectedVersion: protocolVersion, - credentialLeaseId: authorization.kind === "lease" ? authorization.lease.leaseId : null, - credentialExpiresAt: credential.expiresAt, - credentialExpiresAtUnixMs: credential.expiresAtUnixMs, - revocationEpoch: authorization.kind === "lease" ? authorization.lease.revocationEpoch : 0, + credentialLeaseId: authorization.kind === "lease" ? authorization.leaseId : null, + credentialExpiresAt: authorization.expiresAt, + credentialExpiresAtUnixMs: authorization.expiresAtUnixMs, + revocationEpoch: authorization.kind === "lease" ? authorization.revocationEpoch : 0, }; const canonicalChallenge = canonicalJson(challengePayload); const serverProof = domainHmac( @@ -851,6 +956,10 @@ export class Phase3MockCore { ).toString("hex"); connection.pendingChallenge = { authorization, + deadlineUnixMs: Math.min( + authorization.expiresAtUnixMs, + Date.now() + authChallengeTtlMs, + ), canonicalChallenge, serverProof, clientNonce: payload.clientNonce, @@ -870,18 +979,23 @@ export class Phase3MockCore { if ( pending === null || payload === undefined || - payload.credentialId !== - (pending.authorization.kind === "bootstrap" - ? pending.authorization.ticket.credentialId - : pending.authorization.lease.credentialId) || + payload.credentialId !== pending.authorization.credentialId || payload.clientNonce !== pending.clientNonce || payload.serverNonce !== pending.serverNonce ) { connection.close(); return; } + // WebSocket callbacks run synchronously on the mock core's event loop. Re-reading, + // validating, consuming, minting, and persisting here forms one state mutation + // boundary, so another proof cannot interleave with bootstrap consumption. + const authorization = this.#reauthorizePendingChallenge(pending, Date.now()); + if (authorization === null) { + connection.close(); + return; + } const expectedClientProof = domainHmac( - pending.authorization.authKey, + authorization.authKey, "paperclip-runner-client-proof-v1", [Buffer.from(pending.canonicalChallenge), Buffer.from(pending.serverProof)], ); @@ -892,13 +1006,14 @@ export class Phase3MockCore { const clientProof = expectedClientProof.toString("hex"); let leaseToken: string | null = null; let lease: ConnectionLeaseRecord; - if (pending.authorization.kind === "bootstrap") { - pending.authorization.ticket.usedAt = new Date().toISOString(); + if (authorization.kind === "bootstrap") { + authorization.ticket.usedAt = new Date().toISOString(); leaseToken = `lease_${randomUUID()}`; const material = credentialMaterial(leaseToken); const ttlMs = this.fault === "lease-expiry" && !this.#faultTriggered ? 50 : 30_000; const expiresAtUnixMs = Date.now() + ttlMs; lease = { + recordId: `connection_lease_record_${randomUUID()}`, credentialId: material.credentialId, authKeyDigest: `sha256:${material.authKey.toString("hex")}`, leaseId: `connection_lease_${randomUUID()}`, @@ -910,14 +1025,15 @@ export class Phase3MockCore { revokedAt: null, }; this.store.state.leases[material.credentialId] = lease; + this.store.save(); } else { - lease = pending.authorization.lease; + lease = authorization.lease; } connection.pendingChallenge = null; connection.lease = lease; connection.connectionId = `connection_${this.store.state.connectionCount + 1}`; connection.secureChannel = createSecureChannel( - pending.authorization.authKey, + authorization.authKey, pending.canonicalChallenge, pending.serverProof, clientProof,