From 7ac1686b09396680d296eec8e61c9fb3e5f81ed0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 11 Aug 2026 17:30:31 +0000 Subject: [PATCH] feat(runtime-exposure): server-side broker client + exposure lifecycle manager (PAP-17051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the server-side layer of the tailscale_https exposure mode as a self-contained module so it is testable without a live broker, Tailscale, or the host-privileged broker package, and without churning the large concurrently-held workspace-runtime.ts. - broker-client.ts: Unix-socket client speaking the broker's versioned, length-prefixed, byte-bounded wire protocol (expose/remove/list). Injectable connector for tests; never logs lease handles. - port-pair.ts: paired app + Vite HMR companion allocation, only from the dedicated allowlisted range, over an injected availability probe. - exposure-manager.ts: fail-closed lifecycle state machine (pending/ready/ failed/cleanup_pending/removed) over injected broker + HTTPS health probe. Idempotent provision/deprovision with port quarantine on ambiguous cleanup; reconcile adopts owned+desired mappings and removes only owned orphans it holds a handle for — never an unknown/manual mapping or the primary :443. - Tests: 29 cases (broker framing/parse/timeout, port-pair scan/skip/exhaust, provision/deprovision/reconcile incl. proof reconcile never mutates a handle-less orphan). - Backfill exposure:null in the read-model test fixture for the DB column added in 262695ba74. Co-Authored-By: Claude Opus 4.8 --- .../runtime-exposure/broker-client.test.ts | 194 +++++++++++ .../runtime-exposure/broker-client.ts | 268 +++++++++++++++ .../runtime-exposure/exposure-manager.test.ts | 317 ++++++++++++++++++ .../runtime-exposure/exposure-manager.ts | 311 +++++++++++++++++ .../runtime-exposure/port-pair.test.ts | 65 ++++ .../services/runtime-exposure/port-pair.ts | 64 ++++ .../workspace-runtime-read-model.test.ts | 1 + 7 files changed, 1220 insertions(+) create mode 100644 server/src/services/runtime-exposure/broker-client.test.ts create mode 100644 server/src/services/runtime-exposure/broker-client.ts create mode 100644 server/src/services/runtime-exposure/exposure-manager.test.ts create mode 100644 server/src/services/runtime-exposure/exposure-manager.ts create mode 100644 server/src/services/runtime-exposure/port-pair.test.ts create mode 100644 server/src/services/runtime-exposure/port-pair.ts diff --git a/server/src/services/runtime-exposure/broker-client.test.ts b/server/src/services/runtime-exposure/broker-client.test.ts new file mode 100644 index 00000000000..fa31caaf501 --- /dev/null +++ b/server/src/services/runtime-exposure/broker-client.test.ts @@ -0,0 +1,194 @@ +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { existsSync, unlinkSync } from "node:fs"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { BrokerClientError, UnixBrokerClient } from "./broker-client.js"; + +const LENGTH_PREFIX_BYTES = 4; + +/** + * Minimal in-process broker that speaks the exact `[4-byte BE length][json]` + * framing of the real socket server, so these tests exercise the client's + * framing + parsing without linking the host-privileged broker package. + */ +function startFakeBroker( + handler: (request: Record) => unknown, +): { socketPath: string; server: net.Server } { + const socketPath = path.join(os.tmpdir(), `tsh-broker-${randomUUID()}.sock`); + const server = net.createServer((socket) => { + const chunks: Buffer[] = []; + let expected = -1; + socket.on("data", (chunk) => { + chunks.push(chunk); + const buf = Buffer.concat(chunks); + if (expected < 0) { + if (buf.byteLength < LENGTH_PREFIX_BYTES) return; + expected = buf.readUInt32BE(0); + } + if (buf.byteLength < LENGTH_PREFIX_BYTES + expected) return; + const body = buf.subarray(LENGTH_PREFIX_BYTES, LENGTH_PREFIX_BYTES + expected).toString("utf8"); + const request = JSON.parse(body) as Record; + const response = handler(request); + const respBody = Buffer.from(JSON.stringify(response), "utf8"); + const frame = Buffer.allocUnsafe(LENGTH_PREFIX_BYTES + respBody.byteLength); + frame.writeUInt32BE(respBody.byteLength, 0); + respBody.copy(frame, LENGTH_PREFIX_BYTES); + socket.end(frame); + }); + }); + server.listen(socketPath); + return { socketPath, server }; +} + +const servers: net.Server[] = []; +const sockets: string[] = []; + +function fake(handler: (request: Record) => unknown): string { + const { socketPath, server } = startFakeBroker(handler); + servers.push(server); + sockets.push(socketPath); + return socketPath; +} + +afterEach(async () => { + for (const server of servers.splice(0)) { + await new Promise((resolve) => server.close(() => resolve())); + } + for (const socketPath of sockets.splice(0)) { + if (existsSync(socketPath)) { + try { + unlinkSync(socketPath); + } catch { + /* ignore */ + } + } + } +}); + +const RUNTIME_ID = "11111111-2222-4333-8444-555566667777"; + +describe("UnixBrokerClient", () => { + it("round-trips an expose request and parses handle + ports", async () => { + let seen: Record | undefined; + const socketPath = fake((request) => { + seen = request; + return { ok: true, op: "expose", requestId: request.requestId, handle: "h".repeat(20), publicPorts: [42000, 52000] }; + }); + const client = new UnixBrokerClient({ socketPath }); + const result = await client.expose(RUNTIME_ID, [ + { purpose: "app", port: 42000 }, + { purpose: "vite_hmr", port: 52000 }, + ]); + expect(result).toEqual({ handle: "h".repeat(20), publicPorts: [42000, 52000] }); + expect(seen).toMatchObject({ + v: 1, + op: "expose", + runtimeId: RUNTIME_ID, + listeners: [ + { purpose: "app", port: 42000 }, + { purpose: "vite_hmr", port: 52000 }, + ], + }); + expect(typeof seen?.requestId).toBe("string"); + }); + + it("parses a remove response", async () => { + const socketPath = fake((request) => ({ + ok: true, + op: "remove", + requestId: request.requestId, + removedPorts: [42000], + })); + const client = new UnixBrokerClient({ socketPath }); + const result = await client.remove(RUNTIME_ID, "h".repeat(20)); + expect(result.removedPorts).toEqual([42000]); + }); + + it("parses a list response into owned listeners", async () => { + const socketPath = fake((request) => ({ + ok: true, + op: "list", + requestId: request.requestId, + listeners: [{ runtimeId: RUNTIME_ID, port: 42000, purpose: "app" }], + })); + const client = new UnixBrokerClient({ socketPath }); + const listeners = await client.list(); + expect(listeners).toEqual([{ runtimeId: RUNTIME_ID, port: 42000, purpose: "app" }]); + }); + + it("surfaces a broker error response as a coded BrokerClientError", async () => { + const socketPath = fake((request) => ({ + ok: false, + requestId: request.requestId, + code: "port_not_allowlisted", + message: "denied", + })); + const client = new UnixBrokerClient({ socketPath }); + await expect(client.expose(RUNTIME_ID, [{ purpose: "app", port: 42000 }])).rejects.toMatchObject({ + name: "BrokerClientError", + code: "port_not_allowlisted", + }); + }); + + it("rejects a malformed (non-ok) response shape", async () => { + const socketPath = fake(() => ({ surprise: true })); + const client = new UnixBrokerClient({ socketPath }); + await expect(client.list()).rejects.toBeInstanceOf(BrokerClientError); + }); + + it("rejects an expose response missing publicPorts", async () => { + const socketPath = fake((request) => ({ ok: true, op: "expose", requestId: request.requestId, handle: "h".repeat(20) })); + const client = new UnixBrokerClient({ socketPath }); + await expect(client.expose(RUNTIME_ID, [{ purpose: "app", port: 42000 }])).rejects.toMatchObject({ + code: "malformed_response", + }); + }); + + it("times out when the broker accepts but never responds", async () => { + await expect(silentTimeout()).resolves.toBe(true); + }); + + it("errors when the socket path does not exist (transport error)", async () => { + const client = new UnixBrokerClient({ + socketPath: path.join(os.tmpdir(), `missing-${randomUUID()}.sock`), + timeoutMs: 500, + }); + await expect(client.list()).rejects.toMatchObject({ code: "transport_error" }); + }); +}); + +/** Build a truly-silent broker and assert the client times out against it. */ +async function silentTimeout(): Promise { + const socketPath = path.join(os.tmpdir(), `silent-${randomUUID()}.sock`); + const connections: net.Socket[] = []; + const server = net.createServer((socket) => { + // Accept, never respond. Swallow the abortive close the client sends when + // it times out so the connection socket does not emit an unhandled error. + socket.on("error", () => {}); + connections.push(socket); + }); + await new Promise((resolve) => server.listen(socketPath, () => resolve())); + try { + const client = new UnixBrokerClient({ socketPath, timeoutMs: 150 }); + try { + await client.list(); + return false; + } catch (error) { + return error instanceof BrokerClientError && error.code === "cli_timeout"; + } + } finally { + for (const socket of connections) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + if (existsSync(socketPath)) { + try { + unlinkSync(socketPath); + } catch { + /* ignore */ + } + } + } +} diff --git a/server/src/services/runtime-exposure/broker-client.ts b/server/src/services/runtime-exposure/broker-client.ts new file mode 100644 index 00000000000..91ce244fe9b --- /dev/null +++ b/server/src/services/runtime-exposure/broker-client.ts @@ -0,0 +1,268 @@ +/** + * Server-side client for the least-privilege Tailscale HTTPS host broker. + * + * The Paperclip server process holds NO Tailscale operator authority. Its only + * way to publish a same-number HTTPS-to-loopback listener is to ask the broker + * over its Unix domain socket. This client speaks the exact versioned, + * length-prefixed, byte-bounded wire protocol implemented by the broker's + * socket server (PAP-17049 plan, PAP-17050 verdict requirement #5). + * + * Framing (must match `packages/tailscale-https-broker/src/socket-server.ts`): + * [4-byte big-endian body length][utf-8 JSON body] + * One request per connection; the broker writes a single framed response and + * closes the socket. + * + * Everything the broker returns is untrusted: the client validates the response + * shape before handing it back and never logs lease handles. + */ +import net from "node:net"; + +/** + * Must match the broker's `BROKER_PROTOCOL_VERSION`. Kept as a local constant + * (rather than a dependency on the host-side broker package, which is installed + * and versioned separately under its own operator account) so the server does + * not import host-privileged code just to speak the wire protocol. + */ +const BROKER_PROTOCOL_VERSION = 1; + +/** Must match the broker's `MAX_REQUEST_BYTES` (8 KiB). */ +const MAX_REQUEST_BYTES = 8 * 1024; +/** Must match the broker's `MAX_RESPONSE_BYTES` (16 KiB). */ +const MAX_RESPONSE_BYTES = 16 * 1024; +const LENGTH_PREFIX_BYTES = 4; + +/** Listener purposes the broker understands. */ +export type BrokerListenerPurpose = "app" | "vite_hmr"; + +export interface BrokerListenerRequest { + purpose: BrokerListenerPurpose; + /** Public HTTPS port == loopback target port (same-number invariant). */ + port: number; +} + +export interface BrokerExposeResult { + handle: string; + publicPorts: number[]; +} + +export interface BrokerRemoveResult { + removedPorts: number[]; +} + +export interface BrokerOwnedListener { + runtimeId: string; + port: number; + purpose: BrokerListenerPurpose; +} + +/** + * Error surfaced to the exposure manager. `code` is the broker's stable machine + * code (or a transport-level code) so lifecycle logic can branch without + * string-matching human messages. + */ +export class BrokerClientError extends Error { + constructor( + readonly code: string, + message: string, + ) { + super(message); + this.name = "BrokerClientError"; + } +} + +/** + * Minimal broker capability surface the exposure manager depends on. Defining + * it as an interface lets tests inject a fake broker with zero sockets. + */ +export interface BrokerClient { + expose(runtimeId: string, listeners: BrokerListenerRequest[]): Promise; + remove(runtimeId: string, handle: string): Promise; + list(): Promise; +} + +export interface UnixBrokerClientOptions { + socketPath: string; + /** Per-request connect+round-trip deadline. Defaults to 5s. */ + timeoutMs?: number; + /** + * Seam for tests: open a duplex stream to the broker. Defaults to a real Unix + * socket connection. Any injected connector must behave like a `net.Socket`. + */ + connect?: (socketPath: string) => net.Socket; +} + +let requestCounter = 0; + +function nextRequestId(): string { + requestCounter = (requestCounter + 1) % 1_000_000; + // Matches the broker's REQUEST_ID_RE (`[A-Za-z0-9_-]{1,64}`). No time/random + // needed for correctness; the broker keys responses by requestId echo only. + return `req-${requestCounter.toString(36)}`; +} + +/** + * Unix-domain-socket implementation of {@link BrokerClient}. Each call opens a + * fresh connection, sends one length-prefixed frame, reads one length-prefixed + * response frame, and closes. + */ +export class UnixBrokerClient implements BrokerClient { + private readonly socketPath: string; + private readonly timeoutMs: number; + private readonly connect: (socketPath: string) => net.Socket; + + constructor(options: UnixBrokerClientOptions) { + this.socketPath = options.socketPath; + this.timeoutMs = options.timeoutMs ?? 5_000; + this.connect = options.connect ?? ((p) => net.createConnection(p)); + } + + async expose( + runtimeId: string, + listeners: BrokerListenerRequest[], + ): Promise { + const response = await this.roundTrip({ + v: BROKER_PROTOCOL_VERSION, + op: "expose", + requestId: nextRequestId(), + runtimeId, + listeners: listeners.map((l) => ({ purpose: l.purpose, port: l.port })), + }); + if ( + typeof response.handle !== "string" || + !Array.isArray(response.publicPorts) || + !response.publicPorts.every((p) => Number.isInteger(p)) + ) { + throw new BrokerClientError("malformed_response", "expose response missing handle/publicPorts"); + } + return { handle: response.handle, publicPorts: response.publicPorts as number[] }; + } + + async remove(runtimeId: string, handle: string): Promise { + const response = await this.roundTrip({ + v: BROKER_PROTOCOL_VERSION, + op: "remove", + requestId: nextRequestId(), + runtimeId, + handle, + }); + if (!Array.isArray(response.removedPorts) || !response.removedPorts.every((p) => Number.isInteger(p))) { + throw new BrokerClientError("malformed_response", "remove response missing removedPorts"); + } + return { removedPorts: response.removedPorts as number[] }; + } + + async list(): Promise { + const response = await this.roundTrip({ + v: BROKER_PROTOCOL_VERSION, + op: "list", + requestId: nextRequestId(), + }); + if (!Array.isArray(response.listeners)) { + throw new BrokerClientError("malformed_response", "list response missing listeners"); + } + return response.listeners.map((entry): BrokerOwnedListener => { + const listener = entry as Record; + if ( + typeof listener.runtimeId !== "string" || + !Number.isInteger(listener.port) || + (listener.purpose !== "app" && listener.purpose !== "vite_hmr") + ) { + throw new BrokerClientError("malformed_response", "list returned a malformed listener"); + } + return { + runtimeId: listener.runtimeId, + port: listener.port as number, + purpose: listener.purpose, + }; + }); + } + + /** Send one request frame, resolve the single framed response as an object. */ + private roundTrip(request: unknown): Promise> { + const body = Buffer.from(JSON.stringify(request), "utf8"); + if (body.byteLength > MAX_REQUEST_BYTES) { + return Promise.reject(new BrokerClientError("request_too_large", "broker request exceeds size limit")); + } + const frame = Buffer.allocUnsafe(LENGTH_PREFIX_BYTES + body.byteLength); + frame.writeUInt32BE(body.byteLength, 0); + body.copy(frame, LENGTH_PREFIX_BYTES); + + return new Promise((resolve, reject) => { + const socket = this.connect(this.socketPath); + const chunks: Buffer[] = []; + let received = 0; + let expected = -1; + let settled = false; + + const done = (fn: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.removeAllListeners(); + socket.destroy(); + fn(); + }; + + const timer = setTimeout(() => { + done(() => reject(new BrokerClientError("cli_timeout", "broker request timed out"))); + }, this.timeoutMs); + timer.unref?.(); + + socket.on("error", (err: Error) => { + done(() => reject(new BrokerClientError("transport_error", err.message))); + }); + + socket.on("close", () => { + // Closed before a full response was parsed. + done(() => reject(new BrokerClientError("transport_error", "broker closed connection early"))); + }); + + socket.on("data", (chunk: Buffer) => { + received += chunk.byteLength; + if (received > LENGTH_PREFIX_BYTES + MAX_RESPONSE_BYTES) { + done(() => reject(new BrokerClientError("malformed_response", "broker response exceeds size limit"))); + return; + } + chunks.push(chunk); + const buf = Buffer.concat(chunks); + if (expected < 0) { + if (buf.byteLength < LENGTH_PREFIX_BYTES) return; + expected = buf.readUInt32BE(0); + if (expected > MAX_RESPONSE_BYTES) { + done(() => reject(new BrokerClientError("malformed_response", "broker response length invalid"))); + return; + } + } + if (buf.byteLength < LENGTH_PREFIX_BYTES + expected) return; + const payload = buf.subarray(LENGTH_PREFIX_BYTES, LENGTH_PREFIX_BYTES + expected).toString("utf8"); + let parsed: unknown; + try { + parsed = JSON.parse(payload); + } catch { + done(() => reject(new BrokerClientError("malformed_response", "broker response is not valid JSON"))); + return; + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + done(() => reject(new BrokerClientError("malformed_response", "broker response is not an object"))); + return; + } + const obj = parsed as Record; + if (obj.ok === false) { + const code = typeof obj.code === "string" ? obj.code : "internal_error"; + const message = typeof obj.message === "string" ? obj.message : code; + done(() => reject(new BrokerClientError(code, message))); + return; + } + if (obj.ok !== true) { + done(() => reject(new BrokerClientError("malformed_response", "broker response missing ok flag"))); + return; + } + done(() => resolve(obj)); + }); + + socket.on("connect", () => { + socket.write(frame); + }); + }); + } +} diff --git a/server/src/services/runtime-exposure/exposure-manager.test.ts b/server/src/services/runtime-exposure/exposure-manager.test.ts new file mode 100644 index 00000000000..cc3f4d13aa2 --- /dev/null +++ b/server/src/services/runtime-exposure/exposure-manager.test.ts @@ -0,0 +1,317 @@ +import { describe, expect, it } from "vitest"; + +import { RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort } from "@paperclipai/shared"; + +import { + BrokerClientError, + type BrokerClient, + type BrokerExposeResult, + type BrokerListenerRequest, + type BrokerOwnedListener, + type BrokerRemoveResult, +} from "./broker-client.js"; +import { + deprovisionExposure, + provisionExposure, + reconcileExposures, + type ExposureManagerDeps, +} from "./exposure-manager.js"; + +const RUNTIME_ID = "11111111-2222-4333-8444-555566667777"; +const APP_PORT = RUNTIME_EXPOSURE_APP_PORT_MIN; +const HMR_PORT = deriveViteHmrPort(APP_PORT); +const HOSTNAME = "runner-abc.tail-scale.ts.net"; + +const CONFIG = { + type: "tailscale_https" as const, + hostname: "auto" as const, + publicPort: "same" as const, + includePaperclipViteHmr: true, + failurePolicy: "fail_closed" as const, +}; + +interface FakeBrokerScript { + expose?: (runtimeId: string, listeners: BrokerListenerRequest[]) => BrokerExposeResult; + remove?: (runtimeId: string, handle: string) => BrokerRemoveResult; + list?: () => BrokerOwnedListener[]; +} + +interface FakeBrokerCalls { + expose: Array<{ runtimeId: string; listeners: BrokerListenerRequest[] }>; + remove: Array<{ runtimeId: string; handle: string }>; + list: number; +} + +function fakeBroker(script: FakeBrokerScript): { broker: BrokerClient; calls: FakeBrokerCalls } { + const calls: FakeBrokerCalls = { expose: [], remove: [], list: 0 }; + const broker: BrokerClient = { + async expose(runtimeId, listeners) { + calls.expose.push({ runtimeId, listeners }); + if (!script.expose) throw new BrokerClientError("internal_error", "no expose script"); + return script.expose(runtimeId, listeners); + }, + async remove(runtimeId, handle) { + calls.remove.push({ runtimeId, handle }); + if (!script.remove) throw new BrokerClientError("internal_error", "no remove script"); + return script.remove(runtimeId, handle); + }, + async list() { + calls.list += 1; + return script.list ? script.list() : []; + }, + }; + return { broker, calls }; +} + +function deps(broker: BrokerClient, probeHealth: () => Promise): ExposureManagerDeps { + return { broker, probeHealth, now: () => "2026-08-11T00:00:00.000Z" }; +} + +describe("provisionExposure", () => { + it("exposes app + HMR and reports ready once the HTTPS probe validates", async () => { + const { broker, calls } = fakeBroker({ + expose: () => ({ handle: "handle-abcdef1234567890", publicPorts: [APP_PORT, HMR_PORT] }), + }); + const { status, handle } = await provisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + config: CONFIG, + hostname: HOSTNAME, + appPort: APP_PORT, + }); + + expect(calls.expose).toHaveLength(1); + expect(calls.expose[0].listeners).toEqual([ + { purpose: "app", port: APP_PORT }, + { purpose: "vite_hmr", port: HMR_PORT }, + ]); + expect(status.state).toBe("ready"); + expect(status.publicUrl).toBe(`https://${HOSTNAME}:${APP_PORT}`); + expect(status.hostname).toBe(HOSTNAME); + expect(status.listeners).toHaveLength(2); + expect(status.brokerRef).toBe(RUNTIME_ID); + expect(status.lastError).toBeNull(); + expect(handle).toBe("handle-abcdef1234567890"); + }); + + it("omits the HMR listener when HMR is not requested", async () => { + const { broker, calls } = fakeBroker({ + expose: () => ({ handle: "handle-abcdef1234567890", publicPorts: [APP_PORT] }), + }); + const { status } = await provisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + config: { ...CONFIG, includePaperclipViteHmr: false }, + hostname: HOSTNAME, + appPort: APP_PORT, + }); + expect(calls.expose[0].listeners).toEqual([{ purpose: "app", port: APP_PORT }]); + expect(status.listeners).toEqual([{ purpose: "app", publicPort: APP_PORT, targetPort: APP_PORT }]); + expect(status.state).toBe("ready"); + }); + + it("fails closed without calling the broker when the app port is out of range", async () => { + const { broker, calls } = fakeBroker({}); + const { status, handle } = await provisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + config: CONFIG, + hostname: HOSTNAME, + appPort: 3100, // primary app port — never eligible + }); + expect(calls.expose).toHaveLength(0); + expect(status.state).toBe("failed"); + expect(handle).toBeNull(); + }); + + it("fails when the broker rejects the expose request", async () => { + const { broker } = fakeBroker({ + expose: () => { + throw new BrokerClientError("port_not_allowlisted", "denied"); + }, + }); + const { status, handle } = await provisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + config: CONFIG, + hostname: HOSTNAME, + appPort: APP_PORT, + }); + expect(status.state).toBe("failed"); + expect(status.lastError).toBe("port_not_allowlisted"); + expect(status.listeners).toEqual([]); + expect(handle).toBeNull(); + }); + + it("tears the mapping back down and fails when returned ports do not match", async () => { + const { broker, calls } = fakeBroker({ + expose: () => ({ handle: "handle-abcdef1234567890", publicPorts: [APP_PORT] }), // missing HMR + remove: () => ({ removedPorts: [APP_PORT] }), + }); + const { status, handle } = await provisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + config: CONFIG, + hostname: HOSTNAME, + appPort: APP_PORT, + }); + expect(status.state).toBe("failed"); + expect(status.lastError).toMatch(/unexpected public ports/); + expect(calls.remove).toHaveLength(1); // best-effort rollback + expect(handle).toBeNull(); + }); + + it("fail-closed: keeps the handle but reports failed when the HTTPS probe does not validate", async () => { + const { broker } = fakeBroker({ + expose: () => ({ handle: "handle-abcdef1234567890", publicPorts: [APP_PORT, HMR_PORT] }), + }); + const { status, handle } = await provisionExposure(deps(broker, async () => false), { + runtimeId: RUNTIME_ID, + config: CONFIG, + hostname: HOSTNAME, + appPort: APP_PORT, + }); + expect(status.state).toBe("failed"); + expect(status.publicUrl).toBeNull(); + expect(status.listeners).toHaveLength(2); // attempted mapping still surfaced + expect(status.lastError).toMatch(/health probe/); + // Handle retained so the caller can retry or clean up the dangling mapping. + expect(handle).toBe("handle-abcdef1234567890"); + }); + + it("fail-closed: treats a probe that throws as unhealthy", async () => { + const { broker } = fakeBroker({ + expose: () => ({ handle: "handle-abcdef1234567890", publicPorts: [APP_PORT, HMR_PORT] }), + }); + const { status } = await provisionExposure( + deps(broker, async () => { + throw new Error("cert error"); + }), + { runtimeId: RUNTIME_ID, config: CONFIG, hostname: HOSTNAME, appPort: APP_PORT }, + ); + expect(status.state).toBe("failed"); + }); +}); + +describe("deprovisionExposure", () => { + it("is a no-op removal when no handle was ever recorded", async () => { + const { broker, calls } = fakeBroker({}); + const { status, quarantinedPorts } = await deprovisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + handle: null, + ports: [APP_PORT, HMR_PORT], + }); + expect(calls.remove).toHaveLength(0); + expect(status.state).toBe("removed"); + expect(quarantinedPorts).toEqual([]); + }); + + it("removes cleanly with a handle", async () => { + const { broker, calls } = fakeBroker({ remove: () => ({ removedPorts: [APP_PORT, HMR_PORT] }) }); + const { status, quarantinedPorts } = await deprovisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + handle: "handle-abcdef1234567890", + ports: [APP_PORT, HMR_PORT], + }); + expect(calls.remove).toHaveLength(1); + expect(status.state).toBe("removed"); + expect(quarantinedPorts).toEqual([]); + }); + + it("treats an already-gone mapping as removed (idempotent)", async () => { + const { broker } = fakeBroker({ + remove: () => { + throw new BrokerClientError("invalid_handle", "gone"); + }, + }); + const { status, quarantinedPorts } = await deprovisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + handle: "handle-abcdef1234567890", + ports: [APP_PORT, HMR_PORT], + }); + expect(status.state).toBe("removed"); + expect(quarantinedPorts).toEqual([]); + }); + + it("quarantines ports and reports cleanup_pending on an ambiguous cleanup failure", async () => { + const { broker } = fakeBroker({ + remove: () => { + throw new BrokerClientError("cli_error", "tailscale serve failed"); + }, + }); + const { status, quarantinedPorts } = await deprovisionExposure(deps(broker, async () => true), { + runtimeId: RUNTIME_ID, + handle: "handle-abcdef1234567890", + ports: [APP_PORT, HMR_PORT, APP_PORT], + }); + expect(status.state).toBe("cleanup_pending"); + expect(status.lastError).toBe("cli_error"); + expect(quarantinedPorts).toEqual([APP_PORT, HMR_PORT]); // deduped + }); +}); + +describe("reconcileExposures", () => { + it("adopts owned mappings that are still desired", async () => { + const { broker, calls } = fakeBroker({ + list: () => [ + { runtimeId: RUNTIME_ID, port: APP_PORT, purpose: "app" }, + { runtimeId: RUNTIME_ID, port: HMR_PORT, purpose: "vite_hmr" }, + ], + }); + const result = await reconcileExposures(deps(broker, async () => true), { + desiredRuntimeIds: new Set([RUNTIME_ID]), + handlesByRuntimeId: new Map(), + }); + expect(result.adopted).toEqual([RUNTIME_ID]); + expect(result.removedOrphanPorts).toEqual([]); + expect(calls.remove).toHaveLength(0); + }); + + it("removes an orphaned owned mapping when a handle is available", async () => { + const orphan = "99999999-2222-4333-8444-555566667777"; + const { broker, calls } = fakeBroker({ + list: () => [{ runtimeId: orphan, port: APP_PORT, purpose: "app" }], + remove: () => ({ removedPorts: [APP_PORT] }), + }); + const result = await reconcileExposures(deps(broker, async () => true), { + desiredRuntimeIds: new Set([RUNTIME_ID]), + handlesByRuntimeId: new Map([[orphan, "handle-abcdef1234567890"]]), + }); + expect(result.removedOrphanPorts).toEqual([APP_PORT]); + expect(calls.remove).toEqual([{ runtimeId: orphan, handle: "handle-abcdef1234567890" }]); + expect(result.unremovableOrphans).toEqual([]); + }); + + it("never mutates an orphan it has no handle for (no remove call)", async () => { + const orphan = "99999999-2222-4333-8444-555566667777"; + const { broker, calls } = fakeBroker({ + list: () => [{ runtimeId: orphan, port: APP_PORT, purpose: "app" }], + }); + const result = await reconcileExposures(deps(broker, async () => true), { + desiredRuntimeIds: new Set([RUNTIME_ID]), + handlesByRuntimeId: new Map(), + }); + expect(calls.remove).toHaveLength(0); + expect(result.unremovableOrphans).toEqual([orphan]); + expect(result.removedOrphanPorts).toEqual([]); + }); + + it("records non-fatal errors without aborting reconciliation", async () => { + const orphanA = "aaaaaaaa-2222-4333-8444-555566667777"; + const orphanB = "bbbbbbbb-2222-4333-8444-555566667777"; + const { broker } = fakeBroker({ + list: () => [ + { runtimeId: orphanA, port: APP_PORT, purpose: "app" }, + { runtimeId: orphanB, port: APP_PORT + 1, purpose: "app" }, + ], + remove: (runtimeId) => { + if (runtimeId === orphanA) throw new BrokerClientError("cli_error", "boom"); + return { removedPorts: [APP_PORT + 1] }; + }, + }); + const result = await reconcileExposures(deps(broker, async () => true), { + desiredRuntimeIds: new Set(), + handlesByRuntimeId: new Map([ + [orphanA, "handle-aaaaaaaaaaaaaaaa"], + [orphanB, "handle-bbbbbbbbbbbbbbbb"], + ]), + }); + expect(result.errors).toEqual([{ runtimeId: orphanA, code: "cli_error" }]); + expect(result.removedOrphanPorts).toEqual([APP_PORT + 1]); + }); +}); diff --git a/server/src/services/runtime-exposure/exposure-manager.ts b/server/src/services/runtime-exposure/exposure-manager.ts new file mode 100644 index 00000000000..8715f74ff0b --- /dev/null +++ b/server/src/services/runtime-exposure/exposure-manager.ts @@ -0,0 +1,311 @@ +/** + * Runtime exposure lifecycle orchestrator for the `tailscale_https` mode. + * + * Owns the exposure state machine independently of the backend process + * lifecycle (PAP-17049 plan, PAP-17050 verdict). A backend can be running and + * healthy while its HTTPS exposure is still `pending`, `failed`, or + * `cleanup_pending`. This module never touches Tailscale directly — it drives + * the least-privilege broker through an injected {@link BrokerClient} and an + * injected external HTTPS health probe, so every branch is unit-testable + * against a fake broker with no sockets, no CLI, and no certificates. + * + * Invariants enforced here (in addition to the broker's own controls): + * - fail-closed: a configured HTTPS preview is never reported `ready` unless an + * external, normally-validating HTTPS probe succeeds. + * - same-number: only the app port and its derived HMR companion are exposed. + * - idempotent stop/cleanup: a missing/already-removed lease is success, not an + * error; genuine cleanup failures quarantine the ports and never reuse them. + * - reconcile touches only Paperclip-owned mappings the broker reports; it can + * never mutate an unknown/manual mapping or the primary `:443` route (the + * broker's `list` never returns those, and remove requires an owned handle). + */ +import type { RuntimeExposureConfigInput, RuntimeExposureStatus } from "@paperclipai/shared"; +import { + buildRuntimeExposureHealthUrl, + buildRuntimeExposureUrl, + deriveViteHmrPort, + isRuntimeExposureAppPort, +} from "@paperclipai/shared"; + +import { + BrokerClientError, + type BrokerClient, + type BrokerListenerRequest, +} from "./broker-client.js"; + +/** Injected seams — all side effects live behind these for testability. */ +export interface ExposureManagerDeps { + broker: BrokerClient; + /** External HTTPS probe; must reject/return false on cert or connection error. */ + probeHealth: (healthUrl: string) => Promise; + /** ISO timestamp source. Injected so tests are deterministic. */ + now: () => string; +} + +export interface ProvisionInput { + runtimeId: string; + config: RuntimeExposureConfigInput; + /** Resolved Tailscale node DNS hostname (the manager does not resolve it). */ + hostname: string; + /** Loopback app port already allocated in the dedicated range. */ + appPort: number; +} + +export interface ProvisionResult { + status: RuntimeExposureStatus; + /** + * Server-only lease handle required to later remove the mapping. NEVER + * serialized to the UI or embedded in {@link RuntimeExposureStatus}; persist + * it in a server-private store. Null when nothing was exposed. + */ + handle: string | null; +} + +export interface DeprovisionInput { + runtimeId: string; + /** Server-persisted lease handle, or null if none was ever recorded. */ + handle: string | null; + /** Ports previously exposed for this runtime (for quarantine on failure). */ + ports: number[]; +} + +export interface DeprovisionResult { + status: RuntimeExposureStatus; + /** Ports that must be quarantined (never reused) after an ambiguous cleanup. */ + quarantinedPorts: number[]; +} + +/** Broker error codes that mean "the mapping is already gone" — cleanup is a no-op. */ +const ALREADY_GONE_CODES: ReadonlySet = new Set([ + "invalid_handle", + "listener_not_owned", + "listener_ownership_mismatch", +]); + +function baseStatus(now: string): RuntimeExposureStatus { + return { + provider: "tailscale_https", + state: "pending", + publicUrl: null, + hostname: null, + listeners: [], + brokerRef: null, + lastError: null, + updatedAt: now, + }; +} + +/** Build the broker listener request list from a validated exposure config. */ +export function buildListenerRequests( + config: RuntimeExposureConfigInput, + appPort: number, +): BrokerListenerRequest[] { + const listeners: BrokerListenerRequest[] = [{ purpose: "app", port: appPort }]; + if (config.includePaperclipViteHmr) { + listeners.push({ purpose: "vite_hmr", port: deriveViteHmrPort(appPort) }); + } + return listeners; +} + +/** + * Provision (or re-provision) the HTTPS exposure for one runtime service. + * Idempotent at the broker level: a repeated call for a still-valid mapping + * simply re-exposes and re-probes. Always returns a fully-formed status; the + * caller persists both the status and the returned server-only handle. + */ +export async function provisionExposure( + deps: ExposureManagerDeps, + input: ProvisionInput, +): Promise { + const now = deps.now(); + const status = baseStatus(now); + status.hostname = input.hostname; + + if (!isRuntimeExposureAppPort(input.appPort)) { + status.state = "failed"; + status.lastError = "app port outside dedicated runtime exposure range"; + return { status, handle: null }; + } + + const requested = buildListenerRequests(input.config, input.appPort); + + let handle: string; + let publicPorts: number[]; + try { + const result = await deps.broker.expose(input.runtimeId, requested); + handle = result.handle; + publicPorts = result.publicPorts; + } catch (error) { + status.state = "failed"; + status.lastError = brokerErrorCode(error); + return { status, handle: null }; + } + + // The broker must return exactly the ports we asked for (same-number). A + // mismatch is a contract violation: tear the mapping back down and fail. + const requestedPorts = new Set(requested.map((l) => l.port)); + const returnedPorts = new Set(publicPorts); + if (requestedPorts.size !== returnedPorts.size || ![...requestedPorts].every((p) => returnedPorts.has(p))) { + await safeRemove(deps.broker, input.runtimeId, handle); + status.state = "failed"; + status.lastError = "broker returned unexpected public ports"; + return { status, handle: null }; + } + + // Reflect the attempted mapping regardless of health so the UI can show what + // was provisioned even while probing. + status.listeners = requested.map((l) => ({ + purpose: l.purpose, + publicPort: l.port, + targetPort: l.port, + })); + status.brokerRef = input.runtimeId; + + // fail-closed: never report ready until an external, cert-validating probe + // succeeds. Keep the handle so the caller can retry or clean up. + const healthUrl = buildRuntimeExposureHealthUrl(input.hostname, input.appPort); + let healthy = false; + try { + healthy = await deps.probeHealth(healthUrl); + } catch { + healthy = false; + } + if (!healthy) { + status.state = "failed"; + status.publicUrl = null; + status.lastError = "external HTTPS health probe did not validate"; + return { status, handle }; + } + + status.state = "ready"; + status.publicUrl = buildRuntimeExposureUrl(input.hostname, input.appPort); + status.lastError = null; + return { status, handle }; +} + +/** + * Tear down the HTTPS exposure for one runtime service. Idempotent: a null or + * already-gone handle resolves to `removed` with no quarantine. A genuine + * cleanup failure resolves to `cleanup_pending` and quarantines the ports. + */ +export async function deprovisionExposure( + deps: ExposureManagerDeps, + input: DeprovisionInput, +): Promise { + const now = deps.now(); + const status = baseStatus(now); + status.state = "removed"; + + if (input.handle === null) { + // Nothing the server knows to remove — treat as already cleaned up. + return { status, quarantinedPorts: [] }; + } + + try { + await deps.broker.remove(input.runtimeId, input.handle); + return { status, quarantinedPorts: [] }; + } catch (error) { + const code = brokerErrorCode(error); + if (ALREADY_GONE_CODES.has(code)) { + // The mapping is already gone; cleanup is complete and safe. + return { status, quarantinedPorts: [] }; + } + // Ambiguous failure: we cannot prove the mapping is gone. Quarantine the + // ports so they are never reused, and surface cleanup_pending. + status.state = "cleanup_pending"; + status.lastError = code; + return { status, quarantinedPorts: [...new Set(input.ports)] }; + } +} + +export interface ReconcileInput { + /** Runtimes whose exposure SHOULD remain. Everything else owned is an orphan. */ + desiredRuntimeIds: ReadonlySet; + /** Every server-persisted lease handle, keyed by runtimeId (desired + orphaned). */ + handlesByRuntimeId: ReadonlyMap; +} + +export interface ReconcileResult { + /** Owned+desired runtimes confirmed present in the broker's mapping. */ + adopted: string[]; + /** Ports removed because their runtime is no longer desired. */ + removedOrphanPorts: number[]; + /** + * Orphaned owned runtimes we could NOT remove because no handle was persisted. + * Reported for operator/broker-side quarantine; the manager never fabricates a + * handle and never touches a mapping it cannot prove ownership of. + */ + unremovableOrphans: string[]; + /** Non-fatal per-runtime removal errors during reconciliation. */ + errors: Array<{ runtimeId: string; code: string }>; +} + +/** + * Startup / periodic reconciliation. Adopts owned mappings that are still + * desired and removes owned mappings that are not. The broker's `list` only + * ever returns Paperclip-owned listeners, so unknown/manual mappings and the + * primary `:443` route are structurally out of scope here — they can never be + * enumerated, adopted, or removed by this routine. + */ +export async function reconcileExposures( + deps: ExposureManagerDeps, + input: ReconcileInput, +): Promise { + const result: ReconcileResult = { + adopted: [], + removedOrphanPorts: [], + unremovableOrphans: [], + errors: [], + }; + + const owned = await deps.broker.list(); + + // Group owned ports by runtimeId. + const portsByRuntime = new Map(); + for (const listener of owned) { + const ports = portsByRuntime.get(listener.runtimeId) ?? []; + ports.push(listener.port); + portsByRuntime.set(listener.runtimeId, ports); + } + + for (const [runtimeId, ports] of portsByRuntime) { + if (input.desiredRuntimeIds.has(runtimeId)) { + result.adopted.push(runtimeId); + continue; + } + // Orphan: owned by us but no longer desired. + const handle = input.handlesByRuntimeId.get(runtimeId); + if (!handle) { + result.unremovableOrphans.push(runtimeId); + continue; + } + try { + const removed = await deps.broker.remove(runtimeId, handle); + result.removedOrphanPorts.push(...removed.removedPorts); + } catch (error) { + const code = brokerErrorCode(error); + if (ALREADY_GONE_CODES.has(code)) { + // Already gone — reconciled, count its ports as removed for reporting. + result.removedOrphanPorts.push(...ports); + } else { + result.errors.push({ runtimeId, code }); + } + } + } + + return result; +} + +/** Best-effort remove used on rollback paths; swallows broker errors. */ +async function safeRemove(broker: BrokerClient, runtimeId: string, handle: string): Promise { + try { + await broker.remove(runtimeId, handle); + } catch { + /* best effort — the outer flow has already decided to fail */ + } +} + +function brokerErrorCode(error: unknown): string { + if (error instanceof BrokerClientError) return error.code; + return "internal_error"; +} diff --git a/server/src/services/runtime-exposure/port-pair.test.ts b/server/src/services/runtime-exposure/port-pair.test.ts new file mode 100644 index 00000000000..b95ad662809 --- /dev/null +++ b/server/src/services/runtime-exposure/port-pair.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, it } from "vitest"; + +import { + RUNTIME_EXPOSURE_APP_PORT_MIN, + RUNTIME_EXPOSURE_HMR_PORT_OFFSET, +} from "@paperclipai/shared"; + +import { allocateExposurePortPair } from "./port-pair.js"; + +describe("allocateExposurePortPair", () => { + it("returns the first free app port with a free HMR companion", async () => { + const pair = await allocateExposurePortPair({ isPortAvailable: async () => true }); + expect(pair.appPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN); + expect(pair.hmrPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + RUNTIME_EXPOSURE_HMR_PORT_OFFSET); + }); + + it("skips an app port whose HMR companion is busy", async () => { + const busyHmr = RUNTIME_EXPOSURE_APP_PORT_MIN + RUNTIME_EXPOSURE_HMR_PORT_OFFSET; + const pair = await allocateExposurePortPair({ + isPortAvailable: async (port) => port !== busyHmr, + }); + // First app port's companion is busy, so it advances to the next app port. + expect(pair.appPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + 1); + expect(pair.hmrPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + 1 + RUNTIME_EXPOSURE_HMR_PORT_OFFSET); + }); + + it("skips a busy app port", async () => { + const pair = await allocateExposurePortPair({ + isPortAvailable: async (port) => port !== RUNTIME_EXPOSURE_APP_PORT_MIN, + }); + expect(pair.appPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + 1); + }); + + it("never returns a reserved (e.g. quarantined) app or HMR port", async () => { + const reserved = new Set([ + RUNTIME_EXPOSURE_APP_PORT_MIN, + // reserve the SECOND app port's HMR companion too + RUNTIME_EXPOSURE_APP_PORT_MIN + 1 + RUNTIME_EXPOSURE_HMR_PORT_OFFSET, + ]); + const pair = await allocateExposurePortPair({ + isPortAvailable: async () => true, + reserved, + }); + expect(pair.appPort).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + 2); + }); + + it("throws when the dedicated range is exhausted", async () => { + await expect( + allocateExposurePortPair({ isPortAvailable: async () => false }), + ).rejects.toThrow(/no free app\/HMR port pair/); + }); + + it("only probes the companion after the app port is free (short-circuits)", async () => { + const probed: number[] = []; + await allocateExposurePortPair({ + isPortAvailable: async (port) => { + probed.push(port); + return true; + }, + }); + // First two probes are the first app port and its companion, in order. + expect(probed[0]).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN); + expect(probed[1]).toBe(RUNTIME_EXPOSURE_APP_PORT_MIN + RUNTIME_EXPOSURE_HMR_PORT_OFFSET); + }); +}); diff --git a/server/src/services/runtime-exposure/port-pair.ts b/server/src/services/runtime-exposure/port-pair.ts new file mode 100644 index 00000000000..db6324a3006 --- /dev/null +++ b/server/src/services/runtime-exposure/port-pair.ts @@ -0,0 +1,64 @@ +/** + * Paired app + Vite HMR port allocation for the `tailscale_https` exposure mode. + * + * A managed runtime that opts into HTTPS exposure needs TWO deterministically + * related loopback ports reserved together (PAP-17049 plan, PAP-17050 verdict + * requirement #2): the app port and its Paperclip Vite HMR companion at a fixed + * offset. Allocating them as a pair — and only from the dedicated allowlisted + * range — means a compromised caller can never ask the broker to publish an + * arbitrary existing loopback service, and the HMR listener is never orphaned + * from its app listener. + * + * Pure orchestration over an injected availability probe: no sockets here, so + * the scan order and skip logic are unit-testable. + */ +import { + RUNTIME_EXPOSURE_APP_PORT_MIN, + RUNTIME_EXPOSURE_APP_PORT_MAX, + deriveViteHmrPort, + isRuntimeExposureAppPort, +} from "@paperclipai/shared"; + +export interface ExposurePortPair { + appPort: number; + hmrPort: number; +} + +export interface AllocateExposurePortPairInput { + /** + * Returns true if the port can currently be bound loopback-only. Both the app + * port and its HMR companion must pass before a pair is returned. + */ + isPortAvailable: (port: number) => Promise; + /** + * Ports that must never be handed out — e.g. quarantined after an ambiguous + * cleanup, or already reserved by other live runtimes this cycle. + */ + reserved?: ReadonlySet; +} + +/** + * Scan the dedicated app-port range in ascending order and return the first + * app port whose HMR companion is also free and unreserved. Throws when the + * range is exhausted so the caller fails closed rather than binding an + * out-of-range port. + */ +export async function allocateExposurePortPair( + input: AllocateExposurePortPairInput, +): Promise { + const reserved = input.reserved ?? new Set(); + for (let appPort = RUNTIME_EXPOSURE_APP_PORT_MIN; appPort <= RUNTIME_EXPOSURE_APP_PORT_MAX; appPort += 1) { + if (reserved.has(appPort)) continue; + const hmrPort = deriveViteHmrPort(appPort); + if (reserved.has(hmrPort)) continue; + if (!isRuntimeExposureAppPort(appPort)) continue; // defensive; range guarantees it + // Probe the app port first; short-circuit before probing the companion. + if (!(await input.isPortAvailable(appPort))) continue; + if (!(await input.isPortAvailable(hmrPort))) continue; + return { appPort, hmrPort }; + } + throw new Error( + `no free app/HMR port pair available in the dedicated runtime exposure range ` + + `[${RUNTIME_EXPOSURE_APP_PORT_MIN}, ${RUNTIME_EXPOSURE_APP_PORT_MAX}]`, + ); +} diff --git a/server/src/services/workspace-runtime-read-model.test.ts b/server/src/services/workspace-runtime-read-model.test.ts index 1de4250b079..90567f742ad 100644 --- a/server/src/services/workspace-runtime-read-model.test.ts +++ b/server/src/services/workspace-runtime-read-model.test.ts @@ -35,6 +35,7 @@ function runtimeServiceRow( stoppedAt: now, stopPolicy: null, healthStatus: "unknown", + exposure: null, createdAt: now, updatedAt: now, ...overrides,