PaperClipAI/server/src/services/workspace-runtime-exposure.test.ts

import fs from "node:fs/promises";
import net from "node:net";
import os from "node:os";
import path from "node:path";

import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";

import {
  deriveViteHmrPort,
  RUNTIME_EXPOSURE_APP_PORT_MAX,
  RUNTIME_EXPOSURE_APP_PORT_MIN,
} from "@paperclipai/shared";

import type { BrokerClient, BrokerListenerRequest } from "./runtime-exposure/broker-client.js";
import {
  diagnoseRuntimeListenerBinds,
  readListenerBindFacts,
} from "./runtime-exposure/loopback-listener.js";
import {
  classifyExposureHostCollisions,
  type ExposurePortHostState,
  resetRuntimeServicesForTests,
  setWorkspaceRuntimeExposureDepsForTests,
  startRuntimeServicesForWorkspaceControl,
  stopRuntimeServicesForExecutionWorkspace,
} from "./workspace-runtime.js";

const EXECUTION_WORKSPACE_ID = "11111111-2222-4333-8444-555566667777";
const HANDLE = "handle-abcdef1234567890";

// The shared test setup pins the automatic default off so unrelated suites do
// not probe for a real host broker. This suite is about the default, so it opts
// back in and restores the harness value afterwards.
beforeEach(async () => {
  vi.stubEnv("PAPERCLIP_MANAGED_RUNTIME_HTTPS", "auto");
  // Registry records and append-only logs must not share a developer's instance
  // or a previous test's service identity.
  vi.stubEnv("PAPERCLIP_HOME", await fs.mkdtemp(path.join(guestDir, "home-")));
});

afterEach(async () => {
  // These tests spawn real loopback backends on dedicated-range ports; reap them
  // rather than leaving one squatting 42xxx/52xxx for every test in the file.
  try {
    await resetRuntimeServicesForTests({ terminateProcesses: true });
  } finally {
    vi.unstubAllEnvs();
  }
});

function serviceCommand() {
  // Answers `/api/health` the way a real Paperclip dev runtime does: managed
  // publication requires semantic health, not just a 200 (PAP-17572).
  return `node -e 'console.log("fixture started",process.pid,Date.now());const http=require("http");const p=Number(process.env.PORT);for(const q of [p,p+10000].filter(q=>q<65536))http.createServer((rq,r)=>{if(rq.url==="/api/health"){r.setHeader("content-type","application/json");r.end(JSON.stringify({status:"ok"}));return}r.statusCode=200;r.end("ok")}).listen(q,"127.0.0.1",()=>console.log("fixture listening",q,Date.now()));setInterval(()=>{},1000)'`;
}

/**
 * Fake guest dev runners written to disk (PAP-17256).
 *
 * They must be real files named `dev-runner*.mjs` for two reasons: the command
 * has to look like a dev-runner invocation for the rewrite to apply at all, and
 * `node <file> --bind lan` hands the flags to the script, which is exactly how a
 * real `pnpm dev --bind lan` reaches `scripts/dev-runner.ts`.
 */
let guestDir: string;

/**
 * A guest checkout from *before* managed HTTPS exposure existed.
 *
 * Reproduces the behaviour that actually broke the lanes: `scripts/dev-runner.ts`
 * on plain master derives its bind mode from its own `--bind` / `--bind-host`
 * argv and **ignores `PAPERCLIP_BIND` / `PAPERCLIP_MANAGED_RUNTIME_EXPOSURE`
 * entirely** — so env-only hardening cannot reach it. `--bind lan` therefore
 * means `0.0.0.0`, which the broker must refuse.
 */
const PRE_MANAGED_EXPOSURE_GUEST = `
import http from "node:http";
const argv = process.argv.slice(2);
const valueOf = (flag) => {
  const at = argv.indexOf(flag);
  const value = at >= 0 ? argv[at + 1] : undefined;
  return value && !value.startsWith("--") ? value : undefined;
};
const mode = valueOf("--bind") ?? "loopback";
const host = mode === "custom" ? (valueOf("--bind-host") ?? "127.0.0.1")
  : mode === "lan" ? "0.0.0.0"
  : "127.0.0.1";
const p = Number(process.env.PORT);
// Even a pre-exposure checkout answered /api/health semantically; these guests
// model bind behaviour, not health behaviour.
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
for (const q of [p, p + 10000]) {
  http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(q, host);
}
setInterval(() => {}, 1000);
`;

/**
 * The true shape of the deployed failure: a guest that honours the bind argv for
 * its own HTTP listener but whose Vite gets `hmr.port` with no `hmr.server` and
 * no `server.host`, so Vite opens its *own* HMR websocket on the IPv6 wildcard
 * regardless of the bind mode. Plain master's `server/src/app.ts` does exactly
 * this, which is why forcing the bind is necessary but not sufficient there.
 */
const WILDCARD_HMR_GUEST = `
import http from "node:http";
const argv = process.argv.slice(2);
const at = argv.indexOf("--bind-host");
const host = at >= 0 ? argv[at + 1] : "127.0.0.1";
const p = Number(process.env.PORT);
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(p, host);
// No host argument: Vite's own HMR listener lands on the wildcard.
http.createServer((_, r) => { r.statusCode = 426; r.end(); }).listen(p + 10000);
setInterval(() => {}, 1000);
`;

/** A guest so old it has no bind flags at all and always binds the wildcard. */
const ALWAYS_WILDCARD_GUEST = `
import http from "node:http";
const p = Number(process.env.PORT);
const health = (rq, r) => { if (rq.url === "/api/health") { r.setHeader("content-type", "application/json"); r.end(JSON.stringify({ status: "ok" })); return true; } return false; };
for (const q of [p, p + 10000]) {
  http.createServer((rq, r) => { if (health(rq, r)) return; r.statusCode = 200; r.end("ok"); }).listen(q, "0.0.0.0");
}
setInterval(() => {}, 1000);
`;

/**
 * A guest that prints a synthetic `EADDRINUSE` line for its assigned port with no
 * host listener behind it, then exits. It models guest-controlled output that
 * claims a collision the host cannot confirm. The runtime must not quarantine the
 * pair on the printed line alone. Otherwise repeated starts drain the shared
 * exposure-port pool. The start must surface the failure terminally after a single
 * allocation.
 */
const SYNTHETIC_EADDRINUSE_ON_ASSIGNED_PORT_GUEST = `
const p = Number(process.env.PORT);
process.stderr.write("node:events:497\\nError: listen EADDRINUSE: address already in use 127.0.0.1:" + p + "\\n");
process.exit(1);
`;

/**
 * A guest that exits with `EADDRINUSE` on an unrelated auxiliary port, never on
 * its assigned app or HMR port. It models a fixed helper listener (for example an
 * inspector or metrics port) that an external process already holds. The managed
 * start must not quarantine the valid exposure pair; it must surface the failure
 * terminally after a single allocation.
 */
const EADDRINUSE_ON_AUXILIARY_PORT_GUEST = `
import http from "node:http";
process.stderr.write("node:events:497\\nError: listen EADDRINUSE: address already in use 127.0.0.1:39999\\n");
process.exit(1);
`;

/**
 * A guest that fails with `EADDRINUSE` on an unrelated auxiliary port, and also
 * prints the assigned app port on a separate, benign line. It models mixed
 * startup output: one line names the assigned port for an informational reason,
 * a different line reports the auxiliary-port conflict. The parser must match the
 * error and the port on the same line, so the benign mention of the assigned port
 * must not trigger a wrong quarantine. The managed start must surface the failure
 * terminally after a single allocation.
 */
const EADDRINUSE_ON_AUXILIARY_PORT_WITH_ASSIGNED_MENTION_GUEST = `
import http from "node:http";
const p = Number(process.env.PORT);
process.stderr.write("[dev] server ready on http://127.0.0.1:" + p + "/\\n");
process.stderr.write("node:events:497\\nError: listen EADDRINUSE: address already in use 127.0.0.1:39999\\n");
process.exit(1);
`;

beforeAll(async () => {
  guestDir = await fs.mkdtemp(path.join(os.tmpdir(), "pap-17256-guest-"));
  await fs.writeFile(path.join(guestDir, "dev-runner.mjs"), PRE_MANAGED_EXPOSURE_GUEST);
  await fs.writeFile(path.join(guestDir, "dev-runner-legacy.mjs"), ALWAYS_WILDCARD_GUEST);
  await fs.writeFile(path.join(guestDir, "dev-runner-wildcard-hmr.mjs"), WILDCARD_HMR_GUEST);
  await fs.writeFile(
    path.join(guestDir, "dev-runner-bind-conflict.mjs"),
    'process.stderr.write("local_trusted requires server.bind=loopback\\n"); process.exit(1);\n',
  );
  // A guest that prints a synthetic EADDRINUSE line for its assigned port with no
  // host listener behind it. It models guest-controlled output that claims a
  // collision the host cannot confirm. The start must not quarantine the pair.
  await fs.writeFile(
    path.join(guestDir, "dev-runner-eaddrinuse-synthetic.mjs"),
    SYNTHETIC_EADDRINUSE_ON_ASSIGNED_PORT_GUEST,
  );
  // A guest that fails on a fixed auxiliary port, not on its assigned app or HMR
  // port. It models an unrelated helper listener that an external process holds.
  // The assigned exposure pair stays valid, so the start must not quarantine it.
  await fs.writeFile(
    path.join(guestDir, "dev-runner-eaddrinuse-auxiliary.mjs"),
    EADDRINUSE_ON_AUXILIARY_PORT_GUEST,
  );
  // A guest that fails on an auxiliary port and also prints the assigned app port
  // on a separate, benign line. It models mixed output where the assigned port
  // appears for an unrelated reason. The parser must match the error and the port
  // on the same line, so the start must not quarantine the valid exposure pair.
  await fs.writeFile(
    path.join(guestDir, "dev-runner-eaddrinuse-auxiliary-mixed.mjs"),
    EADDRINUSE_ON_AUXILIARY_PORT_WITH_ASSIGNED_MENTION_GUEST,
  );
});

afterAll(async () => {
  await fs.rm(guestDir, { recursive: true, force: true });
});

const guestCommand = (file: string) => `node ${path.join(guestDir, file)}`;

function createBroker() {
  const calls: string[] = [];
  let listeners: BrokerListenerRequest[] = [];
  const broker: BrokerClient = {
    async reserve(_runtimeId, requested) {
      calls.push("reserve");
      listeners = requested;
      return { handle: HANDLE, reservedPorts: requested.map((listener) => listener.port) };
    },
    async expose() {
      calls.push("expose");
      return { handle: HANDLE, publicPorts: listeners.map((listener) => listener.port) };
    },
    async remove() {
      calls.push("remove");
      return { removedPorts: listeners.map((listener) => listener.port) };
    },
    async list() {
      return [];
    },
  };
  return { broker, calls };
}

/**
 * A real loopback bind probe, matching production's `isLoopbackPortAvailable`.
 *
 * The default used to be `async () => true`, which claimed every port in the
 * dedicated range was free. On a developer or canary host that already runs a
 * managed lane on `42000`, that lie made the suite allocate a port something
 * else was listening on — so the spawned guest could not bind, and every
 * assertion downstream failed for a reason that had nothing to do with the
 * behaviour under test. It also meant the suite exercised allocation against a
 * range it never actually checked (PAP-17419; PAP-17255 started this with
 * "make exposure fixture honor occupied ports").
 */
async function isLoopbackPortFree(port: number): Promise<boolean> {
  const facts = await readListenerBindFacts(port);
  if (facts?.present) return false;

  return await new Promise<boolean>((resolve) => {
    const probe = net.createServer();
    probe.unref();
    probe.once("error", () => resolve(false));
    probe.listen(port, "127.0.0.1", () => {
      probe.close(() => resolve(true));
    });
  });
}

/**
 * The first app port at or above `startAt` whose HMR companion is also free on
 * the real host, mirroring the allocator's own scan. The pinned-port test needs
 * a port that is *verifiably* free right now rather than a hard-coded one: the
 * whole dedicated range sits inside the Linux ephemeral port range, so any
 * transient loopback socket on the host (the client side of a readiness probe,
 * a TIME_WAIT remnant from another suite) can occupy a fixed constant at probe
 * time and make the allocator relocate for a reason unrelated to the behaviour
 * under test.
 */
async function findFreeExposureAppPort(startAt: number): Promise<number> {
  for (let appPort = startAt; appPort <= RUNTIME_EXPOSURE_APP_PORT_MAX; appPort += 1) {
    if (await isLoopbackPortFree(appPort) && await isLoopbackPortFree(deriveViteHmrPort(appPort))) {
      return appPort;
    }
  }
  throw new Error("no free app/HMR port pair available in the dedicated runtime exposure range");
}

function installDeps(overrides: {
  broker: BrokerClient;
  probeHealth?: () => Promise<boolean>;
  isBrokerAvailable?: () => Promise<boolean>;
  isPortAvailable?: (port: number) => Promise<boolean>;
  diagnoseListenerBinds?: (ports: number[]) => Promise<string | null>;
  resolveHostname?: () => Promise<string>;
}) {
  setWorkspaceRuntimeExposureDepsForTests({
    broker: overrides.broker,
    isPortAvailable: overrides.isPortAvailable ?? isLoopbackPortFree,
    isBrokerAvailable: overrides.isBrokerAvailable ?? (async () => true),
    resolveHostname: overrides.resolveHostname ?? (async () => "runner.tail123.ts.net"),
    probeHealth: overrides.probeHealth ?? (async () => true),
    now: () => "2026-08-11T00:00:00.000Z",
    // The real /proc-backed diagnosis, not a fake: the fake broker below always
    // says yes, so this is what has to catch an off-loopback bind here.
    diagnoseListenerBinds: overrides.diagnoseListenerBinds ?? diagnoseRuntimeListenerBinds,
  });
}

const DECLARED_EXPOSE = {
  type: "tailscale_https",
  hostname: "auto",
  publicPort: "same",
  includePaperclipViteHmr: true,
  failurePolicy: "fail_closed",
} as const;

/**
 * The pre-feature Paperclip App project template, verbatim: a hard-coded HTTP
 * `urlTemplate`, a pinned port outside the broker's dedicated range, and no
 * exposure declaration at all.
 */
const LEGACY_HTTP_EXPOSE = {
  type: "url",
  urlTemplate: "http://paperclip-dev:{{port}}",
} as const;

function startInput(options?: {
  serviceName?: string;
  expose?: Record<string, unknown> | null;
  port?: Record<string, unknown> | number;
  command?: string;
}) {
  const expose = options?.expose === undefined ? DECLARED_EXPOSE : options.expose;
  return {
    invocationId: "runtime-exposure-test",
    actor: { id: null, name: "Paperclip", companyId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" },
    issue: null,
    workspace: {
      baseCwd: process.cwd(),
      source: "project_primary" as const,
      projectId: null,
      workspaceId: null,
      repoUrl: null,
      repoRef: null,
      strategy: "project_primary" as const,
      cwd: process.cwd(),
      branchName: "test",
      worktreePath: null,
      warnings: [],
      created: false,
      branchCreatedByRuntime: false,
    },
    executionWorkspaceId: EXECUTION_WORKSPACE_ID,
    config: {
      workspaceRuntime: {
        services: [{
          name: options?.serviceName ?? "preview",
          command: options?.command ?? serviceCommand(),
          env: { PAPERCLIP_PUBLIC_URL: "http://127.0.0.1:3100" },
          port: options?.port ?? { type: "auto", envKey: "PORT" },
          // These lifecycle tests spawn real servers; cold CI startup can take
          // five seconds before the first listener is ready.
          readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10 },
          ...(expose ? { expose } : {}),
        }],
      },
    },
    adapterEnv: {},
  };
}

describe("workspace runtime tailscale_https lifecycle", () => {
  it("reserves before spawn, exposes after backend readiness, and removes on stop", async () => {
    const { broker, calls } = createBroker();
    let reservedPorts: number[] = [];
    installDeps({
      broker: {
        ...broker,
        async reserve(runtimeId, listeners) {
          reservedPorts = listeners.map((listener) => listener.port);
          for (const port of reservedPorts) expect(await isLoopbackPortFree(port)).toBe(true);
          return broker.reserve(runtimeId, listeners);
        },
        async expose(...args) {
          for (const port of reservedPorts) {
            const response = await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(5_000) });
            expect(await response.text()).toBe("ok");
          }
          return broker.expose(...args);
        },
      },
    });

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput());
    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    expect(runtime.port).toBeGreaterThanOrEqual(42000);
    expect(runtime.url).toBe(`https://runner.tail123.ts.net:${runtime.port}`);
    expect(runtime.exposure?.state).toBe("ready");

    await stopRuntimeServicesForExecutionWorkspace({
      executionWorkspaceId: EXECUTION_WORKSPACE_ID,
      runtimeServiceId: runtime.id,
    });
    expect(calls).toEqual(["reserve", "expose", "remove"]);
    for (const port of reservedPorts) expect(await isLoopbackPortFree(port)).toBe(true);
  }, 15_000);

  it("fails closed and removes the mapping when external HTTPS validation fails", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker, probeHealth: async () => false });

    await expect(startRuntimeServicesForWorkspaceControl(startInput())).rejects.toThrow(/HTTPS exposure failed/);
    expect(calls).toEqual(["reserve", "expose", "remove"]);
  }, 15_000);
});

describe("automatic tailscale_https default for managed worktree runtimes", () => {
  it("defaults a legacy paperclip-dev service with no exposure block, relocating its pinned port", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    // Exactly the persisted pre-feature shape: pinned 45439 + HTTP urlTemplate.
    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: LEGACY_HTTP_EXPOSE,
      port: 45_439,
    }));

    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    // 45439 is outside the broker allowlist, so the pinned port cannot be kept.
    expect(runtime.port).not.toBe(45_439);
    expect(runtime.port).toBeGreaterThanOrEqual(42_000);
    expect(runtime.port).toBeLessThanOrEqual(42_999);
    // The canonical URL is the verified HTTPS origin; HTTP is never retained.
    expect(runtime.url).toBe(`https://runner.tail123.ts.net:${runtime.port}`);
    expect(runtime.url).not.toContain("http://");
    expect(runtime.exposure?.state).toBe("ready");
  }, 15_000);

  it("keeps an existing runtime port that is already inside the dedicated range", async () => {
    const { broker } = createBroker();
    installDeps({ broker });

    // Pin a pair that is free right now, strictly above the lowest free pair.
    // If the keep-the-preferred-port path were broken, the ascending fallback
    // scan would return that lower pair, so coming back with the pinned port
    // still proves the behaviour — without betting the test on one hard-coded
    // host port (formerly 42500) staying unoccupied for the whole run.
    const lowestFreeAppPort = await findFreeExposureAppPort(RUNTIME_EXPOSURE_APP_PORT_MIN);
    const pinnedPort = await findFreeExposureAppPort(lowestFreeAppPort + 1);

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: LEGACY_HTTP_EXPOSE,
      port: pinnedPort,
    }));

    expect(runtime.port).toBe(pinnedPort);
    expect(runtime.url).toBe(`https://runner.tail123.ts.net:${pinnedPort}`);
  }, 15_000);

  it("preserves a deliberate opt-out and leaves the service on plain HTTP", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: { ...LEGACY_HTTP_EXPOSE, tailscaleHttps: false },
      port: { type: "auto", envKey: "PORT" },
    }));

    expect(calls).toEqual([]);
    expect(runtime.exposure ?? null).toBeNull();
    expect(runtime.url).toBe(`http://paperclip-dev:${runtime.port}`);
  }, 15_000);

  it("leaves an unmanaged/custom service untouched", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "preview",
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    }));

    expect(calls).toEqual([]);
    expect(runtime.exposure ?? null).toBeNull();
    expect(runtime.url).toBe(`http://paperclip-dev:${runtime.port}`);
  }, 15_000);

  it("does not default when the host broker is unavailable", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker, isBrokerAvailable: async () => false });

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    }));

    expect(calls).toEqual([]);
    expect(runtime.exposure ?? null).toBeNull();
  }, 15_000);

  it("still honors an explicit opt-in when the broker socket is missing, failing loudly", async () => {
    // A missing broker must never silently downgrade a requested HTTPS preview.
    const { broker, calls } = createBroker();
    const failing: BrokerClient = {
      ...broker,
      async reserve() {
        calls.push("reserve");
        throw new Error("ENOENT: broker socket missing");
      },
    };
    installDeps({ broker: failing, isBrokerAvailable: async () => false });

    await expect(
      startRuntimeServicesForWorkspaceControl(startInput({ serviceName: "paperclip-dev" })),
    ).rejects.toThrow();
    expect(calls).toEqual(["reserve"]);
  }, 15_000);

  it("is idempotent across repeated starts: the same port pair is reserved again", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const [first] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: LEGACY_HTTP_EXPOSE,
      port: 45_439,
    }));
    const firstPort = first.port;
    await stopRuntimeServicesForExecutionWorkspace({
      executionWorkspaceId: EXECUTION_WORKSPACE_ID,
      runtimeServiceId: first.id,
    });
    calls.length = 0;

    const [second] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      expose: LEGACY_HTTP_EXPOSE,
      port: 45_439,
    }));

    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    expect(second.port).toBe(firstPort);
    expect(second.url).toBe(`https://runner.tail123.ts.net:${firstPort}`);
  }, 25_000);

  it("releases the in-flight pair claim when hostname resolution fails, so a retry storm cannot exhaust the range", async () => {
    // The reservation keeps the winning pair's in-process claim on purpose, for
    // the caller to release on stop/teardown. A hostname failure throws before any
    // runtime record exists, so that teardown can never run for this pair — and
    // removing only the broker reservation would leave the claim held. Each retry
    // would then burn another pair and the lane would report the range exhausted
    // rather than the real cause, a Tailscale outage.
    const reservedAppPorts: number[] = [];
    const broker: BrokerClient = {
      async reserve(_runtimeId, requested) {
        reservedAppPorts.push(requested[0]!.port);
        return { handle: HANDLE, reservedPorts: requested.map((listener) => listener.port) };
      },
      async expose() {
        return { handle: HANDLE, publicPorts: [] };
      },
      async remove() {
        return { removedPorts: [] };
      },
      async list() {
        return [];
      },
    };
    installDeps({
      broker,
      resolveHostname: async () => {
        throw new Error("MagicDNS unavailable");
      },
    });

    for (let attempt = 0; attempt < 3; attempt += 1) {
      await expect(
        startRuntimeServicesForWorkspaceControl(startInput({ serviceName: "paperclip-dev" })),
      ).rejects.toThrow(/MagicDNS hostname unavailable/);
    }

    // Every attempt reserved the same pair. A leaked claim would push each retry
    // onto a fresh pair instead.
    expect(reservedAppPorts).toHaveLength(3);
    expect(new Set(reservedAppPorts).size).toBe(1);
  }, 25_000);
});

/**
 * PAP-17256: the deployed regression. A fresh managed lane on service index 0
 * failed every start with `listener_ownership_mismatch` because the branch
 * checkout it launched bound `0.0.0.0`, and the server only asked for loopback
 * via env vars that checkout never read.
 */
describe("loopback bind is forced on the guest, not merely requested (PAP-17256)", () => {
  it("starts a fresh service-index-0 lane whose guest only honours argv", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      // Verbatim the command every failing lane recorded, modulo the fake guest.
      command: `${guestCommand("dev-runner.mjs")} --bind lan`,
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    }));

    // The launched command carries the loopback bind, replacing `--bind lan`.
    expect(runtime.command).toContain("--bind loopback");
    expect(runtime.command).not.toContain("--bind lan");

    // And because the guest honoured it, both listeners are loopback-only, so
    // the exposure completes instead of being denied.
    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    expect(runtime.exposure?.state).toBe("ready");
    expect(runtime.exposure?.lastError ?? null).toBeNull();
    expect(runtime.port).toBeGreaterThanOrEqual(42_000);
    expect(runtime.url).toBe(`https://runner.tail123.ts.net:${runtime.port}`);

    // Independent confirmation on the live listeners, using the same /proc read
    // the broker's gate performs.
    const hmrPort = runtime.port! + 10_000;
    expect(await diagnoseRuntimeListenerBinds([runtime.port!, hmrPort])).toBeNull();

    await stopRuntimeServicesForExecutionWorkspace({
      executionWorkspaceId: EXECUTION_WORKSPACE_ID,
      runtimeServiceId: runtime.id,
    });
    expect(calls).toEqual(["reserve", "expose", "remove"]);
  }, 20_000);

  // This acceptance fixture inspects live Linux /proc socket tables.
  it.skipIf(process.platform !== "linux")("reaches a terminal failure naming the port and address when a guest still binds the wildcard", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    // A guest with no bind flags at all: the argv rewrite cannot reach it, so
    // this is the residual case that must fail loudly rather than expose.
    await expect(startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      command: guestCommand("dev-runner-legacy.mjs"),
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    }))).rejects.toThrow(/listener_ownership_mismatch.*(?:0\.0\.0\.0|::)/s);

    // Terminal recovery: the reservation is released and nothing was exposed.
    // The diagnosis runs before the broker, so `expose` is never attempted.
    expect(calls).toEqual(["reserve", "remove"]);
  }, 20_000);

  // This acceptance fixture inspects live Linux /proc socket tables.
  it.skipIf(process.platform !== "linux")("explains rather than only coding the failure, so the next operator can act", async () => {
    const { broker } = createBroker();
    installDeps({ broker });

    const error = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      command: guestCommand("dev-runner-legacy.mjs"),
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    })).then(() => null, (err: unknown) => err as Error);

    expect(error).not.toBeNull();
    // The bare code alone is what cost PAP-17254 three diagnostic cycles.
    expect(error!.message).toContain("listener_ownership_mismatch");
    expect(error!.message).toContain("loopback");
    expect(error!.message).toContain("--bind loopback");
  }, 20_000);

  it("leaves a non-Paperclip service's --bind argument alone", async () => {
    // `--bind` means something entirely different to the HTTPS probe canaries
    // (`python3 -m http.server --bind 127.0.0.1`); rewriting it would break them.
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const declared = `${serviceCommand()} -- --bind 127.0.0.1`;
    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "https-probe-canary",
      command: declared,
      port: { type: "auto", envKey: "PORT" },
    }));

    expect(runtime.command).toBe(declared);
    expect(runtime.command).not.toContain("--bind loopback");
    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    expect(runtime.exposure?.state).toBe("ready");
  }, 20_000);

  it("does not rewrite a Paperclip dev command when the service is not exposed", async () => {
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const declared = `${guestCommand("dev-runner.mjs")} --bind lan`;
    const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      command: declared,
      expose: { ...LEGACY_HTTP_EXPOSE, tailscaleHttps: false },
      // This guest opens a second listener at appPort + 10000. An arbitrary
      // ephemeral app port can overflow the valid TCP port range.
      port: await findFreeExposureAppPort(RUNTIME_EXPOSURE_APP_PORT_MIN),
    }));

    expect(calls).toEqual([]);
    expect(runtime.command).toBe(declared);
  }, 20_000);

  it("surfaces a deployment/bind conflict from a guest that exits during startup", async () => {
    const { broker } = createBroker();
    installDeps({ broker });

    await expect(startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      command: guestCommand("dev-runner-bind-conflict.mjs"),
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    }))).rejects.toThrow(
      /deployment\/bind conflict: local_trusted requires server\.bind=loopback.*output: local_trusted requires server\.bind=loopback/s,
    );
  }, 20_000);
});

describe("readiness probes loopback for an exposed runtime (PAP-17256)", () => {
  it("starts even when the only readiness target is the MagicDNS display URL", async () => {
    // The live project config declares a loopback readiness URL, but when it does
    // not, readiness falls back to `expose.urlTemplate` — a MagicDNS name that
    // resolves off-loopback. That only ever answered because the guest was bound
    // to the wildcard, so it must be normalised alongside the bind fix.
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const input = startInput({
      serviceName: "paperclip-dev",
      command: `${guestCommand("dev-runner.mjs")} --bind lan`,
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    });
    // Drop the explicit loopback readiness URL so the fallback path is exercised.
    const service = input.config.workspaceRuntime.services[0] as Record<string, unknown>;
    service.readiness = { type: "http", timeoutSec: 10, intervalMs: 100 };

    const [runtime] = await startRuntimeServicesForWorkspaceControl(input);

    expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
    expect(runtime.exposure?.state).toBe("ready");
    expect(runtime.url).toBe(`https://runner.tail123.ts.net:${runtime.port}`);
  }, 20_000);
});

describe("the deployed failure shape: loopback app port, wildcard HMR (PAP-17256)", () => {
  // This acceptance fixture inspects live Linux /proc socket tables.
  it.skipIf(process.platform !== "linux")("fails terminally naming the HMR port, because forcing the bind cannot reach Vite's own listener", async () => {
    // Plain master's app.ts passes Vite `hmr.port` without `hmr.server` or
    // `server.host`, so the HMR websocket binds `::` no matter what the bind mode
    // is. The argv rewrite fixes the app port; only the preflight catches this.
    const { broker, calls } = createBroker();
    installDeps({ broker });

    const error = await startRuntimeServicesForWorkspaceControl(startInput({
      serviceName: "paperclip-dev",
      command: `${guestCommand("dev-runner-wildcard-hmr.mjs")} --bind lan`,
      expose: LEGACY_HTTP_EXPOSE,
      port: { type: "auto", envKey: "PORT" },
    })).then(() => null, (err: unknown) => err as Error);

    expect(error).not.toBeNull();
    expect(error!.message).toContain("listener_ownership_mismatch");
    // The app port obeyed the forced bind; the HMR companion is the offender, and
    // the diagnosis has to say so rather than blaming the lane as a whole.
    expect(error!.message).toMatch(/port 5\d{4} is bound to/);
    expect(error!.message).not.toMatch(/port 4\d{4} is bound to/);
    expect(calls).toEqual(["reserve", "remove"]);
  }, 20_000);
});

describe.each([false, true])("guest exposure-port collision reporting (base pair occupied: %s)", (basePairOccupied) => {
  // The quarantine decision itself is unit-tested through
  // `classifyExposureHostCollisions` below. A deterministic end-to-end quarantine
  // test is not reachable here: a real host listener that holds the assigned port
  // is intercepted earlier, either by the pre-spawn ownership guard or by the
  // allocated-port bind wait, before the EADDRINUSE-text quarantine branch runs.
  it("does not quarantine the pair when the guest fabricates an assigned-port EADDRINUSE with no host listener", async () => {
    const { broker } = createBroker();
    const reservedAppPorts: number[] = [];
    const recordingBroker: BrokerClient = {
      ...broker,
      async reserve(runtimeId, requested) {
        reservedAppPorts.push(requested[0]!.port);
        return broker.reserve(runtimeId, requested);
      },
    };
    installDeps({
      broker: recordingBroker,
      isPortAvailable: (port) => basePairOccupied &&
        [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)
        ? Promise.resolve(false) : isLoopbackPortFree(port),
    });

    const logs: string[] = [];
    const error = await startRuntimeServicesForWorkspaceControl({
      ...startInput({
        serviceName: "paperclip-dev",
        command: `${guestCommand("dev-runner-eaddrinuse-synthetic.mjs")} --bind lan`,
        expose: LEGACY_HTTP_EXPOSE,
        port: { type: "auto", envKey: "PORT" },
      }),
      onLog: async (_stream, chunk) => {
        logs.push(chunk);
      },
    }).then(() => null, (err: unknown) => err as Error);

    // The assigned pair is free, so the printed EADDRINUSE line is unverified.
    // The start fails terminally after ONE allocation and never burns the pool.
    expect(error).not.toBeNull();
    expect(error!.message).toContain(String(reservedAppPorts[0]));
    expect(error!.message).toContain("not verified");
    expect(reservedAppPorts).toHaveLength(1);
    expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));
    expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);

    // No quarantine and no re-allocation happened for the unverified claim.
    const diagnosis = logs.join("");
    expect(diagnosis).not.toContain("Quarantined pair");
    expect(diagnosis).not.toContain("collided during startup");
  }, 25_000);

  it("does not quarantine the pair for an EADDRINUSE on an unrelated auxiliary port", async () => {
    const { broker } = createBroker();
    const reservedAppPorts: number[] = [];
    const recordingBroker: BrokerClient = {
      ...broker,
      async reserve(runtimeId, requested) {
        reservedAppPorts.push(requested[0]!.port);
        return broker.reserve(runtimeId, requested);
      },
    };
    installDeps({
      broker: recordingBroker,
      isPortAvailable: (port) => basePairOccupied &&
        [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)
        ? Promise.resolve(false) : isLoopbackPortFree(port),
    });

    const logs: string[] = [];
    const error = await startRuntimeServicesForWorkspaceControl({
      ...startInput({
        serviceName: "paperclip-dev",
        command: `${guestCommand("dev-runner-eaddrinuse-auxiliary.mjs")} --bind lan`,
        expose: LEGACY_HTTP_EXPOSE,
        port: { type: "auto", envKey: "PORT" },
      }),
      onLog: async (_stream, chunk) => {
        logs.push(chunk);
      },
    }).then(() => null, (err: unknown) => err as Error);

    // The failure names an unrelated port, so the assigned pair is not a
    // collision. The start fails terminally after ONE allocation, and never
    // burns the bounded retries on a valid pair.
    expect(error).not.toBeNull();
    expect(error!.message).toContain("39999");
    expect(reservedAppPorts).toHaveLength(1);
    expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));
    expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);

    // No quarantine and no re-allocation happened for the auxiliary conflict.
    const diagnosis = logs.join("");
    expect(diagnosis).not.toContain("Quarantined pair");
    expect(diagnosis).not.toContain("collided during startup");
  }, 25_000);

  it("does not quarantine when an auxiliary EADDRINUSE mixes with a benign assigned-port line", async () => {
    const { broker } = createBroker();
    const reservedAppPorts: number[] = [];
    const recordingBroker: BrokerClient = {
      ...broker,
      async reserve(runtimeId, requested) {
        reservedAppPorts.push(requested[0]!.port);
        return broker.reserve(runtimeId, requested);
      },
    };
    installDeps({
      broker: recordingBroker,
      isPortAvailable: (port) => basePairOccupied &&
        [RUNTIME_EXPOSURE_APP_PORT_MIN, deriveViteHmrPort(RUNTIME_EXPOSURE_APP_PORT_MIN)].includes(port)
        ? Promise.resolve(false) : isLoopbackPortFree(port),
    });

    const logs: string[] = [];
    const error = await startRuntimeServicesForWorkspaceControl({
      ...startInput({
        serviceName: "paperclip-dev",
        command: `${guestCommand("dev-runner-eaddrinuse-auxiliary-mixed.mjs")} --bind lan`,
        expose: LEGACY_HTTP_EXPOSE,
        port: { type: "auto", envKey: "PORT" },
      }),
      onLog: async (_stream, chunk) => {
        logs.push(chunk);
      },
    }).then(() => null, (err: unknown) => err as Error);

    // The assigned port appears on a benign line, but EADDRINUSE names only
    // the auxiliary port 39999. The parser matches the error and the port on the
    // same line, so the assigned pair is not a collision. The start fails
    // terminally after ONE allocation and never quarantines the valid pair.
    expect(error).not.toBeNull();
    expect(error!.message).toContain("39999");
    expect(reservedAppPorts).toHaveLength(1);
    expect(reservedAppPorts[0]).toBeGreaterThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MIN + (basePairOccupied ? 1 : 0));
    expect(reservedAppPorts[0]).toBeLessThanOrEqual(RUNTIME_EXPOSURE_APP_PORT_MAX);

    const diagnosis = logs.join("");
    expect(diagnosis).not.toContain("Quarantined pair");
    expect(diagnosis).not.toContain("collided during startup");
  }, 25_000);
});

describe("classifyExposureHostCollisions gates quarantine on verified host state", () => {
  const state = (over: Partial<ExposurePortHostState> & { port: number }): ExposurePortHostState => ({
    named: false,
    listenerPresent: false,
    ownerPid: null,
    ownerProcessGroupId: null,
    ...over,
  });

  it("reports a host collision when a named port has a present listener owned by another process", () => {
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [
        state({ port: 42_000, named: true, listenerPresent: true, ownerPid: 9999 }),
        state({ port: 52_000, named: false, listenerPresent: false, ownerPid: null }),
      ],
    });
    expect(result).toEqual({ hostCollisionPorts: [42_000], hostCollision: true });
  });

  it("treats a present listener with an unknown owner as a host collision", () => {
    // /proc proves a listener, and a guest that lost its assigned port does not
    // hold it, so an unknown owner still counts as the host.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [state({ port: 42_000, named: true, listenerPresent: true, ownerPid: null })],
    });
    expect(result).toEqual({ hostCollisionPorts: [42_000], hostCollision: true });
  });

  it("does not report a collision when the named port has no listener", () => {
    // A synthetic EADDRINUSE line with no host listener behind it. Quarantine here
    // would drain the shared exposure-port pool across repeated starts.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [state({ port: 42_000, named: true, listenerPresent: false, ownerPid: null })],
    });
    expect(result).toEqual({ hostCollisionPorts: [], hostCollision: false });
  });

  it("does not report a collision when the guest itself owns the present listener", () => {
    // The guest bound and holds its own port, so this is not a host collision.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [state({ port: 42_000, named: true, listenerPresent: true, ownerPid: 4242 })],
    });
    expect(result).toEqual({ hostCollisionPorts: [], hostCollision: false });
  });

  it("does not report a collision when a guest descendant owns the present listener", () => {
    // The runtime launches the guest as a shell process group leader (pid 4242).
    // The real dev server binds the port from a descendant (pid 9999) that shares
    // the shell process group. The owner pid differs from the shell pid, but the
    // owner process group id matches it, so this is the guest, not the host. A raw
    // pid equality would quarantine this valid pair until the shared pool drains.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [
        state({
          port: 42_000,
          named: true,
          listenerPresent: true,
          ownerPid: 9999,
          ownerProcessGroupId: 4242,
        }),
      ],
    });
    expect(result).toEqual({ hostCollisionPorts: [], hostCollision: false });
  });

  it("reports a host collision when the owner and its process group are both external", () => {
    // A different process group holds the port, so neither the shell pid nor the
    // process group matches. This is a real external owner and quarantine is right.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [
        state({
          port: 42_000,
          named: true,
          listenerPresent: true,
          ownerPid: 9999,
          ownerProcessGroupId: 8888,
        }),
      ],
    });
    expect(result).toEqual({ hostCollisionPorts: [42_000], hostCollision: true });
  });

  it("ignores a present listener on a port the failure text did not name", () => {
    // An auxiliary-port conflict leaves the assigned pair valid.
    const result = classifyExposureHostCollisions({
      childPid: 4242,
      ports: [state({ port: 42_000, named: false, listenerPresent: true, ownerPid: 9999 })],
    });
    expect(result).toEqual({ hostCollisionPorts: [], hostCollision: false });
  });
});