feat(runtime): backfill pre-existing HTTP-only managed runtimes to HTTPS (PAP-17158)

The automatic default only covers services that start after the deploy. A
workspace that was already running when the feature landed would keep serving
`http://127.0.0.1:<port>` as its canonical URL indefinitely, because startup
reconciliation adopted any healthy backend as-is.

Startup reconciliation now classifies every persisted managed row through
`decideManagedRuntimeExposureBackfill` and stops the eligible HTTP-only ones, so
the desired-state restart that already runs at the end of reconciliation brings
them back through the ordinary fail-closed exposure lifecycle — same workspace,
same runtime-service row, relocated port only when the old one cannot be
published.

The decision function is pure and every branch carries a reason code, because
"which pre-feature workspaces get touched" is the whole risk of this change:

- `already_exposed` makes repeated deploys and restarts converge instead of
  churning listeners.
- `deliberate_opt_out` is checked before broker availability, so an opt-out never
  depends on host state.
- `no_configured_service_entry` refuses to stop a row we could not restart —
  reprovisioning works by stopping the backend and letting desired state bring it
  back, so without a config entry we would take a service down for good.
- `stopped_defaults_on_next_start` leaves stopped rows to the start path.

A backfilled row also drops its HTTP `url` at the moment it is stopped rather
than keeping it until the restart succeeds: if the restart fails, the fail-closed
contract is no URL, not a working HTTP one.

The shared supertest setup pins `PAPERCLIP_MANAGED_RUNTIME_HTTPS=off` so suites
that start a service named `paperclip-dev` do not behave differently on a
broker-capable host than on CI; the suites that exercise the default opt back in.

Covered: 12 decision branches, plus an end-to-end integration test that starts a
pre-feature workspace on a pinned legacy port over plain HTTP, reconciles, and
asserts the same row comes back on a verified `https://<host>:<42xxx>` URL with
the old HTTP backend gone — then reconciles again and asserts zero broker calls
and an unchanged port.
This commit is contained in:
Dotta authored and Dotta committed 2026-08-16 19:26:39 +00:00
1 parent 3ded2a0ed5
commit 2b21fe01bc
6 files changed
+495 -4

No files matched your search

+8
View File
@@ -30,6 +30,14 @@ if (!process.env.CODEX_HOME) {
process.env.CODEX_HOME = codexHome;
}
// The automatic Tailscale HTTPS default (PAP-17158) probes for a real host
// broker socket, so leaving it enabled would make every test that starts a
// service named `paperclip-dev` behave differently on a broker-capable host
// than on CI. Tests that exercise the default opt in explicitly.
if (!process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS) {
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "off";
}
if (!SupertestTest.prototype.__paperclipLoopbackPatched) {
SupertestTest.prototype.serverAddress = function serverAddress(app, path) {
const addr = app.address();
@@ -41,8 +41,10 @@ import {
resolveWorkspaceRuntimeReadinessTimeoutSec,
resolveShell,
sanitizeRuntimeServiceBaseEnv,
setWorkspaceRuntimeExposureDepsForTests,
startRuntimeServicesForWorkspaceControl,
stopRuntimeServicesForExecutionWorkspace,
stopRuntimeServicesForProjectWorkspace,
WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS,
type RealizedExecutionWorkspace,
} from "../services/workspace-runtime.ts";
@@ -6565,6 +6567,222 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
}
}, 20_000);
it("backfills a pre-existing HTTP-only managed worktree runtime to verified HTTPS in place", async () => {
// PAP-17158: an eligible workspace created before the feature must come
// forward on the *same* workspace/runtime-service row — not by recreating it
// — and must never keep its HTTP URL as a healthy fallback.
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-https-backfill-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
const previousHttpsMode = process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
process.env.PAPERCLIP_HOME = paperclipHome;
process.env.PAPERCLIP_INSTANCE_ID = `runtime-https-backfill-${randomUUID()}`;
const reservePort = async () => {
const probe = net.createServer();
await new Promise<void>((resolve) => probe.listen(0, "127.0.0.1", resolve));
const address = probe.address();
const port = typeof address === "object" && address ? address.port : null;
await new Promise<void>((resolve, reject) => {
probe.close((error) => error ? reject(error) : resolve());
});
if (!port) throw new Error("Failed to reserve HTTPS backfill test port");
return port;
};
const isLoopbackPortFree = async (port: number) => {
const probe = net.createServer();
return await new Promise<boolean>((resolve) => {
probe.once("error", () => resolve(false));
probe.listen(port, "127.0.0.1", () => {
probe.close(() => resolve(true));
});
});
};
// Stands in for the persisted template's hard-coded 45439: a pinned port
// outside the broker's dedicated allowlist, so the backfill has to relocate.
const legacyPort = await reservePort();
const companyId = randomUUID();
const projectId = randomUUID();
const projectWorkspaceId = randomUUID();
// Binds the app port and its HMR companion, both loopback-only.
const command =
"node -e \"const http=require('node:http');const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((req,res)=>res.end('ok')).listen(q,'127.0.0.1');setInterval(()=>{},1000)\"";
const workspaceRuntime = {
services: [
{
name: "paperclip-dev",
command,
port: legacyPort,
// The pre-feature block: backend URL only, no exposure declaration.
expose: { type: "url", urlTemplate: "http://127.0.0.1:{{port}}" },
readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 },
lifecycle: "shared",
reuseScope: "project_workspace",
stopPolicy: { type: "manual" },
},
],
};
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(projects).values({
id: projectId,
companyId,
name: "Runtime HTTPS backfill",
status: "in_progress",
});
await db.insert(projectWorkspaces).values({
id: projectWorkspaceId,
companyId,
projectId,
name: "Primary",
sourceType: "local_path",
cwd: workspaceRoot,
isPrimary: true,
metadata: {
runtimeConfig: {
workspaceRuntime,
desiredState: "running",
serviceStates: { "0": "running" },
},
},
});
const actor = { id: null, name: "Paperclip", companyId };
const workspace = {
...buildWorkspace(workspaceRoot),
projectId,
workspaceId: projectWorkspaceId,
};
// A broker fake that remembers what it published, so a second reconciliation
// can recognize its own live listeners instead of tearing them down.
const exposedByRuntimeId = new Map<string, { port: number; purpose: "app" | "vite_hmr" }[]>();
const pendingByHandle = new Map<string, { runtimeId: string; listeners: { port: number; purpose: "app" | "vite_hmr" }[] }>();
let handleCounter = 0;
const brokerCalls: string[] = [];
const broker = {
async reserve(runtimeId: string, listeners: { port: number; purpose: "app" | "vite_hmr" }[]) {
brokerCalls.push("reserve");
handleCounter += 1;
const handle = `backfill-handle-${handleCounter}`;
pendingByHandle.set(handle, { runtimeId, listeners: listeners.map((l) => ({ ...l })) });
return { handle, reservedPorts: listeners.map((listener) => listener.port) };
},
async expose(runtimeId: string, handle: string) {
brokerCalls.push("expose");
const pending = pendingByHandle.get(handle);
if (!pending || pending.runtimeId !== runtimeId) throw new Error("listener_ownership_mismatch");
exposedByRuntimeId.set(runtimeId, pending.listeners);
return { handle, publicPorts: pending.listeners.map((listener) => listener.port) };
},
async remove(runtimeId: string, handle: string) {
brokerCalls.push("remove");
const listeners = exposedByRuntimeId.get(runtimeId) ?? pendingByHandle.get(handle)?.listeners ?? [];
exposedByRuntimeId.delete(runtimeId);
pendingByHandle.delete(handle);
return { removedPorts: listeners.map((listener) => listener.port) };
},
async list() {
return [...exposedByRuntimeId.entries()].flatMap(([runtimeId, listeners]) =>
listeners.map((listener) => ({ runtimeId, port: listener.port, purpose: listener.purpose })),
);
},
};
const installExposureDeps = () => setWorkspaceRuntimeExposureDepsForTests({
broker,
isPortAvailable: isLoopbackPortFree,
isBrokerAvailable: async () => true,
resolveHostname: async () => "runner.tail123.ts.net",
probeHealth: async () => true,
now: () => new Date().toISOString(),
});
try {
// ---- Before: the workspace as it exists today, on plain HTTP. ----
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "off";
const before = await startRuntimeServicesForWorkspaceControl({
db,
actor,
issue: null,
workspace,
config: { workspaceRuntime, desiredState: "running", serviceStates: { "0": "running" } },
adapterEnv: {},
});
const runtimeServiceId = before[0]?.id;
expect(runtimeServiceId).toBeTruthy();
expect(before[0]?.port).toBe(legacyPort);
expect(before[0]?.url).toBe(`http://127.0.0.1:${legacyPort}`);
const [httpRow] = await db.select().from(workspaceRuntimeServices);
expect(httpRow.exposure).toBeNull();
expect(httpRow.url).toBe(`http://127.0.0.1:${legacyPort}`);
await expect(fetch(`http://127.0.0.1:${legacyPort}`)).resolves.toMatchObject({ ok: true });
expect(brokerCalls).toEqual([]);
// ---- Deploy: the feature turns on and Paperclip restarts. ----
await resetRuntimeServicesForTests();
delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
installExposureDeps();
const result = await reconcilePersistedRuntimeServicesOnStartup(db);
expect(result).toMatchObject({ backfilled: 1, restarted: 1, restartFailed: 0 });
expect(brokerCalls).toEqual(["reserve", "expose"]);
// ---- After: same row, verified HTTPS URL, no HTTP anywhere. ----
const afterRows = await db.select().from(workspaceRuntimeServices);
expect(afterRows).toHaveLength(1);
const httpsRow = afterRows[0]!;
expect(httpsRow.id).toBe(runtimeServiceId);
expect(httpsRow.status).toBe("running");
expect(httpsRow.port).not.toBe(legacyPort);
expect(httpsRow.port).toBeGreaterThanOrEqual(42_000);
expect(httpsRow.port).toBeLessThanOrEqual(42_999);
expect(httpsRow.url).toBe(`https://runner.tail123.ts.net:${httpsRow.port}`);
expect(httpsRow.exposure).toMatchObject({
provider: "tailscale_https",
state: "ready",
publicUrl: `https://runner.tail123.ts.net:${httpsRow.port}`,
hostname: "runner.tail123.ts.net",
});
expect(httpsRow.exposureHandle).toBeTruthy();
// The old HTTP backend is gone, not merely shadowed.
await expect(fetch(`http://127.0.0.1:${legacyPort}`)).rejects.toThrow();
// ---- Repeat deploy: converges, no listener churn, same port. ----
await resetRuntimeServicesForTests();
installExposureDeps();
brokerCalls.length = 0;
const second = await reconcilePersistedRuntimeServicesOnStartup(db);
expect(second).toMatchObject({ backfilled: 0 });
expect(brokerCalls).toEqual([]);
const [repeatRow] = await db.select().from(workspaceRuntimeServices);
expect(repeatRow.port).toBe(httpsRow.port);
expect(repeatRow.url).toBe(httpsRow.url);
expect(repeatRow.status).toBe("running");
} finally {
await stopRuntimeServicesForProjectWorkspace({
db,
projectWorkspaceId,
workspaceCwd: workspaceRoot,
}).catch(() => undefined);
await resetRuntimeServicesForTests();
await fs.rm(paperclipHome, { recursive: true, force: true });
await fs.rm(workspaceRoot, { recursive: true, force: true });
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
if (previousHttpsMode === undefined) delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
else process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = previousHttpsMode;
}
}, 40_000);
it("adopts a live auto-port shared service after runtime state is reset", async () => {
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-reconcile-"));
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
+9 -1
View File
@@ -818,12 +818,20 @@ export async function startServer(): Promise<StartedServer> {
void reconcilePersistedRuntimeServicesOnStartup(db as any)
.then((result) => {
if (result.reconciled > 0 || result.restarted > 0 || result.restartFailed > 0) {
if (
result.reconciled > 0
|| result.restarted > 0
|| result.restartFailed > 0
|| result.backfilled > 0
) {
logger.warn(
{
reconciled: result.reconciled,
adopted: result.adopted,
stopped: result.stopped,
// Managed HTTP-only services taken down so they come back on a
// verified HTTPS origin (PAP-17158).
httpsBackfilled: result.backfilled,
restarted: result.restarted,
restartFailed: result.restartFailed,
},
@@ -0,0 +1,117 @@
/**
* Backfill decision coverage for PAP-17158.
*
* `decideManagedRuntimeExposureBackfill` is the whole contract for which
* pre-feature workspaces get upgraded to HTTPS in place and which are left
* alone, so each branch is asserted by its reason code rather than only by the
* resulting action.
*/
import { describe, expect, it } from "vitest";
import { decideManagedRuntimeExposureBackfill } from "./workspace-runtime.js";
/** A persisted, running, pre-feature Paperclip App dev runtime. */
function httpOnlyRunningRow(overrides: Partial<Parameters<typeof decideManagedRuntimeExposureBackfill>[0]> = {}) {
return {
mode: "auto" as const,
brokerAvailable: true,
provider: "local_process",
serviceName: "paperclip-dev",
command: "pnpm dev --bind lan",
status: "running",
hasExposure: false,
declaredIntent: "unset" as const,
...overrides,
};
}
describe("decideManagedRuntimeExposureBackfill", () => {
it("reprovisions a running HTTP-only managed worktree runtime", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow())).toEqual({
action: "reprovision",
reason: "http_only_managed_service",
});
});
it("reprovisions a service that is mid-start as well as one already running", () => {
for (const status of ["provisioning", "starting", "running"]) {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ status })).action).toBe("reprovision");
}
});
it("leaves a stopped service to pick the default up on its next start", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ status: "stopped" }))).toEqual({
action: "keep",
reason: "stopped_defaults_on_next_start",
});
});
it("preserves a deliberate opt-out", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: "disabled" }))).toEqual({
action: "keep",
reason: "deliberate_opt_out",
});
});
it("is idempotent: a row that already carries exposure state is never re-driven", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ hasExposure: true }))).toEqual({
action: "keep",
reason: "already_exposed",
});
// Repeated deploys see the same row and must keep converging on "keep".
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ hasExposure: true })).action).toBe("keep");
});
it("leaves unmanaged and custom services alone", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({
serviceName: "preview",
command: "pnpm vite",
}))).toEqual({ action: "keep", reason: "unmanaged_or_custom_service" });
});
it("leaves external (non local_process) runtime services alone", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ provider: "external" }))).toEqual({
action: "keep",
reason: "not_a_managed_local_process",
});
});
it("does nothing when the automatic default is switched off", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ mode: "off" }))).toEqual({
action: "keep",
reason: "https_default_disabled",
});
});
it("skips the backfill when the host broker is unavailable, unless forced", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ brokerAvailable: false }))).toEqual({
action: "keep",
reason: "broker_unavailable",
});
expect(
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ brokerAvailable: false, mode: "force" })).action,
).toBe("reprovision");
});
it("never takes down a row it cannot restart from a configured entry", () => {
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: null }))).toEqual({
action: "keep",
reason: "no_configured_service_entry",
});
});
it("reprovisions an explicit opt-in that is somehow running without exposure", () => {
expect(
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: "enabled" })).action,
).toBe("reprovision");
});
it("checks the opt-out before broker availability so an opt-out never depends on host state", () => {
expect(
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({
declaredIntent: "disabled",
brokerAvailable: false,
})).reason,
).toBe("deliberate_opt_out");
});
});
@@ -1,4 +1,4 @@
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import type { BrokerClient, BrokerListenerRequest } from "./runtime-exposure/broker-client.js";
import {
@@ -11,7 +11,18 @@ import {
const EXECUTION_WORKSPACE_ID = "11111111-2222-4333-8444-555566667777";
const HANDLE = "handle-abcdef1234567890";
// The shared test setup pins the automatic default off so unrelated suites do
// not probe for a real host broker. This suite is about the default, so it opts
// back in and restores the harness value afterwards.
let previousHttpsMode: string | undefined;
beforeEach(() => {
previousHttpsMode = process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "auto";
});
afterEach(async () => {
if (previousHttpsMode === undefined) delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
else process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = previousHttpsMode;
await resetRuntimeServicesForTests();
});
+131 -2
View File
@@ -15,6 +15,7 @@ import {
readRuntimeExposureIntent,
resolveDeclaredRuntimeExposureConfig,
type RuntimeExposureConfigInput,
type RuntimeExposureIntent,
type RuntimeExposureStatus,
type GitWorktreeBranchAncestryVerdict,
type GitWorktreeBranchIncoherenceEvidence as SharedGitWorktreeBranchIncoherenceEvidence,
@@ -6272,6 +6273,96 @@ export async function listWorkspaceRuntimeServicesForProjectWorkspaces(
return grouped;
}
/**
* Statuses that mean "there is, or is supposed to be, a live backend process".
* A row in one of these states is what the backfill has to reprovision; a
* `stopped` row simply picks the default up on its next start.
*/
const LIVE_RUNTIME_SERVICE_STATUSES = new Set(["provisioning", "starting", "running"]);
export type ManagedRuntimeExposureBackfillDecision = {
action: "keep" | "reprovision";
reason: string;
};
/**
* Decide what the HTTPS backfill should do with one persisted runtime-service
* row (PAP-17158).
*
* Pure so every branch is directly testable: the reasons below are the whole
* contract for which pre-feature workspaces get upgraded and which are left
* exactly as they are.
*
* `declaredIntent === null` means no configured service entry could be matched
* to this row. Such a row is deliberately left alone: reprovisioning works by
* stopping the HTTP backend and letting the desired-state restart bring it back
* with exposure, so without a config to restart from we would take a service
* down and never bring it back.
*/
export function decideManagedRuntimeExposureBackfill(input: {
mode: ManagedRuntimeHttpsMode;
brokerAvailable: boolean;
provider: string;
serviceName: string;
command: string | null;
status: string;
hasExposure: boolean;
declaredIntent: RuntimeExposureIntent | null;
}): ManagedRuntimeExposureBackfillDecision {
if (input.mode === "off") return { action: "keep", reason: "https_default_disabled" };
if (input.provider !== "local_process") return { action: "keep", reason: "not_a_managed_local_process" };
// Idempotence: a row that already carries exposure state is never re-driven,
// so repeated deploys and restarts converge instead of churning listeners.
if (input.hasExposure) return { action: "keep", reason: "already_exposed" };
if (input.declaredIntent === "disabled") return { action: "keep", reason: "deliberate_opt_out" };
if (input.declaredIntent === null) return { action: "keep", reason: "no_configured_service_entry" };
if (!isManagedHttpsDefaultCandidate({ serviceName: input.serviceName, command: input.command })) {
return { action: "keep", reason: "unmanaged_or_custom_service" };
}
if (input.mode !== "force" && !input.brokerAvailable) {
return { action: "keep", reason: "broker_unavailable" };
}
if (!LIVE_RUNTIME_SERVICE_STATUSES.has(input.status)) {
return { action: "keep", reason: "stopped_defaults_on_next_start" };
}
return { action: "reprovision", reason: "http_only_managed_service" };
}
/**
* Look up the exposure intent a persisted runtime-service row inherits from its
* owning workspace configuration, by matching the row's service name against the
* configured entries. Returns null when no entry matches.
*/
async function buildPersistedRuntimeExposureIntentLookup(db: Db) {
const [projectWorkspaceRows, executionWorkspaceRows] = await Promise.all([
db.select().from(projectWorkspaces),
db.select().from(executionWorkspaces),
]);
const projectRuntimeById = new Map(projectWorkspaceRows.map((row) => [
row.id,
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime ?? null,
] as const));
const executionRuntimeById = new Map(executionWorkspaceRows.map((row) => [
row.id,
readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime
?? (row.projectWorkspaceId ? projectRuntimeById.get(row.projectWorkspaceId) ?? null : null),
] as const));
return (row: {
serviceName: string;
projectWorkspaceId: string | null;
executionWorkspaceId: string | null;
}): RuntimeExposureIntent | null => {
const runtime = (row.executionWorkspaceId ? executionRuntimeById.get(row.executionWorkspaceId) : null)
?? (row.projectWorkspaceId ? projectRuntimeById.get(row.projectWorkspaceId) ?? null : null);
if (!runtime) return null;
const entries = listConfiguredRuntimeServiceEntries({ workspaceRuntime: runtime });
const entry = entries.find((candidate) => asString(candidate.name, "service") === row.serviceName);
if (!entry) return null;
return readRuntimeExposureIntent(parseObject(entry.expose));
};
}
export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
const rows = await db
.select()
@@ -6283,6 +6374,13 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
),
);
// Backfill inputs, resolved once per startup rather than per row.
const httpsMode = resolveManagedRuntimeHttpsMode();
const brokerAvailable = httpsMode === "off"
? false
: await workspaceRuntimeExposureDeps.isBrokerAvailable().catch(() => false);
const readDeclaredExposureIntent = await buildPersistedRuntimeExposureIntentLookup(db);
let ownedExposureListeners: Awaited<ReturnType<BrokerClient["list"]>> | null = [];
if (rows.some((row) => row.exposure && row.exposure.state !== "removed")) {
try {
@@ -6295,6 +6393,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
let reconciled = 0;
let adopted = 0;
let stopped = 0;
let backfilled = 0;
for (const row of rows) {
if (row.status === "stopped" && row.exposure && row.exposure.state !== "removed") {
await cleanupPersistedExposureRows(db, [{
@@ -6321,6 +6420,25 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
new URL("/api/health", row.exposure.publicUrl!).toString(),
)
);
// Pre-feature rows carry no exposure state at all. An eligible one that is
// still serving plain HTTP must not be adopted as-is, or the deploy would
// leave `http://paperclip-dev:<port>` as the canonical URL forever. Stopping
// it here hands it to the desired-state restart below, which brings it back
// through the normal fail-closed exposure lifecycle.
const backfillDecision = decideManagedRuntimeExposureBackfill({
mode: httpsMode,
brokerAvailable,
provider: row.provider,
serviceName: row.serviceName,
command: row.command,
status: row.status,
hasExposure: Boolean(row.exposure && row.exposure.state !== "removed"),
declaredIntent: readDeclaredExposureIntent({
serviceName: row.serviceName,
projectWorkspaceId: row.projectWorkspaceId ?? null,
executionWorkspaceId: row.executionWorkspaceId ?? null,
}),
});
let adoptedRecord = await findLocalServiceRegistryRecordByRuntimeServiceId({
runtimeServiceId: row.id,
profileKind: "workspace-runtime",
@@ -6366,9 +6484,11 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
if (adoptedRecord) {
const adoptedUrl = adoptedRecord.url ?? row.backendUrl ?? row.url ?? null;
if (
!exposureHealthMatches
backfillDecision.action === "reprovision"
|| !exposureHealthMatches
|| !(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))
) {
if (backfillDecision.action === "reprovision") backfilled += 1;
await terminateLocalService(adoptedRecord);
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
} else {
@@ -6452,7 +6572,10 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
.set({
status: "stopped",
healthStatus: "unknown",
url: stoppedExposure ? null : row.url,
// A row queued for HTTPS backfill drops its HTTP URL now rather than
// keeping it until the restart succeeds: if the restart fails, the
// fail-closed contract says show no URL, not a working HTTP one.
url: stoppedExposure || backfillDecision.action === "reprovision" ? null : row.url,
exposure: stoppedExposure,
exposureHandle: stoppedExposureHandle,
stoppedAt: now,
@@ -6476,12 +6599,18 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
// was never committed. Re-applying persisted desired state adopts that
// registry entry (or restarts a missing service) and makes it visible to
// workspace cleanup through workspace_runtime_services again.
//
// It is also the second half of the HTTPS backfill: services stopped above as
// HTTP-only come back here through the ordinary start path, which now applies
// the `tailscale_https` default and only reports them healthy behind a
// verified HTTPS URL.
const desiredState = await restartDesiredRuntimeServicesOnStartup(db);
return {
reconciled,
adopted,
stopped,
backfilled,
restarted: desiredState.restarted,
restartFailed: desiredState.failed,
};