mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 20:21:50 +01:00
feat(runtime): backfill pre-existing HTTP-only managed runtimes to HTTPS (PAP-17158)
The automatic default only covers services that start after the deploy. A workspace that was already running when the feature landed would keep serving `http://127.0.0.1:<port>` as its canonical URL indefinitely, because startup reconciliation adopted any healthy backend as-is. Startup reconciliation now classifies every persisted managed row through `decideManagedRuntimeExposureBackfill` and stops the eligible HTTP-only ones, so the desired-state restart that already runs at the end of reconciliation brings them back through the ordinary fail-closed exposure lifecycle — same workspace, same runtime-service row, relocated port only when the old one cannot be published. The decision function is pure and every branch carries a reason code, because "which pre-feature workspaces get touched" is the whole risk of this change: - `already_exposed` makes repeated deploys and restarts converge instead of churning listeners. - `deliberate_opt_out` is checked before broker availability, so an opt-out never depends on host state. - `no_configured_service_entry` refuses to stop a row we could not restart — reprovisioning works by stopping the backend and letting desired state bring it back, so without a config entry we would take a service down for good. - `stopped_defaults_on_next_start` leaves stopped rows to the start path. A backfilled row also drops its HTTP `url` at the moment it is stopped rather than keeping it until the restart succeeds: if the restart fails, the fail-closed contract is no URL, not a working HTTP one. The shared supertest setup pins `PAPERCLIP_MANAGED_RUNTIME_HTTPS=off` so suites that start a service named `paperclip-dev` do not behave differently on a broker-capable host than on CI; the suites that exercise the default opt back in. Covered: 12 decision branches, plus an end-to-end integration test that starts a pre-feature workspace on a pinned legacy port over plain HTTP, reconciles, and asserts the same row comes back on a verified `https://<host>:<42xxx>` URL with the old HTTP backend gone — then reconciles again and asserts zero broker calls and an unchanged port.
This commit is contained in:
1 parent
3ded2a0ed5
commit
2b21fe01bc
6 files changed
+495
-4
No files matched your search
@@ -30,6 +30,14 @@ if (!process.env.CODEX_HOME) {
|
||||
process.env.CODEX_HOME = codexHome;
|
||||
}
|
||||
|
||||
// The automatic Tailscale HTTPS default (PAP-17158) probes for a real host
|
||||
// broker socket, so leaving it enabled would make every test that starts a
|
||||
// service named `paperclip-dev` behave differently on a broker-capable host
|
||||
// than on CI. Tests that exercise the default opt in explicitly.
|
||||
if (!process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS) {
|
||||
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "off";
|
||||
}
|
||||
|
||||
if (!SupertestTest.prototype.__paperclipLoopbackPatched) {
|
||||
SupertestTest.prototype.serverAddress = function serverAddress(app, path) {
|
||||
const addr = app.address();
|
||||
|
||||
@@ -41,8 +41,10 @@ import {
|
||||
resolveWorkspaceRuntimeReadinessTimeoutSec,
|
||||
resolveShell,
|
||||
sanitizeRuntimeServiceBaseEnv,
|
||||
setWorkspaceRuntimeExposureDepsForTests,
|
||||
startRuntimeServicesForWorkspaceControl,
|
||||
stopRuntimeServicesForExecutionWorkspace,
|
||||
stopRuntimeServicesForProjectWorkspace,
|
||||
WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS,
|
||||
type RealizedExecutionWorkspace,
|
||||
} from "../services/workspace-runtime.ts";
|
||||
@@ -6565,6 +6567,222 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => {
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("backfills a pre-existing HTTP-only managed worktree runtime to verified HTTPS in place", async () => {
|
||||
// PAP-17158: an eligible workspace created before the feature must come
|
||||
// forward on the *same* workspace/runtime-service row — not by recreating it
|
||||
// — and must never keep its HTTP URL as a healthy fallback.
|
||||
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-https-backfill-"));
|
||||
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
|
||||
const previousPaperclipHome = process.env.PAPERCLIP_HOME;
|
||||
const previousPaperclipInstanceId = process.env.PAPERCLIP_INSTANCE_ID;
|
||||
const previousHttpsMode = process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
|
||||
process.env.PAPERCLIP_HOME = paperclipHome;
|
||||
process.env.PAPERCLIP_INSTANCE_ID = `runtime-https-backfill-${randomUUID()}`;
|
||||
|
||||
const reservePort = async () => {
|
||||
const probe = net.createServer();
|
||||
await new Promise<void>((resolve) => probe.listen(0, "127.0.0.1", resolve));
|
||||
const address = probe.address();
|
||||
const port = typeof address === "object" && address ? address.port : null;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
probe.close((error) => error ? reject(error) : resolve());
|
||||
});
|
||||
if (!port) throw new Error("Failed to reserve HTTPS backfill test port");
|
||||
return port;
|
||||
};
|
||||
const isLoopbackPortFree = async (port: number) => {
|
||||
const probe = net.createServer();
|
||||
return await new Promise<boolean>((resolve) => {
|
||||
probe.once("error", () => resolve(false));
|
||||
probe.listen(port, "127.0.0.1", () => {
|
||||
probe.close(() => resolve(true));
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// Stands in for the persisted template's hard-coded 45439: a pinned port
|
||||
// outside the broker's dedicated allowlist, so the backfill has to relocate.
|
||||
const legacyPort = await reservePort();
|
||||
const companyId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
const projectWorkspaceId = randomUUID();
|
||||
// Binds the app port and its HMR companion, both loopback-only.
|
||||
const command =
|
||||
"node -e \"const http=require('node:http');const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((req,res)=>res.end('ok')).listen(q,'127.0.0.1');setInterval(()=>{},1000)\"";
|
||||
const workspaceRuntime = {
|
||||
services: [
|
||||
{
|
||||
name: "paperclip-dev",
|
||||
command,
|
||||
port: legacyPort,
|
||||
// The pre-feature block: backend URL only, no exposure declaration.
|
||||
expose: { type: "url", urlTemplate: "http://127.0.0.1:{{port}}" },
|
||||
readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 },
|
||||
lifecycle: "shared",
|
||||
reuseScope: "project_workspace",
|
||||
stopPolicy: { type: "manual" },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: "Runtime HTTPS backfill",
|
||||
status: "in_progress",
|
||||
});
|
||||
await db.insert(projectWorkspaces).values({
|
||||
id: projectWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
name: "Primary",
|
||||
sourceType: "local_path",
|
||||
cwd: workspaceRoot,
|
||||
isPrimary: true,
|
||||
metadata: {
|
||||
runtimeConfig: {
|
||||
workspaceRuntime,
|
||||
desiredState: "running",
|
||||
serviceStates: { "0": "running" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const actor = { id: null, name: "Paperclip", companyId };
|
||||
const workspace = {
|
||||
...buildWorkspace(workspaceRoot),
|
||||
projectId,
|
||||
workspaceId: projectWorkspaceId,
|
||||
};
|
||||
|
||||
// A broker fake that remembers what it published, so a second reconciliation
|
||||
// can recognize its own live listeners instead of tearing them down.
|
||||
const exposedByRuntimeId = new Map<string, { port: number; purpose: "app" | "vite_hmr" }[]>();
|
||||
const pendingByHandle = new Map<string, { runtimeId: string; listeners: { port: number; purpose: "app" | "vite_hmr" }[] }>();
|
||||
let handleCounter = 0;
|
||||
const brokerCalls: string[] = [];
|
||||
const broker = {
|
||||
async reserve(runtimeId: string, listeners: { port: number; purpose: "app" | "vite_hmr" }[]) {
|
||||
brokerCalls.push("reserve");
|
||||
handleCounter += 1;
|
||||
const handle = `backfill-handle-${handleCounter}`;
|
||||
pendingByHandle.set(handle, { runtimeId, listeners: listeners.map((l) => ({ ...l })) });
|
||||
return { handle, reservedPorts: listeners.map((listener) => listener.port) };
|
||||
},
|
||||
async expose(runtimeId: string, handle: string) {
|
||||
brokerCalls.push("expose");
|
||||
const pending = pendingByHandle.get(handle);
|
||||
if (!pending || pending.runtimeId !== runtimeId) throw new Error("listener_ownership_mismatch");
|
||||
exposedByRuntimeId.set(runtimeId, pending.listeners);
|
||||
return { handle, publicPorts: pending.listeners.map((listener) => listener.port) };
|
||||
},
|
||||
async remove(runtimeId: string, handle: string) {
|
||||
brokerCalls.push("remove");
|
||||
const listeners = exposedByRuntimeId.get(runtimeId) ?? pendingByHandle.get(handle)?.listeners ?? [];
|
||||
exposedByRuntimeId.delete(runtimeId);
|
||||
pendingByHandle.delete(handle);
|
||||
return { removedPorts: listeners.map((listener) => listener.port) };
|
||||
},
|
||||
async list() {
|
||||
return [...exposedByRuntimeId.entries()].flatMap(([runtimeId, listeners]) =>
|
||||
listeners.map((listener) => ({ runtimeId, port: listener.port, purpose: listener.purpose })),
|
||||
);
|
||||
},
|
||||
};
|
||||
const installExposureDeps = () => setWorkspaceRuntimeExposureDepsForTests({
|
||||
broker,
|
||||
isPortAvailable: isLoopbackPortFree,
|
||||
isBrokerAvailable: async () => true,
|
||||
resolveHostname: async () => "runner.tail123.ts.net",
|
||||
probeHealth: async () => true,
|
||||
now: () => new Date().toISOString(),
|
||||
});
|
||||
|
||||
try {
|
||||
// ---- Before: the workspace as it exists today, on plain HTTP. ----
|
||||
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "off";
|
||||
const before = await startRuntimeServicesForWorkspaceControl({
|
||||
db,
|
||||
actor,
|
||||
issue: null,
|
||||
workspace,
|
||||
config: { workspaceRuntime, desiredState: "running", serviceStates: { "0": "running" } },
|
||||
adapterEnv: {},
|
||||
});
|
||||
const runtimeServiceId = before[0]?.id;
|
||||
expect(runtimeServiceId).toBeTruthy();
|
||||
expect(before[0]?.port).toBe(legacyPort);
|
||||
expect(before[0]?.url).toBe(`http://127.0.0.1:${legacyPort}`);
|
||||
const [httpRow] = await db.select().from(workspaceRuntimeServices);
|
||||
expect(httpRow.exposure).toBeNull();
|
||||
expect(httpRow.url).toBe(`http://127.0.0.1:${legacyPort}`);
|
||||
await expect(fetch(`http://127.0.0.1:${legacyPort}`)).resolves.toMatchObject({ ok: true });
|
||||
expect(brokerCalls).toEqual([]);
|
||||
|
||||
// ---- Deploy: the feature turns on and Paperclip restarts. ----
|
||||
await resetRuntimeServicesForTests();
|
||||
delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
|
||||
installExposureDeps();
|
||||
|
||||
const result = await reconcilePersistedRuntimeServicesOnStartup(db);
|
||||
expect(result).toMatchObject({ backfilled: 1, restarted: 1, restartFailed: 0 });
|
||||
expect(brokerCalls).toEqual(["reserve", "expose"]);
|
||||
|
||||
// ---- After: same row, verified HTTPS URL, no HTTP anywhere. ----
|
||||
const afterRows = await db.select().from(workspaceRuntimeServices);
|
||||
expect(afterRows).toHaveLength(1);
|
||||
const httpsRow = afterRows[0]!;
|
||||
expect(httpsRow.id).toBe(runtimeServiceId);
|
||||
expect(httpsRow.status).toBe("running");
|
||||
expect(httpsRow.port).not.toBe(legacyPort);
|
||||
expect(httpsRow.port).toBeGreaterThanOrEqual(42_000);
|
||||
expect(httpsRow.port).toBeLessThanOrEqual(42_999);
|
||||
expect(httpsRow.url).toBe(`https://runner.tail123.ts.net:${httpsRow.port}`);
|
||||
expect(httpsRow.exposure).toMatchObject({
|
||||
provider: "tailscale_https",
|
||||
state: "ready",
|
||||
publicUrl: `https://runner.tail123.ts.net:${httpsRow.port}`,
|
||||
hostname: "runner.tail123.ts.net",
|
||||
});
|
||||
expect(httpsRow.exposureHandle).toBeTruthy();
|
||||
// The old HTTP backend is gone, not merely shadowed.
|
||||
await expect(fetch(`http://127.0.0.1:${legacyPort}`)).rejects.toThrow();
|
||||
|
||||
// ---- Repeat deploy: converges, no listener churn, same port. ----
|
||||
await resetRuntimeServicesForTests();
|
||||
installExposureDeps();
|
||||
brokerCalls.length = 0;
|
||||
const second = await reconcilePersistedRuntimeServicesOnStartup(db);
|
||||
expect(second).toMatchObject({ backfilled: 0 });
|
||||
expect(brokerCalls).toEqual([]);
|
||||
const [repeatRow] = await db.select().from(workspaceRuntimeServices);
|
||||
expect(repeatRow.port).toBe(httpsRow.port);
|
||||
expect(repeatRow.url).toBe(httpsRow.url);
|
||||
expect(repeatRow.status).toBe("running");
|
||||
} finally {
|
||||
await stopRuntimeServicesForProjectWorkspace({
|
||||
db,
|
||||
projectWorkspaceId,
|
||||
workspaceCwd: workspaceRoot,
|
||||
}).catch(() => undefined);
|
||||
await resetRuntimeServicesForTests();
|
||||
await fs.rm(paperclipHome, { recursive: true, force: true });
|
||||
await fs.rm(workspaceRoot, { recursive: true, force: true });
|
||||
if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME;
|
||||
else process.env.PAPERCLIP_HOME = previousPaperclipHome;
|
||||
if (previousPaperclipInstanceId === undefined) delete process.env.PAPERCLIP_INSTANCE_ID;
|
||||
else process.env.PAPERCLIP_INSTANCE_ID = previousPaperclipInstanceId;
|
||||
if (previousHttpsMode === undefined) delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
|
||||
else process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = previousHttpsMode;
|
||||
}
|
||||
}, 40_000);
|
||||
|
||||
it("adopts a live auto-port shared service after runtime state is reset", async () => {
|
||||
const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-reconcile-"));
|
||||
const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-"));
|
||||
|
||||
+9
-1
@@ -818,12 +818,20 @@ export async function startServer(): Promise<StartedServer> {
|
||||
|
||||
void reconcilePersistedRuntimeServicesOnStartup(db as any)
|
||||
.then((result) => {
|
||||
if (result.reconciled > 0 || result.restarted > 0 || result.restartFailed > 0) {
|
||||
if (
|
||||
result.reconciled > 0
|
||||
|| result.restarted > 0
|
||||
|| result.restartFailed > 0
|
||||
|| result.backfilled > 0
|
||||
) {
|
||||
logger.warn(
|
||||
{
|
||||
reconciled: result.reconciled,
|
||||
adopted: result.adopted,
|
||||
stopped: result.stopped,
|
||||
// Managed HTTP-only services taken down so they come back on a
|
||||
// verified HTTPS origin (PAP-17158).
|
||||
httpsBackfilled: result.backfilled,
|
||||
restarted: result.restarted,
|
||||
restartFailed: result.restartFailed,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* Backfill decision coverage for PAP-17158.
|
||||
*
|
||||
* `decideManagedRuntimeExposureBackfill` is the whole contract for which
|
||||
* pre-feature workspaces get upgraded to HTTPS in place and which are left
|
||||
* alone, so each branch is asserted by its reason code rather than only by the
|
||||
* resulting action.
|
||||
*/
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { decideManagedRuntimeExposureBackfill } from "./workspace-runtime.js";
|
||||
|
||||
/** A persisted, running, pre-feature Paperclip App dev runtime. */
|
||||
function httpOnlyRunningRow(overrides: Partial<Parameters<typeof decideManagedRuntimeExposureBackfill>[0]> = {}) {
|
||||
return {
|
||||
mode: "auto" as const,
|
||||
brokerAvailable: true,
|
||||
provider: "local_process",
|
||||
serviceName: "paperclip-dev",
|
||||
command: "pnpm dev --bind lan",
|
||||
status: "running",
|
||||
hasExposure: false,
|
||||
declaredIntent: "unset" as const,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("decideManagedRuntimeExposureBackfill", () => {
|
||||
it("reprovisions a running HTTP-only managed worktree runtime", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow())).toEqual({
|
||||
action: "reprovision",
|
||||
reason: "http_only_managed_service",
|
||||
});
|
||||
});
|
||||
|
||||
it("reprovisions a service that is mid-start as well as one already running", () => {
|
||||
for (const status of ["provisioning", "starting", "running"]) {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ status })).action).toBe("reprovision");
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves a stopped service to pick the default up on its next start", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ status: "stopped" }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "stopped_defaults_on_next_start",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a deliberate opt-out", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: "disabled" }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "deliberate_opt_out",
|
||||
});
|
||||
});
|
||||
|
||||
it("is idempotent: a row that already carries exposure state is never re-driven", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ hasExposure: true }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "already_exposed",
|
||||
});
|
||||
// Repeated deploys see the same row and must keep converging on "keep".
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ hasExposure: true })).action).toBe("keep");
|
||||
});
|
||||
|
||||
it("leaves unmanaged and custom services alone", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({
|
||||
serviceName: "preview",
|
||||
command: "pnpm vite",
|
||||
}))).toEqual({ action: "keep", reason: "unmanaged_or_custom_service" });
|
||||
});
|
||||
|
||||
it("leaves external (non local_process) runtime services alone", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ provider: "external" }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "not_a_managed_local_process",
|
||||
});
|
||||
});
|
||||
|
||||
it("does nothing when the automatic default is switched off", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ mode: "off" }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "https_default_disabled",
|
||||
});
|
||||
});
|
||||
|
||||
it("skips the backfill when the host broker is unavailable, unless forced", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ brokerAvailable: false }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "broker_unavailable",
|
||||
});
|
||||
expect(
|
||||
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ brokerAvailable: false, mode: "force" })).action,
|
||||
).toBe("reprovision");
|
||||
});
|
||||
|
||||
it("never takes down a row it cannot restart from a configured entry", () => {
|
||||
expect(decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: null }))).toEqual({
|
||||
action: "keep",
|
||||
reason: "no_configured_service_entry",
|
||||
});
|
||||
});
|
||||
|
||||
it("reprovisions an explicit opt-in that is somehow running without exposure", () => {
|
||||
expect(
|
||||
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({ declaredIntent: "enabled" })).action,
|
||||
).toBe("reprovision");
|
||||
});
|
||||
|
||||
it("checks the opt-out before broker availability so an opt-out never depends on host state", () => {
|
||||
expect(
|
||||
decideManagedRuntimeExposureBackfill(httpOnlyRunningRow({
|
||||
declaredIntent: "disabled",
|
||||
brokerAvailable: false,
|
||||
})).reason,
|
||||
).toBe("deliberate_opt_out");
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import type { BrokerClient, BrokerListenerRequest } from "./runtime-exposure/broker-client.js";
|
||||
import {
|
||||
@@ -11,7 +11,18 @@ import {
|
||||
const EXECUTION_WORKSPACE_ID = "11111111-2222-4333-8444-555566667777";
|
||||
const HANDLE = "handle-abcdef1234567890";
|
||||
|
||||
// The shared test setup pins the automatic default off so unrelated suites do
|
||||
// not probe for a real host broker. This suite is about the default, so it opts
|
||||
// back in and restores the harness value afterwards.
|
||||
let previousHttpsMode: string | undefined;
|
||||
beforeEach(() => {
|
||||
previousHttpsMode = process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
|
||||
process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = "auto";
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (previousHttpsMode === undefined) delete process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS;
|
||||
else process.env.PAPERCLIP_MANAGED_RUNTIME_HTTPS = previousHttpsMode;
|
||||
await resetRuntimeServicesForTests();
|
||||
});
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
readRuntimeExposureIntent,
|
||||
resolveDeclaredRuntimeExposureConfig,
|
||||
type RuntimeExposureConfigInput,
|
||||
type RuntimeExposureIntent,
|
||||
type RuntimeExposureStatus,
|
||||
type GitWorktreeBranchAncestryVerdict,
|
||||
type GitWorktreeBranchIncoherenceEvidence as SharedGitWorktreeBranchIncoherenceEvidence,
|
||||
@@ -6272,6 +6273,96 @@ export async function listWorkspaceRuntimeServicesForProjectWorkspaces(
|
||||
return grouped;
|
||||
}
|
||||
|
||||
/**
|
||||
* Statuses that mean "there is, or is supposed to be, a live backend process".
|
||||
* A row in one of these states is what the backfill has to reprovision; a
|
||||
* `stopped` row simply picks the default up on its next start.
|
||||
*/
|
||||
const LIVE_RUNTIME_SERVICE_STATUSES = new Set(["provisioning", "starting", "running"]);
|
||||
|
||||
export type ManagedRuntimeExposureBackfillDecision = {
|
||||
action: "keep" | "reprovision";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Decide what the HTTPS backfill should do with one persisted runtime-service
|
||||
* row (PAP-17158).
|
||||
*
|
||||
* Pure so every branch is directly testable: the reasons below are the whole
|
||||
* contract for which pre-feature workspaces get upgraded and which are left
|
||||
* exactly as they are.
|
||||
*
|
||||
* `declaredIntent === null` means no configured service entry could be matched
|
||||
* to this row. Such a row is deliberately left alone: reprovisioning works by
|
||||
* stopping the HTTP backend and letting the desired-state restart bring it back
|
||||
* with exposure, so without a config to restart from we would take a service
|
||||
* down and never bring it back.
|
||||
*/
|
||||
export function decideManagedRuntimeExposureBackfill(input: {
|
||||
mode: ManagedRuntimeHttpsMode;
|
||||
brokerAvailable: boolean;
|
||||
provider: string;
|
||||
serviceName: string;
|
||||
command: string | null;
|
||||
status: string;
|
||||
hasExposure: boolean;
|
||||
declaredIntent: RuntimeExposureIntent | null;
|
||||
}): ManagedRuntimeExposureBackfillDecision {
|
||||
if (input.mode === "off") return { action: "keep", reason: "https_default_disabled" };
|
||||
if (input.provider !== "local_process") return { action: "keep", reason: "not_a_managed_local_process" };
|
||||
// Idempotence: a row that already carries exposure state is never re-driven,
|
||||
// so repeated deploys and restarts converge instead of churning listeners.
|
||||
if (input.hasExposure) return { action: "keep", reason: "already_exposed" };
|
||||
if (input.declaredIntent === "disabled") return { action: "keep", reason: "deliberate_opt_out" };
|
||||
if (input.declaredIntent === null) return { action: "keep", reason: "no_configured_service_entry" };
|
||||
if (!isManagedHttpsDefaultCandidate({ serviceName: input.serviceName, command: input.command })) {
|
||||
return { action: "keep", reason: "unmanaged_or_custom_service" };
|
||||
}
|
||||
if (input.mode !== "force" && !input.brokerAvailable) {
|
||||
return { action: "keep", reason: "broker_unavailable" };
|
||||
}
|
||||
if (!LIVE_RUNTIME_SERVICE_STATUSES.has(input.status)) {
|
||||
return { action: "keep", reason: "stopped_defaults_on_next_start" };
|
||||
}
|
||||
return { action: "reprovision", reason: "http_only_managed_service" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up the exposure intent a persisted runtime-service row inherits from its
|
||||
* owning workspace configuration, by matching the row's service name against the
|
||||
* configured entries. Returns null when no entry matches.
|
||||
*/
|
||||
async function buildPersistedRuntimeExposureIntentLookup(db: Db) {
|
||||
const [projectWorkspaceRows, executionWorkspaceRows] = await Promise.all([
|
||||
db.select().from(projectWorkspaces),
|
||||
db.select().from(executionWorkspaces),
|
||||
]);
|
||||
const projectRuntimeById = new Map(projectWorkspaceRows.map((row) => [
|
||||
row.id,
|
||||
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime ?? null,
|
||||
] as const));
|
||||
const executionRuntimeById = new Map(executionWorkspaceRows.map((row) => [
|
||||
row.id,
|
||||
readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime
|
||||
?? (row.projectWorkspaceId ? projectRuntimeById.get(row.projectWorkspaceId) ?? null : null),
|
||||
] as const));
|
||||
|
||||
return (row: {
|
||||
serviceName: string;
|
||||
projectWorkspaceId: string | null;
|
||||
executionWorkspaceId: string | null;
|
||||
}): RuntimeExposureIntent | null => {
|
||||
const runtime = (row.executionWorkspaceId ? executionRuntimeById.get(row.executionWorkspaceId) : null)
|
||||
?? (row.projectWorkspaceId ? projectRuntimeById.get(row.projectWorkspaceId) ?? null : null);
|
||||
if (!runtime) return null;
|
||||
const entries = listConfiguredRuntimeServiceEntries({ workspaceRuntime: runtime });
|
||||
const entry = entries.find((candidate) => asString(candidate.name, "service") === row.serviceName);
|
||||
if (!entry) return null;
|
||||
return readRuntimeExposureIntent(parseObject(entry.expose));
|
||||
};
|
||||
}
|
||||
|
||||
export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
const rows = await db
|
||||
.select()
|
||||
@@ -6283,6 +6374,13 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
),
|
||||
);
|
||||
|
||||
// Backfill inputs, resolved once per startup rather than per row.
|
||||
const httpsMode = resolveManagedRuntimeHttpsMode();
|
||||
const brokerAvailable = httpsMode === "off"
|
||||
? false
|
||||
: await workspaceRuntimeExposureDeps.isBrokerAvailable().catch(() => false);
|
||||
const readDeclaredExposureIntent = await buildPersistedRuntimeExposureIntentLookup(db);
|
||||
|
||||
let ownedExposureListeners: Awaited<ReturnType<BrokerClient["list"]>> | null = [];
|
||||
if (rows.some((row) => row.exposure && row.exposure.state !== "removed")) {
|
||||
try {
|
||||
@@ -6295,6 +6393,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
let reconciled = 0;
|
||||
let adopted = 0;
|
||||
let stopped = 0;
|
||||
let backfilled = 0;
|
||||
for (const row of rows) {
|
||||
if (row.status === "stopped" && row.exposure && row.exposure.state !== "removed") {
|
||||
await cleanupPersistedExposureRows(db, [{
|
||||
@@ -6321,6 +6420,25 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
new URL("/api/health", row.exposure.publicUrl!).toString(),
|
||||
)
|
||||
);
|
||||
// Pre-feature rows carry no exposure state at all. An eligible one that is
|
||||
// still serving plain HTTP must not be adopted as-is, or the deploy would
|
||||
// leave `http://paperclip-dev:<port>` as the canonical URL forever. Stopping
|
||||
// it here hands it to the desired-state restart below, which brings it back
|
||||
// through the normal fail-closed exposure lifecycle.
|
||||
const backfillDecision = decideManagedRuntimeExposureBackfill({
|
||||
mode: httpsMode,
|
||||
brokerAvailable,
|
||||
provider: row.provider,
|
||||
serviceName: row.serviceName,
|
||||
command: row.command,
|
||||
status: row.status,
|
||||
hasExposure: Boolean(row.exposure && row.exposure.state !== "removed"),
|
||||
declaredIntent: readDeclaredExposureIntent({
|
||||
serviceName: row.serviceName,
|
||||
projectWorkspaceId: row.projectWorkspaceId ?? null,
|
||||
executionWorkspaceId: row.executionWorkspaceId ?? null,
|
||||
}),
|
||||
});
|
||||
let adoptedRecord = await findLocalServiceRegistryRecordByRuntimeServiceId({
|
||||
runtimeServiceId: row.id,
|
||||
profileKind: "workspace-runtime",
|
||||
@@ -6366,9 +6484,11 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
if (adoptedRecord) {
|
||||
const adoptedUrl = adoptedRecord.url ?? row.backendUrl ?? row.url ?? null;
|
||||
if (
|
||||
!exposureHealthMatches
|
||||
backfillDecision.action === "reprovision"
|
||||
|| !exposureHealthMatches
|
||||
|| !(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))
|
||||
) {
|
||||
if (backfillDecision.action === "reprovision") backfilled += 1;
|
||||
await terminateLocalService(adoptedRecord);
|
||||
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
|
||||
} else {
|
||||
@@ -6452,7 +6572,10 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
.set({
|
||||
status: "stopped",
|
||||
healthStatus: "unknown",
|
||||
url: stoppedExposure ? null : row.url,
|
||||
// A row queued for HTTPS backfill drops its HTTP URL now rather than
|
||||
// keeping it until the restart succeeds: if the restart fails, the
|
||||
// fail-closed contract says show no URL, not a working HTTP one.
|
||||
url: stoppedExposure || backfillDecision.action === "reprovision" ? null : row.url,
|
||||
exposure: stoppedExposure,
|
||||
exposureHandle: stoppedExposureHandle,
|
||||
stoppedAt: now,
|
||||
@@ -6476,12 +6599,18 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
// was never committed. Re-applying persisted desired state adopts that
|
||||
// registry entry (or restarts a missing service) and makes it visible to
|
||||
// workspace cleanup through workspace_runtime_services again.
|
||||
//
|
||||
// It is also the second half of the HTTPS backfill: services stopped above as
|
||||
// HTTP-only come back here through the ordinary start path, which now applies
|
||||
// the `tailscale_https` default and only reports them healthy behind a
|
||||
// verified HTTPS URL.
|
||||
const desiredState = await restartDesiredRuntimeServicesOnStartup(db);
|
||||
|
||||
return {
|
||||
reconciled,
|
||||
adopted,
|
||||
stopped,
|
||||
backfilled,
|
||||
restarted: desiredState.restarted,
|
||||
restartFailed: desiredState.failed,
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user