mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-28 20:21:50 +01:00
feat(runtime): wire automatic Tailscale HTTPS lifecycle (PAP-17051)
Persist private lease state, reserve app/HMR pairs before spawn, bind Paperclip previews to loopback, publish only after validating HTTPS, and reconcile or clean up every lifecycle path. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
1696e17c67
commit
831a2097ec
11 files changed
+677
-90
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveViteHmrHost, resolveViteHmrPort } from "../app.ts";
|
||||
import { resolveViteHmrHost, resolveViteHmrPort, resolveViteHmrProtocol } from "../app.ts";
|
||||
|
||||
describe("resolveViteHmrPort", () => {
|
||||
it("uses serverPort + 10000 when the result stays in range", () => {
|
||||
@@ -24,8 +24,17 @@ describe("resolveViteHmrHost", () => {
|
||||
expect(resolveViteHmrHost("::")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps concrete bind hosts", () => {
|
||||
expect(resolveViteHmrHost("127.0.0.1")).toBe("127.0.0.1");
|
||||
it("uses the browser hostname for loopback while keeping non-loopback concrete hosts", () => {
|
||||
expect(resolveViteHmrHost("127.0.0.1")).toBeUndefined();
|
||||
expect(resolveViteHmrHost("localhost")).toBeUndefined();
|
||||
expect(resolveViteHmrHost("paperclip-dev")).toBe("paperclip-dev");
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveViteHmrProtocol", () => {
|
||||
it("selects secure WebSockets for HTTPS branch runtimes", () => {
|
||||
expect(resolveViteHmrProtocol("wss")).toBe("wss");
|
||||
expect(resolveViteHmrProtocol(undefined)).toBeUndefined();
|
||||
expect(() => resolveViteHmrProtocol("https")).toThrow(/ws or wss/);
|
||||
});
|
||||
});
|
||||
@@ -90,13 +90,14 @@ describe("boardMutationGuard", () => {
|
||||
expect([200, 204]).toContain(res.status);
|
||||
});
|
||||
|
||||
it("allows board mutations when x-forwarded-host matches origin", async () => {
|
||||
it("allows HTTPS branch-runtime mutations when forwarded MagicDNS host and non-standard port match", async () => {
|
||||
const app = createApp("board");
|
||||
const res = await request(app)
|
||||
.post("/mutate")
|
||||
.set("Host", "127.0.0.1")
|
||||
.set("X-Forwarded-Host", "10.90.10.20:3443")
|
||||
.set("Origin", "https://10.90.10.20:3443")
|
||||
.set("X-Forwarded-Host", "branch-runner.tail123.ts.net:42000")
|
||||
.set("X-Forwarded-Proto", "https")
|
||||
.set("Origin", "https://branch-runner.tail123.ts.net:42000")
|
||||
.send({ ok: true });
|
||||
expect([200, 204]).toContain(res.status);
|
||||
});
|
||||
|
||||
+21
-6
@@ -3,7 +3,7 @@ import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
|
||||
import { derivePaperclipViteHmrPort, type DeploymentExposure, type DeploymentMode } from "@paperclipai/shared";
|
||||
import type { InspectDatabaseBackupHealthOptions } from "./services/database-backup-health.js";
|
||||
import type { StorageService } from "./storage/types.js";
|
||||
import { httpLogger, errorHandler } from "./middleware/index.js";
|
||||
@@ -131,18 +131,27 @@ export function isDatabaseConnectionUnavailableError(err: unknown): boolean {
|
||||
}
|
||||
|
||||
export function resolveViteHmrPort(serverPort: number): number {
|
||||
if (serverPort <= 55_535) {
|
||||
return serverPort + 10_000;
|
||||
}
|
||||
return Math.max(1_024, serverPort - 10_000);
|
||||
return derivePaperclipViteHmrPort(serverPort);
|
||||
}
|
||||
|
||||
export function resolveViteHmrHost(bindHost: string): string | undefined {
|
||||
const normalized = bindHost.trim().toLowerCase();
|
||||
if (normalized === "0.0.0.0" || normalized === "::") return undefined;
|
||||
if (
|
||||
normalized === "0.0.0.0"
|
||||
|| normalized === "::"
|
||||
|| normalized === "127.0.0.1"
|
||||
|| normalized === "::1"
|
||||
|| normalized === "localhost"
|
||||
) return undefined;
|
||||
return bindHost;
|
||||
}
|
||||
|
||||
export function resolveViteHmrProtocol(value: string | undefined): "ws" | "wss" | undefined {
|
||||
if (!value) return undefined;
|
||||
if (value === "ws" || value === "wss") return value;
|
||||
throw new Error("PAPERCLIP_VITE_HMR_PROTOCOL must be ws or wss");
|
||||
}
|
||||
|
||||
export function shouldServeViteDevHtml(req: ExpressRequest): boolean {
|
||||
const pathname = req.path;
|
||||
if (VITE_DEV_STATIC_PATHS.has(pathname)) return false;
|
||||
@@ -642,14 +651,20 @@ export async function createApp(
|
||||
const publicUiRoot = path.resolve(uiRoot, "public");
|
||||
const hmrPort = resolveViteHmrPort(opts.serverPort);
|
||||
const hmrHost = resolveViteHmrHost(opts.bindHost);
|
||||
const hmrProtocol = resolveViteHmrProtocol(process.env.PAPERCLIP_VITE_HMR_PROTOCOL);
|
||||
const { createServer: createViteServer } = await import("vite");
|
||||
const vite = await createViteServer({
|
||||
root: uiRoot,
|
||||
appType: "custom",
|
||||
server: {
|
||||
// Listener binding and browser HMR hostname are deliberately separate:
|
||||
// exposed branch runtimes stay loopback-only while the browser uses the
|
||||
// current MagicDNS hostname through the broker's HTTPS listener.
|
||||
host: opts.bindHost,
|
||||
middlewareMode: true,
|
||||
hmr: {
|
||||
...(hmrHost ? { host: hmrHost } : {}),
|
||||
...(hmrProtocol ? { protocol: hmrProtocol } : {}),
|
||||
port: hmrPort,
|
||||
clientPort: hmrPort,
|
||||
},
|
||||
|
||||
@@ -5543,6 +5543,7 @@ export function issueRoutes(
|
||||
startedAt: service.startedAt,
|
||||
stoppedAt: service.stoppedAt,
|
||||
healthStatus: service.healthStatus,
|
||||
exposure: service.exposure ?? null,
|
||||
configIndex: service.configIndex ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import type { BrokerClient, BrokerListenerRequest } from "./runtime-exposure/broker-client.js";
|
||||
import {
|
||||
resetRuntimeServicesForTests,
|
||||
setWorkspaceRuntimeExposureDepsForTests,
|
||||
startRuntimeServicesForWorkspaceControl,
|
||||
stopRuntimeServicesForExecutionWorkspace,
|
||||
} from "./workspace-runtime.js";
|
||||
|
||||
const EXECUTION_WORKSPACE_ID = "11111111-2222-4333-8444-555566667777";
|
||||
const HANDLE = "handle-abcdef1234567890";
|
||||
|
||||
afterEach(async () => {
|
||||
await resetRuntimeServicesForTests();
|
||||
});
|
||||
|
||||
function serviceCommand() {
|
||||
return `node -e 'const http=require("http");const p=Number(process.env.PORT);for(const q of [p,p+10000])http.createServer((_,r)=>{r.statusCode=200;r.end("ok")}).listen(q,"127.0.0.1");setInterval(()=>{},1000)'`;
|
||||
}
|
||||
|
||||
function createBroker() {
|
||||
const calls: string[] = [];
|
||||
let listeners: BrokerListenerRequest[] = [];
|
||||
const broker: BrokerClient = {
|
||||
async reserve(_runtimeId, requested) {
|
||||
calls.push("reserve");
|
||||
listeners = requested;
|
||||
return { handle: HANDLE, reservedPorts: requested.map((listener) => listener.port) };
|
||||
},
|
||||
async expose() {
|
||||
calls.push("expose");
|
||||
return { handle: HANDLE, publicPorts: listeners.map((listener) => listener.port) };
|
||||
},
|
||||
async remove() {
|
||||
calls.push("remove");
|
||||
return { removedPorts: listeners.map((listener) => listener.port) };
|
||||
},
|
||||
async list() {
|
||||
return [];
|
||||
},
|
||||
};
|
||||
return { broker, calls };
|
||||
}
|
||||
|
||||
function startInput() {
|
||||
return {
|
||||
invocationId: "runtime-exposure-test",
|
||||
actor: { id: null, name: "Paperclip", companyId: "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" },
|
||||
issue: null,
|
||||
workspace: {
|
||||
baseCwd: process.cwd(),
|
||||
source: "project_primary" as const,
|
||||
projectId: null,
|
||||
workspaceId: null,
|
||||
repoUrl: null,
|
||||
repoRef: null,
|
||||
strategy: "project_primary" as const,
|
||||
cwd: process.cwd(),
|
||||
branchName: "test",
|
||||
worktreePath: null,
|
||||
warnings: [],
|
||||
created: false,
|
||||
},
|
||||
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
|
||||
config: {
|
||||
workspaceRuntime: {
|
||||
services: [{
|
||||
name: "preview",
|
||||
command: serviceCommand(),
|
||||
port: { type: "auto", envKey: "PORT" },
|
||||
readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 5 },
|
||||
expose: {
|
||||
type: "tailscale_https",
|
||||
hostname: "auto",
|
||||
publicPort: "same",
|
||||
includePaperclipViteHmr: true,
|
||||
failurePolicy: "fail_closed",
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
adapterEnv: {},
|
||||
};
|
||||
}
|
||||
|
||||
describe("workspace runtime tailscale_https lifecycle", () => {
|
||||
it("reserves before spawn, exposes after backend readiness, and removes on stop", async () => {
|
||||
const { broker, calls } = createBroker();
|
||||
setWorkspaceRuntimeExposureDepsForTests({
|
||||
broker,
|
||||
isPortAvailable: async () => true,
|
||||
resolveHostname: async () => "runner.tail123.ts.net",
|
||||
probeHealth: async () => true,
|
||||
now: () => "2026-08-11T00:00:00.000Z",
|
||||
});
|
||||
|
||||
const [runtime] = await startRuntimeServicesForWorkspaceControl(startInput());
|
||||
expect(calls.slice(0, 2)).toEqual(["reserve", "expose"]);
|
||||
expect(runtime.port).toBeGreaterThanOrEqual(42000);
|
||||
expect(runtime.url).toBe(`https://runner.tail123.ts.net:${runtime.port}`);
|
||||
expect(runtime.exposure?.state).toBe("ready");
|
||||
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
executionWorkspaceId: EXECUTION_WORKSPACE_ID,
|
||||
runtimeServiceId: runtime.id,
|
||||
});
|
||||
expect(calls).toEqual(["reserve", "expose", "remove"]);
|
||||
}, 15_000);
|
||||
|
||||
it("fails closed and removes the mapping when external HTTPS validation fails", async () => {
|
||||
const { broker, calls } = createBroker();
|
||||
setWorkspaceRuntimeExposureDepsForTests({
|
||||
broker,
|
||||
isPortAvailable: async () => true,
|
||||
resolveHostname: async () => "runner.tail123.ts.net",
|
||||
probeHealth: async () => false,
|
||||
now: () => "2026-08-11T00:00:00.000Z",
|
||||
});
|
||||
|
||||
await expect(startRuntimeServicesForWorkspaceControl(startInput())).rejects.toThrow(/HTTPS exposure failed/);
|
||||
expect(calls).toEqual(["reserve", "expose", "remove"]);
|
||||
}, 15_000);
|
||||
});
|
||||
@@ -36,6 +36,8 @@ function runtimeServiceRow(
|
||||
stopPolicy: null,
|
||||
healthStatus: "unknown",
|
||||
exposure: null,
|
||||
exposureHandle: null,
|
||||
backendUrl: null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
...overrides,
|
||||
|
||||
@@ -56,6 +56,7 @@ function runtimeService(
|
||||
stoppedAt: null,
|
||||
stopPolicy: null,
|
||||
healthStatus: "healthy",
|
||||
exposure: null,
|
||||
reused: false,
|
||||
...overrides,
|
||||
};
|
||||
|
||||
@@ -11,6 +11,9 @@ import type { Db } from "@paperclipai/db";
|
||||
import { executionWorkspaces, issueComments, issues, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db";
|
||||
import {
|
||||
listWorkspaceServiceCommandDefinitions,
|
||||
parseRuntimeExposureConfig,
|
||||
type RuntimeExposureConfigInput,
|
||||
type RuntimeExposureStatus,
|
||||
type GitWorktreeBranchAncestryVerdict,
|
||||
type GitWorktreeBranchIncoherenceEvidence as SharedGitWorktreeBranchIncoherenceEvidence,
|
||||
type GitWorktreeInProgressOperation,
|
||||
@@ -48,6 +51,15 @@ import {
|
||||
WORKTREE_INSTANCE_ROOT_METADATA_KEY,
|
||||
type WorktreeInstancePointer,
|
||||
} from "./workspace-instance-cleanup.js";
|
||||
import { UnixBrokerClient, type BrokerClient } from "./runtime-exposure/broker-client.js";
|
||||
import {
|
||||
deprovisionExposure,
|
||||
provisionExposure,
|
||||
reserveExposure,
|
||||
type ExposureManagerDeps,
|
||||
} from "./runtime-exposure/exposure-manager.js";
|
||||
import { allocateExposurePortPair } from "./runtime-exposure/port-pair.js";
|
||||
import { resolveTailscaleDnsName } from "./runtime-exposure/tailscale-hostname.js";
|
||||
|
||||
export function resolveShell(): string {
|
||||
const fallback = process.platform === "win32" ? "sh" : "/bin/sh";
|
||||
@@ -156,6 +168,7 @@ export interface RuntimeServiceRef {
|
||||
stoppedAt: string | null;
|
||||
stopPolicy: Record<string, unknown> | null;
|
||||
healthStatus: "unknown" | "healthy" | "unhealthy";
|
||||
exposure: RuntimeExposureStatus | null;
|
||||
reused: boolean;
|
||||
}
|
||||
|
||||
@@ -168,6 +181,11 @@ interface RuntimeServiceRecord extends RuntimeServiceRef {
|
||||
serviceKey: string;
|
||||
profileKind: string;
|
||||
processGroupId: number | null;
|
||||
/** Server-private broker lease handle; never returned by toRuntimeServiceRef. */
|
||||
exposureHandle: string | null;
|
||||
/** Loopback URL used for backend readiness/adoption; never serialized. */
|
||||
backendUrl: string | null;
|
||||
exposureConfig: RuntimeExposureConfigInput | null;
|
||||
}
|
||||
|
||||
type LocalRuntimeServiceStart = {
|
||||
@@ -188,9 +206,11 @@ const runtimeServicesById = new Map<string, RuntimeServiceRecord>();
|
||||
const runtimeServicesByReuseKey = new Map<string, string>();
|
||||
const runtimeServiceLeasesByRun = new Map<string, string[]>();
|
||||
const runtimeProvisionByWorkspace = new Map<string, Promise<void>>();
|
||||
const quarantinedRuntimeExposurePorts = new Set<number>();
|
||||
const DEFAULT_EXECUTE_PROCESS_OUTPUT_BYTES = 256 * 1024;
|
||||
export const WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS = 32;
|
||||
const ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES = ["provisioning", "starting", "running"] as const;
|
||||
const DEFAULT_TAILSCALE_BROKER_SOCKET = "/run/paperclip-tailscale-broker/broker.sock";
|
||||
|
||||
class RuntimeServicePortBindCollision extends Error {
|
||||
readonly port: number;
|
||||
@@ -202,6 +222,58 @@ class RuntimeServicePortBindCollision extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export type WorkspaceRuntimeExposureDeps = ExposureManagerDeps & {
|
||||
resolveHostname: () => Promise<string>;
|
||||
isPortAvailable: (port: number) => Promise<boolean>;
|
||||
};
|
||||
|
||||
async function isLoopbackPortAvailable(port: number): Promise<boolean> {
|
||||
return await new Promise<boolean>((resolve) => {
|
||||
const server = net.createServer();
|
||||
server.unref();
|
||||
server.once("error", () => resolve(false));
|
||||
server.listen(port, "127.0.0.1", () => {
|
||||
server.close(() => resolve(true));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function defaultWorkspaceRuntimeExposureDeps(): WorkspaceRuntimeExposureDeps {
|
||||
const broker: BrokerClient = new UnixBrokerClient({
|
||||
socketPath: process.env.PAPERCLIP_TAILSCALE_BROKER_SOCKET?.trim() || DEFAULT_TAILSCALE_BROKER_SOCKET,
|
||||
});
|
||||
return {
|
||||
broker,
|
||||
resolveHostname: () => resolveTailscaleDnsName(),
|
||||
isPortAvailable: isLoopbackPortAvailable,
|
||||
probeHealth: async (url) => {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(5_000),
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
now: () => new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
let workspaceRuntimeExposureDeps = defaultWorkspaceRuntimeExposureDeps();
|
||||
|
||||
/** Test-only seam; resetRuntimeServicesForTests restores production defaults. */
|
||||
export function setWorkspaceRuntimeExposureDepsForTests(deps: WorkspaceRuntimeExposureDeps) {
|
||||
workspaceRuntimeExposureDeps = deps;
|
||||
}
|
||||
|
||||
function configuredRuntimeExposure(service: Record<string, unknown>): RuntimeExposureConfigInput | null {
|
||||
const expose = parseObject(service.expose);
|
||||
if (asString(expose.type, "") !== "tailscale_https") return null;
|
||||
return parseRuntimeExposureConfig(expose);
|
||||
}
|
||||
|
||||
type ProcessOutputCapture = {
|
||||
text: string;
|
||||
truncated: boolean;
|
||||
@@ -221,6 +293,8 @@ export async function resetRuntimeServicesForTests() {
|
||||
runtimeServicesByReuseKey.clear();
|
||||
runtimeServiceLeasesByRun.clear();
|
||||
runtimeProvisionByWorkspace.clear();
|
||||
quarantinedRuntimeExposurePorts.clear();
|
||||
workspaceRuntimeExposureDeps = defaultWorkspaceRuntimeExposureDeps();
|
||||
}
|
||||
|
||||
function stableStringify(value: unknown): string {
|
||||
@@ -434,6 +508,7 @@ function toRuntimeServiceRef(record: RuntimeServiceRecord, overrides?: Partial<R
|
||||
startedAt: record.startedAt,
|
||||
stoppedAt: record.stoppedAt,
|
||||
stopPolicy: record.stopPolicy,
|
||||
exposure: record.exposure,
|
||||
healthStatus: record.healthStatus,
|
||||
reused: record.reused,
|
||||
...overrides,
|
||||
@@ -3734,6 +3809,53 @@ async function allocateIsolatedWorkspacePort(input: {
|
||||
});
|
||||
}
|
||||
|
||||
async function collectReservedExposurePorts(db: Db | undefined, companyId: string): Promise<Set<number>> {
|
||||
const reserved = new Set(quarantinedRuntimeExposurePorts);
|
||||
for (const record of runtimeServicesById.values()) {
|
||||
if (record.companyId !== companyId || !record.exposure) continue;
|
||||
for (const listener of record.exposure.listeners) reserved.add(listener.targetPort);
|
||||
}
|
||||
if (!db) return reserved;
|
||||
const rows = await db
|
||||
.select({ exposure: workspaceRuntimeServices.exposure })
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(eq(workspaceRuntimeServices.companyId, companyId));
|
||||
for (const row of rows) {
|
||||
if (!row.exposure || row.exposure.state === "removed") continue;
|
||||
for (const listener of row.exposure.listeners) reserved.add(listener.targetPort);
|
||||
}
|
||||
return reserved;
|
||||
}
|
||||
|
||||
async function allocateAndReserveExposure(input: {
|
||||
db?: Db;
|
||||
companyId: string;
|
||||
runtimeId: string;
|
||||
config: RuntimeExposureConfigInput;
|
||||
}): Promise<{ appPort: number; status: RuntimeExposureStatus; handle: string }> {
|
||||
const reserved = await collectReservedExposurePorts(input.db, input.companyId);
|
||||
const retryable = new Set(["reservation_conflict", "manual_mapping_present", "quarantined"]);
|
||||
while (true) {
|
||||
const pair = await allocateExposurePortPair({
|
||||
isPortAvailable: workspaceRuntimeExposureDeps.isPortAvailable,
|
||||
reserved,
|
||||
});
|
||||
const result = await reserveExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: input.runtimeId,
|
||||
config: input.config,
|
||||
appPort: pair.appPort,
|
||||
});
|
||||
if (result.handle) {
|
||||
return { appPort: pair.appPort, status: result.status, handle: result.handle };
|
||||
}
|
||||
if (!result.status.lastError || !retryable.has(result.status.lastError)) {
|
||||
throw new Error(`HTTPS exposure reservation failed: ${result.status.lastError ?? "unknown broker error"}`);
|
||||
}
|
||||
reserved.add(pair.appPort);
|
||||
reserved.add(pair.hmrPort);
|
||||
}
|
||||
}
|
||||
|
||||
function buildTemplateData(input: {
|
||||
workspace: RealizedExecutionWorkspace;
|
||||
agent: ExecutionWorkspaceAgentRef;
|
||||
@@ -3827,6 +3949,7 @@ function resolveRuntimeServiceReuseIdentity(input: {
|
||||
cwd: serviceCwd,
|
||||
port: identityPort,
|
||||
env: renderedEnv,
|
||||
expose: input.service.expose ?? null,
|
||||
}),
|
||||
)
|
||||
.digest("hex")
|
||||
@@ -4115,6 +4238,9 @@ function toPersistedWorkspaceRuntimeService(record: RuntimeServiceRecord): typeo
|
||||
startedAt: new Date(record.startedAt),
|
||||
stoppedAt: record.stoppedAt ? new Date(record.stoppedAt) : null,
|
||||
stopPolicy: record.stopPolicy,
|
||||
exposure: record.exposure,
|
||||
exposureHandle: record.exposureHandle,
|
||||
backendUrl: record.backendUrl,
|
||||
healthStatus: record.healthStatus,
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
@@ -4151,6 +4277,9 @@ async function persistRuntimeServiceRecord(db: Db | undefined, record: RuntimeSe
|
||||
startedAt: values.startedAt,
|
||||
stoppedAt: values.stoppedAt,
|
||||
stopPolicy: values.stopPolicy,
|
||||
exposure: values.exposure,
|
||||
exposureHandle: values.exposureHandle,
|
||||
backendUrl: values.backendUrl,
|
||||
healthStatus: values.healthStatus,
|
||||
updatedAt: values.updatedAt,
|
||||
},
|
||||
@@ -4287,6 +4416,7 @@ export function normalizeAdapterManagedRuntimeServices(input: {
|
||||
stoppedAt: status === "running" || status === "starting" ? null : nowIso,
|
||||
stopPolicy: report.stopPolicy ?? null,
|
||||
healthStatus,
|
||||
exposure: null,
|
||||
reused: false,
|
||||
};
|
||||
});
|
||||
@@ -4455,6 +4585,7 @@ function createProvisioningRuntimeServiceRecord(
|
||||
stoppedAt: null,
|
||||
stopPolicy: parseObject(input.service.stopPolicy),
|
||||
healthStatus: "unknown",
|
||||
exposure: null,
|
||||
reused: false,
|
||||
db: input.db,
|
||||
child: null,
|
||||
@@ -4464,6 +4595,9 @@ function createProvisioningRuntimeServiceRecord(
|
||||
serviceKey: `runtime-provision:${runtimeProvisionWorkspaceKey(input)}:${id}`,
|
||||
profileKind: "workspace-runtime",
|
||||
processGroupId: null,
|
||||
exposureHandle: null,
|
||||
backendUrl: null,
|
||||
exposureConfig: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -4490,8 +4624,13 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
const explicitPort = identity.explicitPort;
|
||||
const identityPort = identity.identityPort;
|
||||
const portType = asString(portConfig.type, "");
|
||||
const exposureConfig = configuredRuntimeExposure(input.service);
|
||||
if (exposureConfig && portType !== "auto") {
|
||||
throw new Error(`Runtime service "${serviceName}" must use port.type=auto with tailscale_https exposure`);
|
||||
}
|
||||
const canAllocateFixedPort = Boolean(
|
||||
input.allowFixedPortFallback
|
||||
!exposureConfig
|
||||
&& input.allowFixedPortFallback
|
||||
&& input.db
|
||||
&& input.executionWorkspaceId
|
||||
&& explicitPort > 0,
|
||||
@@ -4506,15 +4645,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
scopeType: input.scopeType,
|
||||
scopeId: input.scopeId,
|
||||
});
|
||||
let reusableStoppedPort: number | null = null;
|
||||
if (portType === "auto" && stoppedReuseCandidate?.port && !input.excludedPorts?.has(stoppedReuseCandidate.port)) {
|
||||
reusableStoppedPort = await canBindRuntimePort(stoppedReuseCandidate.port)
|
||||
? stoppedReuseCandidate.port
|
||||
: null;
|
||||
}
|
||||
let port: number | null = null;
|
||||
if (portType === "auto") {
|
||||
port = reusableStoppedPort;
|
||||
const runtimeId = input.runtimeServiceId ?? stoppedReuseCandidate?.id ?? randomUUID();
|
||||
const reservedExposure = exposureConfig
|
||||
? await allocateAndReserveExposure({
|
||||
db: input.db,
|
||||
companyId: input.agent.companyId,
|
||||
runtimeId,
|
||||
config: exposureConfig,
|
||||
})
|
||||
: null;
|
||||
let port: number | null = reservedExposure?.appPort ?? null;
|
||||
if (!reservedExposure && portType === "auto") {
|
||||
if (stoppedReuseCandidate?.port && !input.excludedPorts?.has(stoppedReuseCandidate.port)) {
|
||||
port = await canBindRuntimePort(stoppedReuseCandidate.port)
|
||||
? stoppedReuseCandidate.port
|
||||
: null;
|
||||
}
|
||||
for (let attempt = 0; port === null && attempt < WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS; attempt += 1) {
|
||||
const candidate = await allocatePort();
|
||||
if (!input.excludedPorts?.has(candidate)) port = candidate;
|
||||
@@ -4526,7 +4672,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
remediation: "Retry the start or configure a different runtime service port.",
|
||||
});
|
||||
}
|
||||
} else if (canAllocateFixedPort) {
|
||||
} else if (!reservedExposure && canAllocateFixedPort) {
|
||||
port = await allocateIsolatedWorkspacePort({
|
||||
db: input.db,
|
||||
companyId: input.agent.companyId,
|
||||
@@ -4535,9 +4681,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
stoppedPort: stoppedReuseCandidate?.port ?? null,
|
||||
excludedPorts: input.excludedPorts ?? new Set<number>(),
|
||||
});
|
||||
} else {
|
||||
} else if (!reservedExposure) {
|
||||
port = explicitPort > 0 ? explicitPort : null;
|
||||
}
|
||||
let exposureHostname: string | null = null;
|
||||
if (reservedExposure) {
|
||||
try {
|
||||
exposureHostname = await workspaceRuntimeExposureDeps.resolveHostname();
|
||||
reservedExposure.status.hostname = exposureHostname;
|
||||
reservedExposure.status.updatedAt = new Date().toISOString();
|
||||
} catch {
|
||||
await workspaceRuntimeExposureDeps.broker
|
||||
.remove(runtimeId, reservedExposure.handle)
|
||||
.catch(() => undefined);
|
||||
throw new Error("HTTPS exposure failed: Tailscale MagicDNS hostname unavailable");
|
||||
}
|
||||
}
|
||||
const templateData = buildTemplateData({
|
||||
workspace: input.workspace,
|
||||
agent: input.agent,
|
||||
@@ -4561,12 +4720,26 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
env[portEnvKey] = String(port);
|
||||
}
|
||||
|
||||
if (exposureConfig) {
|
||||
// Paperclip dev-runtime-specific hardening. Other managed processes are
|
||||
// still rejected by the broker unless /proc proves loopback-only listeners.
|
||||
env.PAPERCLIP_BIND = "custom";
|
||||
env.PAPERCLIP_BIND_HOST = "127.0.0.1";
|
||||
env.PAPERCLIP_VITE_HMR_PROTOCOL = "wss";
|
||||
env.PAPERCLIP_MANAGED_RUNTIME_EXPOSURE = "tailscale_https";
|
||||
env.PAPERCLIP_ALLOWED_HOSTNAMES = exposureHostname!;
|
||||
env.PAPERCLIP_AUTH_BASE_URL_MODE = "explicit";
|
||||
env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = `https://${exposureHostname}:${port}`;
|
||||
env.PAPERCLIP_PUBLIC_URL = `https://${exposureHostname}:${port}`;
|
||||
}
|
||||
|
||||
const expose = parseObject(input.service.expose);
|
||||
const readiness = parseObject(input.service.readiness);
|
||||
const urlTemplate =
|
||||
asString(expose.urlTemplate, "") ||
|
||||
asString(readiness.urlTemplate, "");
|
||||
const url = urlTemplate ? renderTemplate(urlTemplate, templateData) : null;
|
||||
const backendUrl = urlTemplate ? renderTemplate(urlTemplate, templateData) : null;
|
||||
let url = exposureConfig ? null : backendUrl;
|
||||
const readinessUrlTemplate = asString(readiness.urlTemplate, "");
|
||||
const readinessUrl = readinessUrlTemplate ? renderTemplate(readinessUrlTemplate, templateData) : null;
|
||||
const stopPolicy = parseObject(input.service.stopPolicy);
|
||||
@@ -4584,7 +4757,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
reuseKey: input.reuseKey,
|
||||
},
|
||||
});
|
||||
const adoptedRecord = await findAdoptableLocalService({
|
||||
const adoptedRecord = exposureConfig ? null : await findAdoptableLocalService({
|
||||
serviceKey,
|
||||
profileKind: "workspace-runtime",
|
||||
serviceName,
|
||||
@@ -4592,10 +4765,10 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
cwd: serviceCwd,
|
||||
envFingerprint: serviceIdentityFingerprint,
|
||||
port: port ?? identityPort,
|
||||
url,
|
||||
url: backendUrl,
|
||||
});
|
||||
if (adoptedRecord) {
|
||||
const adoptedUrl = adoptedRecord.url ?? url;
|
||||
const adoptedUrl = adoptedRecord.url ?? backendUrl;
|
||||
if (!(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName, command }))) {
|
||||
await terminateLocalService(adoptedRecord);
|
||||
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
|
||||
@@ -4627,6 +4800,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
stoppedAt: null,
|
||||
stopPolicy,
|
||||
healthStatus: "healthy",
|
||||
exposure: null,
|
||||
reused: true,
|
||||
db: input.db,
|
||||
child: null,
|
||||
@@ -4636,6 +4810,9 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
serviceKey,
|
||||
profileKind: "workspace-runtime",
|
||||
processGroupId: adoptedRecord.processGroupId ?? null,
|
||||
exposureHandle: null,
|
||||
backendUrl: adoptedUrl,
|
||||
exposureConfig: null,
|
||||
},
|
||||
readiness: Promise.resolve(),
|
||||
};
|
||||
@@ -4663,9 +4840,66 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
}
|
||||
}
|
||||
|
||||
await ensureServerWorkspaceLinksCurrent(serviceCwd, {
|
||||
onLog: input.onLog,
|
||||
});
|
||||
const nowIso = new Date().toISOString();
|
||||
const record: RuntimeServiceRecord = {
|
||||
id: runtimeId,
|
||||
companyId: input.agent.companyId,
|
||||
projectId: input.workspace.projectId,
|
||||
projectWorkspaceId: input.workspace.workspaceId,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
issueId: input.issue?.id ?? null,
|
||||
serviceName,
|
||||
status: "starting",
|
||||
lifecycle,
|
||||
scopeType: input.scopeType,
|
||||
scopeId: input.scopeId,
|
||||
reuseKey: input.reuseKey,
|
||||
command,
|
||||
cwd: serviceCwd,
|
||||
port,
|
||||
url,
|
||||
provider: "local_process",
|
||||
providerRef: null,
|
||||
ownerAgentId: input.agent.id ?? null,
|
||||
startedByRunId,
|
||||
lastUsedAt: nowIso,
|
||||
startedAt: nowIso,
|
||||
stoppedAt: null,
|
||||
stopPolicy,
|
||||
healthStatus: "unknown",
|
||||
exposure: reservedExposure?.status ?? null,
|
||||
reused: false,
|
||||
db: input.db,
|
||||
child: null,
|
||||
leaseRunIds: leaseRunId ? new Set([leaseRunId]) : new Set(),
|
||||
idleTimer: null,
|
||||
envFingerprint,
|
||||
serviceKey,
|
||||
profileKind: "workspace-runtime",
|
||||
processGroupId: null,
|
||||
exposureHandle: reservedExposure?.handle ?? null,
|
||||
backendUrl,
|
||||
exposureConfig,
|
||||
};
|
||||
if (reservedExposure) {
|
||||
// The broker reservation and its unguessable handle must be durable before
|
||||
// the child can bind, so a server crash cannot lose cleanup authority.
|
||||
try {
|
||||
await persistRuntimeServiceRecord(input.db, record);
|
||||
} catch (error) {
|
||||
await cleanupRecordExposure(record);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await ensureServerWorkspaceLinksCurrent(serviceCwd, {
|
||||
onLog: input.onLog,
|
||||
});
|
||||
} catch (error) {
|
||||
if (reservedExposure) await cleanupRecordExposure(record);
|
||||
throw error;
|
||||
}
|
||||
|
||||
const shell = resolveShell();
|
||||
const child = spawn(shell, ["-lc", command], {
|
||||
@@ -4674,6 +4908,9 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
record.child = child;
|
||||
record.providerRef = child.pid ? String(child.pid) : null;
|
||||
record.processGroupId = child.pid ?? null;
|
||||
const spawnErrorPromise = new Promise<never>((_, reject) => {
|
||||
child.once("error", (err) => {
|
||||
reject(err);
|
||||
@@ -4699,44 +4936,6 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
if (input.onLog) await input.onLog("stderr", `[service:${serviceName}] ${text}`);
|
||||
});
|
||||
|
||||
const nowIso = new Date().toISOString();
|
||||
const record: RuntimeServiceRecord = {
|
||||
id: input.runtimeServiceId ?? stoppedReuseCandidate?.id ?? randomUUID(),
|
||||
companyId: input.agent.companyId,
|
||||
projectId: input.workspace.projectId,
|
||||
projectWorkspaceId: input.workspace.workspaceId,
|
||||
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
||||
issueId: input.issue?.id ?? null,
|
||||
serviceName,
|
||||
status: "starting",
|
||||
lifecycle,
|
||||
scopeType: input.scopeType,
|
||||
scopeId: input.scopeId,
|
||||
reuseKey: input.reuseKey,
|
||||
command,
|
||||
cwd: serviceCwd,
|
||||
port,
|
||||
url,
|
||||
provider: "local_process",
|
||||
providerRef: child.pid ? String(child.pid) : null,
|
||||
ownerAgentId: input.agent.id ?? null,
|
||||
startedByRunId,
|
||||
lastUsedAt: nowIso,
|
||||
startedAt: nowIso,
|
||||
stoppedAt: null,
|
||||
stopPolicy,
|
||||
healthStatus: "unknown",
|
||||
reused: false,
|
||||
db: input.db,
|
||||
child,
|
||||
leaseRunIds: leaseRunId ? new Set([leaseRunId]) : new Set(),
|
||||
idleTimer: null,
|
||||
envFingerprint,
|
||||
serviceKey,
|
||||
profileKind: "workspace-runtime",
|
||||
processGroupId: child.pid ?? null,
|
||||
};
|
||||
|
||||
if (child.pid) {
|
||||
await writeLocalServiceRegistryRecord({
|
||||
version: 1,
|
||||
@@ -4747,7 +4946,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
cwd: serviceCwd,
|
||||
envFingerprint: serviceIdentityFingerprint,
|
||||
port,
|
||||
url,
|
||||
url: backendUrl,
|
||||
pid: child.pid,
|
||||
processGroupId: child.pid,
|
||||
provider: "local_process",
|
||||
@@ -4768,7 +4967,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
|
||||
const readinessPromise = Promise.race([
|
||||
Promise.all([
|
||||
waitForReadiness({ service: input.service, serviceName, command, url, readinessUrl }),
|
||||
waitForReadiness({ service: input.service, serviceName, command, url: backendUrl, readinessUrl }),
|
||||
canAllocateFixedPort && port
|
||||
? waitForAllocatedPortBind({ service: input.service, port, child })
|
||||
: Promise.resolve(),
|
||||
@@ -4776,6 +4975,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
spawnErrorPromise,
|
||||
earlyExitPromise,
|
||||
]).then(async () => {
|
||||
if (record.exposureConfig && record.exposureHandle && record.port) {
|
||||
const provisioned = await provisionExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: record.id,
|
||||
config: record.exposureConfig,
|
||||
handle: record.exposureHandle,
|
||||
hostname: exposureHostname!,
|
||||
appPort: record.port,
|
||||
});
|
||||
record.exposure = provisioned.status;
|
||||
record.exposureHandle = provisioned.handle;
|
||||
record.url = provisioned.status.publicUrl;
|
||||
await persistRuntimeServiceRecord(record.db, record);
|
||||
if (provisioned.status.state !== "ready" || !record.url) {
|
||||
throw new Error(`HTTPS exposure failed: ${provisioned.status.lastError ?? "unknown error"}`);
|
||||
}
|
||||
}
|
||||
record.status = "running";
|
||||
record.healthStatus = "healthy";
|
||||
record.lastUsedAt = new Date().toISOString();
|
||||
@@ -4786,10 +5001,12 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
});
|
||||
}).catch(async (err) => {
|
||||
const failureMessage = err instanceof Error ? err.message : String(err);
|
||||
const bindCollision = err instanceof RuntimeServicePortBindCollision || Boolean(
|
||||
port
|
||||
&& (input.allowFixedPortFallback || portType === "auto")
|
||||
&& /(?:EADDRINUSE|address already in use)/i.test(`${failureMessage}\n${stderrExcerpt}`),
|
||||
const bindCollision = !exposureConfig && (
|
||||
err instanceof RuntimeServicePortBindCollision || Boolean(
|
||||
port
|
||||
&& (input.allowFixedPortFallback || portType === "auto")
|
||||
&& /(?:EADDRINUSE|address already in use)/i.test(`${failureMessage}\n${stderrExcerpt}`),
|
||||
)
|
||||
);
|
||||
if (child.pid) {
|
||||
await terminateLocalService({
|
||||
@@ -4797,11 +5014,15 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
processGroupId: child.pid,
|
||||
});
|
||||
}
|
||||
await cleanupRecordExposure(record, { preserveFailure: true });
|
||||
record.status = "stopped";
|
||||
record.healthStatus = "unhealthy";
|
||||
record.lastUsedAt = new Date().toISOString();
|
||||
record.stoppedAt = new Date().toISOString();
|
||||
await removeLocalServiceRegistryRecord(record.serviceKey).catch(() => undefined);
|
||||
if (exposureConfig) {
|
||||
await persistRuntimeServiceRecord(record.db, record).catch(() => undefined);
|
||||
}
|
||||
if (bindCollision && port) throw new RuntimeServicePortBindCollision(port);
|
||||
throw new Error(
|
||||
`Failed to start runtime service "${serviceName}": ${failureMessage}${stderrExcerpt ? ` | stderr: ${stderrExcerpt.trim()}` : ""}`,
|
||||
@@ -4948,6 +5169,37 @@ function scheduleIdleStop(record: RuntimeServiceRecord) {
|
||||
}, idleSeconds * 1000);
|
||||
}
|
||||
|
||||
async function cleanupRecordExposure(
|
||||
record: RuntimeServiceRecord,
|
||||
options?: { preserveFailure?: boolean },
|
||||
) {
|
||||
if (!record.exposure) return;
|
||||
const previous = record.exposure;
|
||||
const ports = previous.listeners.map((listener) => listener.targetPort);
|
||||
const result = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: record.id,
|
||||
handle: record.exposureHandle,
|
||||
ports,
|
||||
});
|
||||
for (const port of result.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
||||
if (result.status.state === "removed") {
|
||||
record.exposureHandle = null;
|
||||
record.exposure = options?.preserveFailure && previous.state === "failed"
|
||||
? { ...previous, publicUrl: null, updatedAt: new Date().toISOString() }
|
||||
: { ...previous, state: "removed", publicUrl: null, lastError: null, updatedAt: new Date().toISOString() };
|
||||
} else {
|
||||
record.exposure = {
|
||||
...previous,
|
||||
state: "cleanup_pending",
|
||||
publicUrl: null,
|
||||
lastError: result.status.lastError,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
record.url = null;
|
||||
await persistRuntimeServiceRecord(record.db, record).catch(() => undefined);
|
||||
}
|
||||
|
||||
async function stopRuntimeService(serviceId: string) {
|
||||
const record = runtimeServicesById.get(serviceId);
|
||||
if (!record) return;
|
||||
@@ -4960,6 +5212,7 @@ async function stopRuntimeService(serviceId: string) {
|
||||
if (record.reuseKey && runtimeServicesByReuseKey.get(record.reuseKey) === record.id) {
|
||||
runtimeServicesByReuseKey.delete(record.reuseKey);
|
||||
}
|
||||
await cleanupRecordExposure(record);
|
||||
if (record.child && record.child.pid) {
|
||||
await terminateLocalService({
|
||||
pid: record.child.pid,
|
||||
@@ -4983,6 +5236,43 @@ async function markPersistedRuntimeServicesStoppedForExecutionWorkspace(input: {
|
||||
executionWorkspaceId: string;
|
||||
}) {
|
||||
const now = new Date();
|
||||
const exposureRows = await input.db
|
||||
.select({
|
||||
id: workspaceRuntimeServices.id,
|
||||
exposure: workspaceRuntimeServices.exposure,
|
||||
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
||||
})
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceRuntimeServices.executionWorkspaceId, input.executionWorkspaceId),
|
||||
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
||||
),
|
||||
);
|
||||
for (const row of exposureRows) {
|
||||
if (!row.exposure) continue;
|
||||
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: row.id,
|
||||
handle: row.exposureHandle,
|
||||
ports: row.exposure.listeners.map((listener) => listener.targetPort),
|
||||
});
|
||||
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
||||
const exposure: RuntimeExposureStatus = {
|
||||
...row.exposure,
|
||||
state: cleanup.status.state,
|
||||
publicUrl: null,
|
||||
lastError: cleanup.status.lastError,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
await input.db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
exposure,
|
||||
exposureHandle: cleanup.status.state === "removed" ? null : row.exposureHandle,
|
||||
url: null,
|
||||
})
|
||||
.where(eq(workspaceRuntimeServices.id, row.id));
|
||||
}
|
||||
await input.db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
@@ -5000,6 +5290,39 @@ async function markPersistedRuntimeServicesStoppedForExecutionWorkspace(input: {
|
||||
);
|
||||
}
|
||||
|
||||
async function cleanupPersistedExposureRows(
|
||||
db: Db,
|
||||
rows: Array<{
|
||||
id: string;
|
||||
exposure: RuntimeExposureStatus | null;
|
||||
exposureHandle: string | null;
|
||||
}>,
|
||||
) {
|
||||
for (const row of rows) {
|
||||
if (!row.exposure) continue;
|
||||
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: row.id,
|
||||
handle: row.exposureHandle,
|
||||
ports: row.exposure.listeners.map((listener) => listener.targetPort),
|
||||
});
|
||||
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
||||
await db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
url: null,
|
||||
exposure: {
|
||||
...row.exposure,
|
||||
state: cleanup.status.state,
|
||||
publicUrl: null,
|
||||
lastError: cleanup.status.lastError,
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
exposureHandle: cleanup.status.state === "removed" ? null : row.exposureHandle,
|
||||
})
|
||||
.where(eq(workspaceRuntimeServices.id, row.id));
|
||||
}
|
||||
}
|
||||
|
||||
function registerRuntimeService(db: Db | undefined, record: RuntimeServiceRecord) {
|
||||
record.db = db;
|
||||
runtimeServicesById.set(record.id, record);
|
||||
@@ -5019,8 +5342,11 @@ function registerRuntimeService(db: Db | undefined, record: RuntimeServiceRecord
|
||||
if (current.reuseKey && runtimeServicesByReuseKey.get(current.reuseKey) === current.id) {
|
||||
runtimeServicesByReuseKey.delete(current.reuseKey);
|
||||
}
|
||||
void removeLocalServiceRegistryRecord(current.serviceKey);
|
||||
void persistRuntimeServiceRecord(db, current);
|
||||
void (async () => {
|
||||
await cleanupRecordExposure(current);
|
||||
await removeLocalServiceRegistryRecord(current.serviceKey);
|
||||
await persistRuntimeServiceRecord(db, current);
|
||||
})();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -5436,8 +5762,17 @@ export async function startRuntimeServicesForWorkspaceControl(
|
||||
const invocationId = input.invocationId ?? randomUUID();
|
||||
const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input);
|
||||
const provisionCoordinator = createRuntimeProvisionCoordinator();
|
||||
const hasHttpsExposure = rawServices.some((service) => configuredRuntimeExposure(service) !== null);
|
||||
|
||||
if (rawServices.length === 0 || !input.db || (!input.executionWorkspaceId && !input.workspace.workspaceId)) {
|
||||
if (
|
||||
rawServices.length === 0
|
||||
|| !input.db
|
||||
|| (!input.executionWorkspaceId && !input.workspace.workspaceId)
|
||||
// The reservation row must commit before the backend binds. Keeping this
|
||||
// path outside the parent-row transaction avoids a crash window where the
|
||||
// broker lease exists but the DB transaction has not committed its handle.
|
||||
|| hasHttpsExposure
|
||||
) {
|
||||
const batch = await startRuntimeServicesForWorkspaceControlUnlocked(
|
||||
input,
|
||||
rawServices,
|
||||
@@ -5680,6 +6015,15 @@ export async function stopRuntimeServicesForExecutionWorkspace(input: {
|
||||
if (input.db) {
|
||||
if (input.runtimeServiceId) {
|
||||
const now = new Date();
|
||||
const rows = await input.db
|
||||
.select({
|
||||
id: workspaceRuntimeServices.id,
|
||||
exposure: workspaceRuntimeServices.exposure,
|
||||
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
||||
})
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(eq(workspaceRuntimeServices.id, input.runtimeServiceId));
|
||||
await cleanupPersistedExposureRows(input.db, rows);
|
||||
await input.db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
@@ -5717,6 +6061,22 @@ export async function stopRuntimeServicesForProjectWorkspace(input: {
|
||||
|
||||
if (input.db) {
|
||||
const now = new Date();
|
||||
const exposureCondition = input.runtimeServiceId
|
||||
? eq(workspaceRuntimeServices.id, input.runtimeServiceId)
|
||||
: and(
|
||||
eq(workspaceRuntimeServices.projectWorkspaceId, input.projectWorkspaceId),
|
||||
eq(workspaceRuntimeServices.scopeType, "project_workspace"),
|
||||
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
||||
);
|
||||
const exposureRows = await input.db
|
||||
.select({
|
||||
id: workspaceRuntimeServices.id,
|
||||
exposure: workspaceRuntimeServices.exposure,
|
||||
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
||||
})
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(exposureCondition);
|
||||
await cleanupPersistedExposureRows(input.db, exposureRows);
|
||||
await input.db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
@@ -5727,13 +6087,7 @@ export async function stopRuntimeServicesForProjectWorkspace(input: {
|
||||
updatedAt: now,
|
||||
})
|
||||
.where(
|
||||
input.runtimeServiceId
|
||||
? eq(workspaceRuntimeServices.id, input.runtimeServiceId)
|
||||
: and(
|
||||
eq(workspaceRuntimeServices.projectWorkspaceId, input.projectWorkspaceId),
|
||||
eq(workspaceRuntimeServices.scopeType, "project_workspace"),
|
||||
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
||||
),
|
||||
exposureCondition,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5777,10 +6131,44 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
),
|
||||
);
|
||||
|
||||
let ownedExposureListeners: Awaited<ReturnType<BrokerClient["list"]>> | null = [];
|
||||
if (rows.some((row) => row.exposure && row.exposure.state !== "removed")) {
|
||||
try {
|
||||
ownedExposureListeners = await workspaceRuntimeExposureDeps.broker.list();
|
||||
} catch {
|
||||
ownedExposureListeners = null;
|
||||
}
|
||||
}
|
||||
|
||||
let reconciled = 0;
|
||||
let adopted = 0;
|
||||
let stopped = 0;
|
||||
for (const row of rows) {
|
||||
if (row.status === "stopped" && row.exposure && row.exposure.state !== "removed") {
|
||||
await cleanupPersistedExposureRows(db, [{
|
||||
id: row.id,
|
||||
exposure: row.exposure,
|
||||
exposureHandle: row.exposureHandle,
|
||||
}]);
|
||||
reconciled += 1;
|
||||
continue;
|
||||
}
|
||||
const rowExposureListeners = ownedExposureListeners?.filter((listener) => listener.runtimeId === row.id) ?? [];
|
||||
const exposureMappingMatches = !row.exposure || (
|
||||
row.exposure.state === "ready"
|
||||
&& Boolean(row.exposureHandle)
|
||||
&& rowExposureListeners.length === row.exposure.listeners.length
|
||||
&& row.exposure.listeners.every((expected) => rowExposureListeners.some((actual) => (
|
||||
actual.port === expected.targetPort && actual.purpose === expected.purpose
|
||||
)))
|
||||
);
|
||||
const exposureHealthMatches = !row.exposure || (
|
||||
exposureMappingMatches
|
||||
&& Boolean(row.exposure.publicUrl)
|
||||
&& await workspaceRuntimeExposureDeps.probeHealth(
|
||||
new URL("/api/health", row.exposure.publicUrl!).toString(),
|
||||
)
|
||||
);
|
||||
let adoptedRecord = await findLocalServiceRegistryRecordByRuntimeServiceId({
|
||||
runtimeServiceId: row.id,
|
||||
profileKind: "workspace-runtime",
|
||||
@@ -5820,12 +6208,16 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
cwd: row.cwd,
|
||||
envFingerprint: row.reuseKey ?? "",
|
||||
port: row.port ?? null,
|
||||
url: row.url ?? null,
|
||||
url: row.backendUrl ?? row.url ?? null,
|
||||
});
|
||||
}
|
||||
if (adoptedRecord) {
|
||||
const adoptedUrl = adoptedRecord.url ?? row.url ?? null;
|
||||
if (!(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))) {
|
||||
const adoptedUrl = adoptedRecord.url ?? row.backendUrl ?? row.url ?? null;
|
||||
if (
|
||||
!exposureHealthMatches
|
||||
|| !(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))
|
||||
) {
|
||||
await terminateLocalService(adoptedRecord);
|
||||
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
|
||||
} else {
|
||||
const record: RuntimeServiceRecord = {
|
||||
@@ -5844,7 +6236,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
command: row.command ?? null,
|
||||
cwd: row.cwd ?? null,
|
||||
port: adoptedRecord.port ?? row.port ?? null,
|
||||
url: adoptedRecord.url ?? row.url ?? null,
|
||||
url: row.exposure?.publicUrl ?? adoptedRecord.url ?? row.url ?? null,
|
||||
provider: "local_process",
|
||||
providerRef: String(adoptedRecord.pid),
|
||||
ownerAgentId: row.ownerAgentId ?? null,
|
||||
@@ -5854,6 +6246,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
stoppedAt: null,
|
||||
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
|
||||
healthStatus: "healthy",
|
||||
exposure: row.exposure ?? null,
|
||||
reused: true,
|
||||
db,
|
||||
child: null,
|
||||
@@ -5863,6 +6256,9 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
serviceKey: adoptedRecord.serviceKey,
|
||||
profileKind: "workspace-runtime",
|
||||
processGroupId: adoptedRecord.processGroupId ?? null,
|
||||
exposureHandle: row.exposureHandle ?? null,
|
||||
backendUrl: adoptedUrl,
|
||||
exposureConfig: null,
|
||||
};
|
||||
registerRuntimeService(db, record);
|
||||
await touchLocalServiceRegistryRecord(adoptedRecord.serviceKey, {
|
||||
@@ -5881,11 +6277,32 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
let stoppedExposure = row.exposure ?? null;
|
||||
let stoppedExposureHandle = row.exposureHandle ?? null;
|
||||
if (stoppedExposure) {
|
||||
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
||||
runtimeId: row.id,
|
||||
handle: stoppedExposureHandle,
|
||||
ports: stoppedExposure.listeners.map((listener) => listener.targetPort),
|
||||
});
|
||||
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
||||
stoppedExposure = {
|
||||
...stoppedExposure,
|
||||
state: cleanup.status.state,
|
||||
publicUrl: null,
|
||||
lastError: cleanup.status.lastError,
|
||||
updatedAt: now.toISOString(),
|
||||
};
|
||||
if (cleanup.status.state === "removed") stoppedExposureHandle = null;
|
||||
}
|
||||
await db
|
||||
.update(workspaceRuntimeServices)
|
||||
.set({
|
||||
status: "stopped",
|
||||
healthStatus: "unknown",
|
||||
url: stoppedExposure ? null : row.url,
|
||||
exposure: stoppedExposure,
|
||||
exposureHandle: stoppedExposureHandle,
|
||||
stoppedAt: now,
|
||||
lastUsedAt: now,
|
||||
updatedAt: now,
|
||||
@@ -6073,6 +6490,9 @@ export async function persistAdapterManagedRuntimeServices(input: {
|
||||
startedAt,
|
||||
stoppedAt: ref.stoppedAt ? new Date(ref.stoppedAt) : null,
|
||||
stopPolicy: ref.stopPolicy,
|
||||
exposure: null,
|
||||
exposureHandle: null,
|
||||
backendUrl: null,
|
||||
healthStatus: ref.healthStatus,
|
||||
createdAt,
|
||||
updatedAt: new Date(),
|
||||
@@ -6102,6 +6522,9 @@ export async function persistAdapterManagedRuntimeServices(input: {
|
||||
startedAt,
|
||||
stoppedAt: ref.stoppedAt ? new Date(ref.stoppedAt) : null,
|
||||
stopPolicy: ref.stopPolicy,
|
||||
exposure: null,
|
||||
exposureHandle: null,
|
||||
backendUrl: null,
|
||||
healthStatus: ref.healthStatus,
|
||||
updatedAt: new Date(),
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user