|
|
|
@@ -11,6 +11,9 @@ import type { Db } from "@paperclipai/db";
|
|
|
|
|
import { executionWorkspaces, issueComments, issues, projectWorkspaces, workspaceRuntimeServices } from "@paperclipai/db";
|
|
|
|
|
import {
|
|
|
|
|
listWorkspaceServiceCommandDefinitions,
|
|
|
|
|
parseRuntimeExposureConfig,
|
|
|
|
|
type RuntimeExposureConfigInput,
|
|
|
|
|
type RuntimeExposureStatus,
|
|
|
|
|
type GitWorktreeBranchAncestryVerdict,
|
|
|
|
|
type GitWorktreeBranchIncoherenceEvidence as SharedGitWorktreeBranchIncoherenceEvidence,
|
|
|
|
|
type GitWorktreeInProgressOperation,
|
|
|
|
@@ -48,6 +51,15 @@ import {
|
|
|
|
|
WORKTREE_INSTANCE_ROOT_METADATA_KEY,
|
|
|
|
|
type WorktreeInstancePointer,
|
|
|
|
|
} from "./workspace-instance-cleanup.js";
|
|
|
|
|
import { UnixBrokerClient, type BrokerClient } from "./runtime-exposure/broker-client.js";
|
|
|
|
|
import {
|
|
|
|
|
deprovisionExposure,
|
|
|
|
|
provisionExposure,
|
|
|
|
|
reserveExposure,
|
|
|
|
|
type ExposureManagerDeps,
|
|
|
|
|
} from "./runtime-exposure/exposure-manager.js";
|
|
|
|
|
import { allocateExposurePortPair } from "./runtime-exposure/port-pair.js";
|
|
|
|
|
import { resolveTailscaleDnsName } from "./runtime-exposure/tailscale-hostname.js";
|
|
|
|
|
|
|
|
|
|
export function resolveShell(): string {
|
|
|
|
|
const fallback = process.platform === "win32" ? "sh" : "/bin/sh";
|
|
|
|
@@ -156,6 +168,7 @@ export interface RuntimeServiceRef {
|
|
|
|
|
stoppedAt: string | null;
|
|
|
|
|
stopPolicy: Record<string, unknown> | null;
|
|
|
|
|
healthStatus: "unknown" | "healthy" | "unhealthy";
|
|
|
|
|
exposure: RuntimeExposureStatus | null;
|
|
|
|
|
reused: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -168,6 +181,11 @@ interface RuntimeServiceRecord extends RuntimeServiceRef {
|
|
|
|
|
serviceKey: string;
|
|
|
|
|
profileKind: string;
|
|
|
|
|
processGroupId: number | null;
|
|
|
|
|
/** Server-private broker lease handle; never returned by toRuntimeServiceRef. */
|
|
|
|
|
exposureHandle: string | null;
|
|
|
|
|
/** Loopback URL used for backend readiness/adoption; never serialized. */
|
|
|
|
|
backendUrl: string | null;
|
|
|
|
|
exposureConfig: RuntimeExposureConfigInput | null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type LocalRuntimeServiceStart = {
|
|
|
|
@@ -188,9 +206,11 @@ const runtimeServicesById = new Map<string, RuntimeServiceRecord>();
|
|
|
|
|
const runtimeServicesByReuseKey = new Map<string, string>();
|
|
|
|
|
const runtimeServiceLeasesByRun = new Map<string, string[]>();
|
|
|
|
|
const runtimeProvisionByWorkspace = new Map<string, Promise<void>>();
|
|
|
|
|
const quarantinedRuntimeExposurePorts = new Set<number>();
|
|
|
|
|
const DEFAULT_EXECUTE_PROCESS_OUTPUT_BYTES = 256 * 1024;
|
|
|
|
|
export const WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS = 32;
|
|
|
|
|
const ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES = ["provisioning", "starting", "running"] as const;
|
|
|
|
|
const DEFAULT_TAILSCALE_BROKER_SOCKET = "/run/paperclip-tailscale-broker/broker.sock";
|
|
|
|
|
|
|
|
|
|
class RuntimeServicePortBindCollision extends Error {
|
|
|
|
|
readonly port: number;
|
|
|
|
@@ -202,6 +222,58 @@ class RuntimeServicePortBindCollision extends Error {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export type WorkspaceRuntimeExposureDeps = ExposureManagerDeps & {
|
|
|
|
|
resolveHostname: () => Promise<string>;
|
|
|
|
|
isPortAvailable: (port: number) => Promise<boolean>;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
async function isLoopbackPortAvailable(port: number): Promise<boolean> {
|
|
|
|
|
return await new Promise<boolean>((resolve) => {
|
|
|
|
|
const server = net.createServer();
|
|
|
|
|
server.unref();
|
|
|
|
|
server.once("error", () => resolve(false));
|
|
|
|
|
server.listen(port, "127.0.0.1", () => {
|
|
|
|
|
server.close(() => resolve(true));
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function defaultWorkspaceRuntimeExposureDeps(): WorkspaceRuntimeExposureDeps {
|
|
|
|
|
const broker: BrokerClient = new UnixBrokerClient({
|
|
|
|
|
socketPath: process.env.PAPERCLIP_TAILSCALE_BROKER_SOCKET?.trim() || DEFAULT_TAILSCALE_BROKER_SOCKET,
|
|
|
|
|
});
|
|
|
|
|
return {
|
|
|
|
|
broker,
|
|
|
|
|
resolveHostname: () => resolveTailscaleDnsName(),
|
|
|
|
|
isPortAvailable: isLoopbackPortAvailable,
|
|
|
|
|
probeHealth: async (url) => {
|
|
|
|
|
try {
|
|
|
|
|
const response = await fetch(url, {
|
|
|
|
|
redirect: "error",
|
|
|
|
|
signal: AbortSignal.timeout(5_000),
|
|
|
|
|
});
|
|
|
|
|
return response.ok;
|
|
|
|
|
} catch {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
now: () => new Date().toISOString(),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let workspaceRuntimeExposureDeps = defaultWorkspaceRuntimeExposureDeps();
|
|
|
|
|
|
|
|
|
|
/** Test-only seam; resetRuntimeServicesForTests restores production defaults. */
|
|
|
|
|
export function setWorkspaceRuntimeExposureDepsForTests(deps: WorkspaceRuntimeExposureDeps) {
|
|
|
|
|
workspaceRuntimeExposureDeps = deps;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function configuredRuntimeExposure(service: Record<string, unknown>): RuntimeExposureConfigInput | null {
|
|
|
|
|
const expose = parseObject(service.expose);
|
|
|
|
|
if (asString(expose.type, "") !== "tailscale_https") return null;
|
|
|
|
|
return parseRuntimeExposureConfig(expose);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type ProcessOutputCapture = {
|
|
|
|
|
text: string;
|
|
|
|
|
truncated: boolean;
|
|
|
|
@@ -221,6 +293,8 @@ export async function resetRuntimeServicesForTests() {
|
|
|
|
|
runtimeServicesByReuseKey.clear();
|
|
|
|
|
runtimeServiceLeasesByRun.clear();
|
|
|
|
|
runtimeProvisionByWorkspace.clear();
|
|
|
|
|
quarantinedRuntimeExposurePorts.clear();
|
|
|
|
|
workspaceRuntimeExposureDeps = defaultWorkspaceRuntimeExposureDeps();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function stableStringify(value: unknown): string {
|
|
|
|
@@ -434,6 +508,7 @@ function toRuntimeServiceRef(record: RuntimeServiceRecord, overrides?: Partial<R
|
|
|
|
|
startedAt: record.startedAt,
|
|
|
|
|
stoppedAt: record.stoppedAt,
|
|
|
|
|
stopPolicy: record.stopPolicy,
|
|
|
|
|
exposure: record.exposure,
|
|
|
|
|
healthStatus: record.healthStatus,
|
|
|
|
|
reused: record.reused,
|
|
|
|
|
...overrides,
|
|
|
|
@@ -3734,6 +3809,53 @@ async function allocateIsolatedWorkspacePort(input: {
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function collectReservedExposurePorts(db: Db | undefined, companyId: string): Promise<Set<number>> {
|
|
|
|
|
const reserved = new Set(quarantinedRuntimeExposurePorts);
|
|
|
|
|
for (const record of runtimeServicesById.values()) {
|
|
|
|
|
if (record.companyId !== companyId || !record.exposure) continue;
|
|
|
|
|
for (const listener of record.exposure.listeners) reserved.add(listener.targetPort);
|
|
|
|
|
}
|
|
|
|
|
if (!db) return reserved;
|
|
|
|
|
const rows = await db
|
|
|
|
|
.select({ exposure: workspaceRuntimeServices.exposure })
|
|
|
|
|
.from(workspaceRuntimeServices)
|
|
|
|
|
.where(eq(workspaceRuntimeServices.companyId, companyId));
|
|
|
|
|
for (const row of rows) {
|
|
|
|
|
if (!row.exposure || row.exposure.state === "removed") continue;
|
|
|
|
|
for (const listener of row.exposure.listeners) reserved.add(listener.targetPort);
|
|
|
|
|
}
|
|
|
|
|
return reserved;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function allocateAndReserveExposure(input: {
|
|
|
|
|
db?: Db;
|
|
|
|
|
companyId: string;
|
|
|
|
|
runtimeId: string;
|
|
|
|
|
config: RuntimeExposureConfigInput;
|
|
|
|
|
}): Promise<{ appPort: number; status: RuntimeExposureStatus; handle: string }> {
|
|
|
|
|
const reserved = await collectReservedExposurePorts(input.db, input.companyId);
|
|
|
|
|
const retryable = new Set(["reservation_conflict", "manual_mapping_present", "quarantined"]);
|
|
|
|
|
while (true) {
|
|
|
|
|
const pair = await allocateExposurePortPair({
|
|
|
|
|
isPortAvailable: workspaceRuntimeExposureDeps.isPortAvailable,
|
|
|
|
|
reserved,
|
|
|
|
|
});
|
|
|
|
|
const result = await reserveExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: input.runtimeId,
|
|
|
|
|
config: input.config,
|
|
|
|
|
appPort: pair.appPort,
|
|
|
|
|
});
|
|
|
|
|
if (result.handle) {
|
|
|
|
|
return { appPort: pair.appPort, status: result.status, handle: result.handle };
|
|
|
|
|
}
|
|
|
|
|
if (!result.status.lastError || !retryable.has(result.status.lastError)) {
|
|
|
|
|
throw new Error(`HTTPS exposure reservation failed: ${result.status.lastError ?? "unknown broker error"}`);
|
|
|
|
|
}
|
|
|
|
|
reserved.add(pair.appPort);
|
|
|
|
|
reserved.add(pair.hmrPort);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildTemplateData(input: {
|
|
|
|
|
workspace: RealizedExecutionWorkspace;
|
|
|
|
|
agent: ExecutionWorkspaceAgentRef;
|
|
|
|
@@ -3827,6 +3949,7 @@ function resolveRuntimeServiceReuseIdentity(input: {
|
|
|
|
|
cwd: serviceCwd,
|
|
|
|
|
port: identityPort,
|
|
|
|
|
env: renderedEnv,
|
|
|
|
|
expose: input.service.expose ?? null,
|
|
|
|
|
}),
|
|
|
|
|
)
|
|
|
|
|
.digest("hex")
|
|
|
|
@@ -4115,6 +4238,9 @@ function toPersistedWorkspaceRuntimeService(record: RuntimeServiceRecord): typeo
|
|
|
|
|
startedAt: new Date(record.startedAt),
|
|
|
|
|
stoppedAt: record.stoppedAt ? new Date(record.stoppedAt) : null,
|
|
|
|
|
stopPolicy: record.stopPolicy,
|
|
|
|
|
exposure: record.exposure,
|
|
|
|
|
exposureHandle: record.exposureHandle,
|
|
|
|
|
backendUrl: record.backendUrl,
|
|
|
|
|
healthStatus: record.healthStatus,
|
|
|
|
|
updatedAt: new Date(),
|
|
|
|
|
};
|
|
|
|
@@ -4151,6 +4277,9 @@ async function persistRuntimeServiceRecord(db: Db | undefined, record: RuntimeSe
|
|
|
|
|
startedAt: values.startedAt,
|
|
|
|
|
stoppedAt: values.stoppedAt,
|
|
|
|
|
stopPolicy: values.stopPolicy,
|
|
|
|
|
exposure: values.exposure,
|
|
|
|
|
exposureHandle: values.exposureHandle,
|
|
|
|
|
backendUrl: values.backendUrl,
|
|
|
|
|
healthStatus: values.healthStatus,
|
|
|
|
|
updatedAt: values.updatedAt,
|
|
|
|
|
},
|
|
|
|
@@ -4287,6 +4416,7 @@ export function normalizeAdapterManagedRuntimeServices(input: {
|
|
|
|
|
stoppedAt: status === "running" || status === "starting" ? null : nowIso,
|
|
|
|
|
stopPolicy: report.stopPolicy ?? null,
|
|
|
|
|
healthStatus,
|
|
|
|
|
exposure: null,
|
|
|
|
|
reused: false,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
@@ -4455,6 +4585,7 @@ function createProvisioningRuntimeServiceRecord(
|
|
|
|
|
stoppedAt: null,
|
|
|
|
|
stopPolicy: parseObject(input.service.stopPolicy),
|
|
|
|
|
healthStatus: "unknown",
|
|
|
|
|
exposure: null,
|
|
|
|
|
reused: false,
|
|
|
|
|
db: input.db,
|
|
|
|
|
child: null,
|
|
|
|
@@ -4464,6 +4595,9 @@ function createProvisioningRuntimeServiceRecord(
|
|
|
|
|
serviceKey: `runtime-provision:${runtimeProvisionWorkspaceKey(input)}:${id}`,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
processGroupId: null,
|
|
|
|
|
exposureHandle: null,
|
|
|
|
|
backendUrl: null,
|
|
|
|
|
exposureConfig: null,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -4490,8 +4624,13 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
const explicitPort = identity.explicitPort;
|
|
|
|
|
const identityPort = identity.identityPort;
|
|
|
|
|
const portType = asString(portConfig.type, "");
|
|
|
|
|
const exposureConfig = configuredRuntimeExposure(input.service);
|
|
|
|
|
if (exposureConfig && portType !== "auto") {
|
|
|
|
|
throw new Error(`Runtime service "${serviceName}" must use port.type=auto with tailscale_https exposure`);
|
|
|
|
|
}
|
|
|
|
|
const canAllocateFixedPort = Boolean(
|
|
|
|
|
input.allowFixedPortFallback
|
|
|
|
|
!exposureConfig
|
|
|
|
|
&& input.allowFixedPortFallback
|
|
|
|
|
&& input.db
|
|
|
|
|
&& input.executionWorkspaceId
|
|
|
|
|
&& explicitPort > 0,
|
|
|
|
@@ -4506,15 +4645,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
scopeType: input.scopeType,
|
|
|
|
|
scopeId: input.scopeId,
|
|
|
|
|
});
|
|
|
|
|
let reusableStoppedPort: number | null = null;
|
|
|
|
|
if (portType === "auto" && stoppedReuseCandidate?.port && !input.excludedPorts?.has(stoppedReuseCandidate.port)) {
|
|
|
|
|
reusableStoppedPort = await canBindRuntimePort(stoppedReuseCandidate.port)
|
|
|
|
|
? stoppedReuseCandidate.port
|
|
|
|
|
: null;
|
|
|
|
|
}
|
|
|
|
|
let port: number | null = null;
|
|
|
|
|
if (portType === "auto") {
|
|
|
|
|
port = reusableStoppedPort;
|
|
|
|
|
const runtimeId = input.runtimeServiceId ?? stoppedReuseCandidate?.id ?? randomUUID();
|
|
|
|
|
const reservedExposure = exposureConfig
|
|
|
|
|
? await allocateAndReserveExposure({
|
|
|
|
|
db: input.db,
|
|
|
|
|
companyId: input.agent.companyId,
|
|
|
|
|
runtimeId,
|
|
|
|
|
config: exposureConfig,
|
|
|
|
|
})
|
|
|
|
|
: null;
|
|
|
|
|
let port: number | null = reservedExposure?.appPort ?? null;
|
|
|
|
|
if (!reservedExposure && portType === "auto") {
|
|
|
|
|
if (stoppedReuseCandidate?.port && !input.excludedPorts?.has(stoppedReuseCandidate.port)) {
|
|
|
|
|
port = await canBindRuntimePort(stoppedReuseCandidate.port)
|
|
|
|
|
? stoppedReuseCandidate.port
|
|
|
|
|
: null;
|
|
|
|
|
}
|
|
|
|
|
for (let attempt = 0; port === null && attempt < WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS; attempt += 1) {
|
|
|
|
|
const candidate = await allocatePort();
|
|
|
|
|
if (!input.excludedPorts?.has(candidate)) port = candidate;
|
|
|
|
@@ -4526,7 +4672,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
remediation: "Retry the start or configure a different runtime service port.",
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
} else if (canAllocateFixedPort) {
|
|
|
|
|
} else if (!reservedExposure && canAllocateFixedPort) {
|
|
|
|
|
port = await allocateIsolatedWorkspacePort({
|
|
|
|
|
db: input.db,
|
|
|
|
|
companyId: input.agent.companyId,
|
|
|
|
@@ -4535,9 +4681,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
stoppedPort: stoppedReuseCandidate?.port ?? null,
|
|
|
|
|
excludedPorts: input.excludedPorts ?? new Set<number>(),
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
} else if (!reservedExposure) {
|
|
|
|
|
port = explicitPort > 0 ? explicitPort : null;
|
|
|
|
|
}
|
|
|
|
|
let exposureHostname: string | null = null;
|
|
|
|
|
if (reservedExposure) {
|
|
|
|
|
try {
|
|
|
|
|
exposureHostname = await workspaceRuntimeExposureDeps.resolveHostname();
|
|
|
|
|
reservedExposure.status.hostname = exposureHostname;
|
|
|
|
|
reservedExposure.status.updatedAt = new Date().toISOString();
|
|
|
|
|
} catch {
|
|
|
|
|
await workspaceRuntimeExposureDeps.broker
|
|
|
|
|
.remove(runtimeId, reservedExposure.handle)
|
|
|
|
|
.catch(() => undefined);
|
|
|
|
|
throw new Error("HTTPS exposure failed: Tailscale MagicDNS hostname unavailable");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const templateData = buildTemplateData({
|
|
|
|
|
workspace: input.workspace,
|
|
|
|
|
agent: input.agent,
|
|
|
|
@@ -4561,12 +4720,26 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
env[portEnvKey] = String(port);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (exposureConfig) {
|
|
|
|
|
// Paperclip dev-runtime-specific hardening. Other managed processes are
|
|
|
|
|
// still rejected by the broker unless /proc proves loopback-only listeners.
|
|
|
|
|
env.PAPERCLIP_BIND = "custom";
|
|
|
|
|
env.PAPERCLIP_BIND_HOST = "127.0.0.1";
|
|
|
|
|
env.PAPERCLIP_VITE_HMR_PROTOCOL = "wss";
|
|
|
|
|
env.PAPERCLIP_MANAGED_RUNTIME_EXPOSURE = "tailscale_https";
|
|
|
|
|
env.PAPERCLIP_ALLOWED_HOSTNAMES = exposureHostname!;
|
|
|
|
|
env.PAPERCLIP_AUTH_BASE_URL_MODE = "explicit";
|
|
|
|
|
env.PAPERCLIP_AUTH_PUBLIC_BASE_URL = `https://${exposureHostname}:${port}`;
|
|
|
|
|
env.PAPERCLIP_PUBLIC_URL = `https://${exposureHostname}:${port}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const expose = parseObject(input.service.expose);
|
|
|
|
|
const readiness = parseObject(input.service.readiness);
|
|
|
|
|
const urlTemplate =
|
|
|
|
|
asString(expose.urlTemplate, "") ||
|
|
|
|
|
asString(readiness.urlTemplate, "");
|
|
|
|
|
const url = urlTemplate ? renderTemplate(urlTemplate, templateData) : null;
|
|
|
|
|
const backendUrl = urlTemplate ? renderTemplate(urlTemplate, templateData) : null;
|
|
|
|
|
let url = exposureConfig ? null : backendUrl;
|
|
|
|
|
const readinessUrlTemplate = asString(readiness.urlTemplate, "");
|
|
|
|
|
const readinessUrl = readinessUrlTemplate ? renderTemplate(readinessUrlTemplate, templateData) : null;
|
|
|
|
|
const stopPolicy = parseObject(input.service.stopPolicy);
|
|
|
|
@@ -4584,7 +4757,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
reuseKey: input.reuseKey,
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
const adoptedRecord = await findAdoptableLocalService({
|
|
|
|
|
const adoptedRecord = exposureConfig ? null : await findAdoptableLocalService({
|
|
|
|
|
serviceKey,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
serviceName,
|
|
|
|
@@ -4592,10 +4765,10 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
cwd: serviceCwd,
|
|
|
|
|
envFingerprint: serviceIdentityFingerprint,
|
|
|
|
|
port: port ?? identityPort,
|
|
|
|
|
url,
|
|
|
|
|
url: backendUrl,
|
|
|
|
|
});
|
|
|
|
|
if (adoptedRecord) {
|
|
|
|
|
const adoptedUrl = adoptedRecord.url ?? url;
|
|
|
|
|
const adoptedUrl = adoptedRecord.url ?? backendUrl;
|
|
|
|
|
if (!(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName, command }))) {
|
|
|
|
|
await terminateLocalService(adoptedRecord);
|
|
|
|
|
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
|
|
|
|
@@ -4627,6 +4800,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
stoppedAt: null,
|
|
|
|
|
stopPolicy,
|
|
|
|
|
healthStatus: "healthy",
|
|
|
|
|
exposure: null,
|
|
|
|
|
reused: true,
|
|
|
|
|
db: input.db,
|
|
|
|
|
child: null,
|
|
|
|
@@ -4636,6 +4810,9 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
serviceKey,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
processGroupId: adoptedRecord.processGroupId ?? null,
|
|
|
|
|
exposureHandle: null,
|
|
|
|
|
backendUrl: adoptedUrl,
|
|
|
|
|
exposureConfig: null,
|
|
|
|
|
},
|
|
|
|
|
readiness: Promise.resolve(),
|
|
|
|
|
};
|
|
|
|
@@ -4663,9 +4840,66 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await ensureServerWorkspaceLinksCurrent(serviceCwd, {
|
|
|
|
|
onLog: input.onLog,
|
|
|
|
|
});
|
|
|
|
|
const nowIso = new Date().toISOString();
|
|
|
|
|
const record: RuntimeServiceRecord = {
|
|
|
|
|
id: runtimeId,
|
|
|
|
|
companyId: input.agent.companyId,
|
|
|
|
|
projectId: input.workspace.projectId,
|
|
|
|
|
projectWorkspaceId: input.workspace.workspaceId,
|
|
|
|
|
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
|
|
|
|
issueId: input.issue?.id ?? null,
|
|
|
|
|
serviceName,
|
|
|
|
|
status: "starting",
|
|
|
|
|
lifecycle,
|
|
|
|
|
scopeType: input.scopeType,
|
|
|
|
|
scopeId: input.scopeId,
|
|
|
|
|
reuseKey: input.reuseKey,
|
|
|
|
|
command,
|
|
|
|
|
cwd: serviceCwd,
|
|
|
|
|
port,
|
|
|
|
|
url,
|
|
|
|
|
provider: "local_process",
|
|
|
|
|
providerRef: null,
|
|
|
|
|
ownerAgentId: input.agent.id ?? null,
|
|
|
|
|
startedByRunId,
|
|
|
|
|
lastUsedAt: nowIso,
|
|
|
|
|
startedAt: nowIso,
|
|
|
|
|
stoppedAt: null,
|
|
|
|
|
stopPolicy,
|
|
|
|
|
healthStatus: "unknown",
|
|
|
|
|
exposure: reservedExposure?.status ?? null,
|
|
|
|
|
reused: false,
|
|
|
|
|
db: input.db,
|
|
|
|
|
child: null,
|
|
|
|
|
leaseRunIds: leaseRunId ? new Set([leaseRunId]) : new Set(),
|
|
|
|
|
idleTimer: null,
|
|
|
|
|
envFingerprint,
|
|
|
|
|
serviceKey,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
processGroupId: null,
|
|
|
|
|
exposureHandle: reservedExposure?.handle ?? null,
|
|
|
|
|
backendUrl,
|
|
|
|
|
exposureConfig,
|
|
|
|
|
};
|
|
|
|
|
if (reservedExposure) {
|
|
|
|
|
// The broker reservation and its unguessable handle must be durable before
|
|
|
|
|
// the child can bind, so a server crash cannot lose cleanup authority.
|
|
|
|
|
try {
|
|
|
|
|
await persistRuntimeServiceRecord(input.db, record);
|
|
|
|
|
} catch (error) {
|
|
|
|
|
await cleanupRecordExposure(record);
|
|
|
|
|
throw error;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await ensureServerWorkspaceLinksCurrent(serviceCwd, {
|
|
|
|
|
onLog: input.onLog,
|
|
|
|
|
});
|
|
|
|
|
} catch (error) {
|
|
|
|
|
if (reservedExposure) await cleanupRecordExposure(record);
|
|
|
|
|
throw error;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const shell = resolveShell();
|
|
|
|
|
const child = spawn(shell, ["-lc", command], {
|
|
|
|
@@ -4674,6 +4908,9 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
detached: process.platform !== "win32",
|
|
|
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
|
|
|
});
|
|
|
|
|
record.child = child;
|
|
|
|
|
record.providerRef = child.pid ? String(child.pid) : null;
|
|
|
|
|
record.processGroupId = child.pid ?? null;
|
|
|
|
|
const spawnErrorPromise = new Promise<never>((_, reject) => {
|
|
|
|
|
child.once("error", (err) => {
|
|
|
|
|
reject(err);
|
|
|
|
@@ -4699,44 +4936,6 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
if (input.onLog) await input.onLog("stderr", `[service:${serviceName}] ${text}`);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const nowIso = new Date().toISOString();
|
|
|
|
|
const record: RuntimeServiceRecord = {
|
|
|
|
|
id: input.runtimeServiceId ?? stoppedReuseCandidate?.id ?? randomUUID(),
|
|
|
|
|
companyId: input.agent.companyId,
|
|
|
|
|
projectId: input.workspace.projectId,
|
|
|
|
|
projectWorkspaceId: input.workspace.workspaceId,
|
|
|
|
|
executionWorkspaceId: input.executionWorkspaceId ?? null,
|
|
|
|
|
issueId: input.issue?.id ?? null,
|
|
|
|
|
serviceName,
|
|
|
|
|
status: "starting",
|
|
|
|
|
lifecycle,
|
|
|
|
|
scopeType: input.scopeType,
|
|
|
|
|
scopeId: input.scopeId,
|
|
|
|
|
reuseKey: input.reuseKey,
|
|
|
|
|
command,
|
|
|
|
|
cwd: serviceCwd,
|
|
|
|
|
port,
|
|
|
|
|
url,
|
|
|
|
|
provider: "local_process",
|
|
|
|
|
providerRef: child.pid ? String(child.pid) : null,
|
|
|
|
|
ownerAgentId: input.agent.id ?? null,
|
|
|
|
|
startedByRunId,
|
|
|
|
|
lastUsedAt: nowIso,
|
|
|
|
|
startedAt: nowIso,
|
|
|
|
|
stoppedAt: null,
|
|
|
|
|
stopPolicy,
|
|
|
|
|
healthStatus: "unknown",
|
|
|
|
|
reused: false,
|
|
|
|
|
db: input.db,
|
|
|
|
|
child,
|
|
|
|
|
leaseRunIds: leaseRunId ? new Set([leaseRunId]) : new Set(),
|
|
|
|
|
idleTimer: null,
|
|
|
|
|
envFingerprint,
|
|
|
|
|
serviceKey,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
processGroupId: child.pid ?? null,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
if (child.pid) {
|
|
|
|
|
await writeLocalServiceRegistryRecord({
|
|
|
|
|
version: 1,
|
|
|
|
@@ -4747,7 +4946,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
cwd: serviceCwd,
|
|
|
|
|
envFingerprint: serviceIdentityFingerprint,
|
|
|
|
|
port,
|
|
|
|
|
url,
|
|
|
|
|
url: backendUrl,
|
|
|
|
|
pid: child.pid,
|
|
|
|
|
processGroupId: child.pid,
|
|
|
|
|
provider: "local_process",
|
|
|
|
@@ -4768,7 +4967,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
|
|
|
|
|
const readinessPromise = Promise.race([
|
|
|
|
|
Promise.all([
|
|
|
|
|
waitForReadiness({ service: input.service, serviceName, command, url, readinessUrl }),
|
|
|
|
|
waitForReadiness({ service: input.service, serviceName, command, url: backendUrl, readinessUrl }),
|
|
|
|
|
canAllocateFixedPort && port
|
|
|
|
|
? waitForAllocatedPortBind({ service: input.service, port, child })
|
|
|
|
|
: Promise.resolve(),
|
|
|
|
@@ -4776,6 +4975,22 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
spawnErrorPromise,
|
|
|
|
|
earlyExitPromise,
|
|
|
|
|
]).then(async () => {
|
|
|
|
|
if (record.exposureConfig && record.exposureHandle && record.port) {
|
|
|
|
|
const provisioned = await provisionExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: record.id,
|
|
|
|
|
config: record.exposureConfig,
|
|
|
|
|
handle: record.exposureHandle,
|
|
|
|
|
hostname: exposureHostname!,
|
|
|
|
|
appPort: record.port,
|
|
|
|
|
});
|
|
|
|
|
record.exposure = provisioned.status;
|
|
|
|
|
record.exposureHandle = provisioned.handle;
|
|
|
|
|
record.url = provisioned.status.publicUrl;
|
|
|
|
|
await persistRuntimeServiceRecord(record.db, record);
|
|
|
|
|
if (provisioned.status.state !== "ready" || !record.url) {
|
|
|
|
|
throw new Error(`HTTPS exposure failed: ${provisioned.status.lastError ?? "unknown error"}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
record.status = "running";
|
|
|
|
|
record.healthStatus = "healthy";
|
|
|
|
|
record.lastUsedAt = new Date().toISOString();
|
|
|
|
@@ -4786,10 +5001,12 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
});
|
|
|
|
|
}).catch(async (err) => {
|
|
|
|
|
const failureMessage = err instanceof Error ? err.message : String(err);
|
|
|
|
|
const bindCollision = err instanceof RuntimeServicePortBindCollision || Boolean(
|
|
|
|
|
port
|
|
|
|
|
&& (input.allowFixedPortFallback || portType === "auto")
|
|
|
|
|
&& /(?:EADDRINUSE|address already in use)/i.test(`${failureMessage}\n${stderrExcerpt}`),
|
|
|
|
|
const bindCollision = !exposureConfig && (
|
|
|
|
|
err instanceof RuntimeServicePortBindCollision || Boolean(
|
|
|
|
|
port
|
|
|
|
|
&& (input.allowFixedPortFallback || portType === "auto")
|
|
|
|
|
&& /(?:EADDRINUSE|address already in use)/i.test(`${failureMessage}\n${stderrExcerpt}`),
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
if (child.pid) {
|
|
|
|
|
await terminateLocalService({
|
|
|
|
@@ -4797,11 +5014,15 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
|
|
|
|
processGroupId: child.pid,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
await cleanupRecordExposure(record, { preserveFailure: true });
|
|
|
|
|
record.status = "stopped";
|
|
|
|
|
record.healthStatus = "unhealthy";
|
|
|
|
|
record.lastUsedAt = new Date().toISOString();
|
|
|
|
|
record.stoppedAt = new Date().toISOString();
|
|
|
|
|
await removeLocalServiceRegistryRecord(record.serviceKey).catch(() => undefined);
|
|
|
|
|
if (exposureConfig) {
|
|
|
|
|
await persistRuntimeServiceRecord(record.db, record).catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
if (bindCollision && port) throw new RuntimeServicePortBindCollision(port);
|
|
|
|
|
throw new Error(
|
|
|
|
|
`Failed to start runtime service "${serviceName}": ${failureMessage}${stderrExcerpt ? ` | stderr: ${stderrExcerpt.trim()}` : ""}`,
|
|
|
|
@@ -4948,6 +5169,37 @@ function scheduleIdleStop(record: RuntimeServiceRecord) {
|
|
|
|
|
}, idleSeconds * 1000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function cleanupRecordExposure(
|
|
|
|
|
record: RuntimeServiceRecord,
|
|
|
|
|
options?: { preserveFailure?: boolean },
|
|
|
|
|
) {
|
|
|
|
|
if (!record.exposure) return;
|
|
|
|
|
const previous = record.exposure;
|
|
|
|
|
const ports = previous.listeners.map((listener) => listener.targetPort);
|
|
|
|
|
const result = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: record.id,
|
|
|
|
|
handle: record.exposureHandle,
|
|
|
|
|
ports,
|
|
|
|
|
});
|
|
|
|
|
for (const port of result.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
|
|
|
|
if (result.status.state === "removed") {
|
|
|
|
|
record.exposureHandle = null;
|
|
|
|
|
record.exposure = options?.preserveFailure && previous.state === "failed"
|
|
|
|
|
? { ...previous, publicUrl: null, updatedAt: new Date().toISOString() }
|
|
|
|
|
: { ...previous, state: "removed", publicUrl: null, lastError: null, updatedAt: new Date().toISOString() };
|
|
|
|
|
} else {
|
|
|
|
|
record.exposure = {
|
|
|
|
|
...previous,
|
|
|
|
|
state: "cleanup_pending",
|
|
|
|
|
publicUrl: null,
|
|
|
|
|
lastError: result.status.lastError,
|
|
|
|
|
updatedAt: new Date().toISOString(),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
record.url = null;
|
|
|
|
|
await persistRuntimeServiceRecord(record.db, record).catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function stopRuntimeService(serviceId: string) {
|
|
|
|
|
const record = runtimeServicesById.get(serviceId);
|
|
|
|
|
if (!record) return;
|
|
|
|
@@ -4960,6 +5212,7 @@ async function stopRuntimeService(serviceId: string) {
|
|
|
|
|
if (record.reuseKey && runtimeServicesByReuseKey.get(record.reuseKey) === record.id) {
|
|
|
|
|
runtimeServicesByReuseKey.delete(record.reuseKey);
|
|
|
|
|
}
|
|
|
|
|
await cleanupRecordExposure(record);
|
|
|
|
|
if (record.child && record.child.pid) {
|
|
|
|
|
await terminateLocalService({
|
|
|
|
|
pid: record.child.pid,
|
|
|
|
@@ -4983,6 +5236,43 @@ async function markPersistedRuntimeServicesStoppedForExecutionWorkspace(input: {
|
|
|
|
|
executionWorkspaceId: string;
|
|
|
|
|
}) {
|
|
|
|
|
const now = new Date();
|
|
|
|
|
const exposureRows = await input.db
|
|
|
|
|
.select({
|
|
|
|
|
id: workspaceRuntimeServices.id,
|
|
|
|
|
exposure: workspaceRuntimeServices.exposure,
|
|
|
|
|
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
|
|
|
|
})
|
|
|
|
|
.from(workspaceRuntimeServices)
|
|
|
|
|
.where(
|
|
|
|
|
and(
|
|
|
|
|
eq(workspaceRuntimeServices.executionWorkspaceId, input.executionWorkspaceId),
|
|
|
|
|
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
|
|
|
|
),
|
|
|
|
|
);
|
|
|
|
|
for (const row of exposureRows) {
|
|
|
|
|
if (!row.exposure) continue;
|
|
|
|
|
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: row.id,
|
|
|
|
|
handle: row.exposureHandle,
|
|
|
|
|
ports: row.exposure.listeners.map((listener) => listener.targetPort),
|
|
|
|
|
});
|
|
|
|
|
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
|
|
|
|
const exposure: RuntimeExposureStatus = {
|
|
|
|
|
...row.exposure,
|
|
|
|
|
state: cleanup.status.state,
|
|
|
|
|
publicUrl: null,
|
|
|
|
|
lastError: cleanup.status.lastError,
|
|
|
|
|
updatedAt: now.toISOString(),
|
|
|
|
|
};
|
|
|
|
|
await input.db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
|
exposure,
|
|
|
|
|
exposureHandle: cleanup.status.state === "removed" ? null : row.exposureHandle,
|
|
|
|
|
url: null,
|
|
|
|
|
})
|
|
|
|
|
.where(eq(workspaceRuntimeServices.id, row.id));
|
|
|
|
|
}
|
|
|
|
|
await input.db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
@@ -5000,6 +5290,39 @@ async function markPersistedRuntimeServicesStoppedForExecutionWorkspace(input: {
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function cleanupPersistedExposureRows(
|
|
|
|
|
db: Db,
|
|
|
|
|
rows: Array<{
|
|
|
|
|
id: string;
|
|
|
|
|
exposure: RuntimeExposureStatus | null;
|
|
|
|
|
exposureHandle: string | null;
|
|
|
|
|
}>,
|
|
|
|
|
) {
|
|
|
|
|
for (const row of rows) {
|
|
|
|
|
if (!row.exposure) continue;
|
|
|
|
|
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: row.id,
|
|
|
|
|
handle: row.exposureHandle,
|
|
|
|
|
ports: row.exposure.listeners.map((listener) => listener.targetPort),
|
|
|
|
|
});
|
|
|
|
|
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
|
|
|
|
await db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
|
url: null,
|
|
|
|
|
exposure: {
|
|
|
|
|
...row.exposure,
|
|
|
|
|
state: cleanup.status.state,
|
|
|
|
|
publicUrl: null,
|
|
|
|
|
lastError: cleanup.status.lastError,
|
|
|
|
|
updatedAt: new Date().toISOString(),
|
|
|
|
|
},
|
|
|
|
|
exposureHandle: cleanup.status.state === "removed" ? null : row.exposureHandle,
|
|
|
|
|
})
|
|
|
|
|
.where(eq(workspaceRuntimeServices.id, row.id));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function registerRuntimeService(db: Db | undefined, record: RuntimeServiceRecord) {
|
|
|
|
|
record.db = db;
|
|
|
|
|
runtimeServicesById.set(record.id, record);
|
|
|
|
@@ -5019,8 +5342,11 @@ function registerRuntimeService(db: Db | undefined, record: RuntimeServiceRecord
|
|
|
|
|
if (current.reuseKey && runtimeServicesByReuseKey.get(current.reuseKey) === current.id) {
|
|
|
|
|
runtimeServicesByReuseKey.delete(current.reuseKey);
|
|
|
|
|
}
|
|
|
|
|
void removeLocalServiceRegistryRecord(current.serviceKey);
|
|
|
|
|
void persistRuntimeServiceRecord(db, current);
|
|
|
|
|
void (async () => {
|
|
|
|
|
await cleanupRecordExposure(current);
|
|
|
|
|
await removeLocalServiceRegistryRecord(current.serviceKey);
|
|
|
|
|
await persistRuntimeServiceRecord(db, current);
|
|
|
|
|
})();
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -5436,8 +5762,17 @@ export async function startRuntimeServicesForWorkspaceControl(
|
|
|
|
|
const invocationId = input.invocationId ?? randomUUID();
|
|
|
|
|
const runtimeProvisionCommand = resolveRuntimeProvisionCommand(input);
|
|
|
|
|
const provisionCoordinator = createRuntimeProvisionCoordinator();
|
|
|
|
|
const hasHttpsExposure = rawServices.some((service) => configuredRuntimeExposure(service) !== null);
|
|
|
|
|
|
|
|
|
|
if (rawServices.length === 0 || !input.db || (!input.executionWorkspaceId && !input.workspace.workspaceId)) {
|
|
|
|
|
if (
|
|
|
|
|
rawServices.length === 0
|
|
|
|
|
|| !input.db
|
|
|
|
|
|| (!input.executionWorkspaceId && !input.workspace.workspaceId)
|
|
|
|
|
// The reservation row must commit before the backend binds. Keeping this
|
|
|
|
|
// path outside the parent-row transaction avoids a crash window where the
|
|
|
|
|
// broker lease exists but the DB transaction has not committed its handle.
|
|
|
|
|
|| hasHttpsExposure
|
|
|
|
|
) {
|
|
|
|
|
const batch = await startRuntimeServicesForWorkspaceControlUnlocked(
|
|
|
|
|
input,
|
|
|
|
|
rawServices,
|
|
|
|
@@ -5680,6 +6015,15 @@ export async function stopRuntimeServicesForExecutionWorkspace(input: {
|
|
|
|
|
if (input.db) {
|
|
|
|
|
if (input.runtimeServiceId) {
|
|
|
|
|
const now = new Date();
|
|
|
|
|
const rows = await input.db
|
|
|
|
|
.select({
|
|
|
|
|
id: workspaceRuntimeServices.id,
|
|
|
|
|
exposure: workspaceRuntimeServices.exposure,
|
|
|
|
|
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
|
|
|
|
})
|
|
|
|
|
.from(workspaceRuntimeServices)
|
|
|
|
|
.where(eq(workspaceRuntimeServices.id, input.runtimeServiceId));
|
|
|
|
|
await cleanupPersistedExposureRows(input.db, rows);
|
|
|
|
|
await input.db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
@@ -5717,6 +6061,22 @@ export async function stopRuntimeServicesForProjectWorkspace(input: {
|
|
|
|
|
|
|
|
|
|
if (input.db) {
|
|
|
|
|
const now = new Date();
|
|
|
|
|
const exposureCondition = input.runtimeServiceId
|
|
|
|
|
? eq(workspaceRuntimeServices.id, input.runtimeServiceId)
|
|
|
|
|
: and(
|
|
|
|
|
eq(workspaceRuntimeServices.projectWorkspaceId, input.projectWorkspaceId),
|
|
|
|
|
eq(workspaceRuntimeServices.scopeType, "project_workspace"),
|
|
|
|
|
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
|
|
|
|
);
|
|
|
|
|
const exposureRows = await input.db
|
|
|
|
|
.select({
|
|
|
|
|
id: workspaceRuntimeServices.id,
|
|
|
|
|
exposure: workspaceRuntimeServices.exposure,
|
|
|
|
|
exposureHandle: workspaceRuntimeServices.exposureHandle,
|
|
|
|
|
})
|
|
|
|
|
.from(workspaceRuntimeServices)
|
|
|
|
|
.where(exposureCondition);
|
|
|
|
|
await cleanupPersistedExposureRows(input.db, exposureRows);
|
|
|
|
|
await input.db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
@@ -5727,13 +6087,7 @@ export async function stopRuntimeServicesForProjectWorkspace(input: {
|
|
|
|
|
updatedAt: now,
|
|
|
|
|
})
|
|
|
|
|
.where(
|
|
|
|
|
input.runtimeServiceId
|
|
|
|
|
? eq(workspaceRuntimeServices.id, input.runtimeServiceId)
|
|
|
|
|
: and(
|
|
|
|
|
eq(workspaceRuntimeServices.projectWorkspaceId, input.projectWorkspaceId),
|
|
|
|
|
eq(workspaceRuntimeServices.scopeType, "project_workspace"),
|
|
|
|
|
inArray(workspaceRuntimeServices.status, ["provisioning", "starting", "running"]),
|
|
|
|
|
),
|
|
|
|
|
exposureCondition,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@@ -5777,10 +6131,44 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let ownedExposureListeners: Awaited<ReturnType<BrokerClient["list"]>> | null = [];
|
|
|
|
|
if (rows.some((row) => row.exposure && row.exposure.state !== "removed")) {
|
|
|
|
|
try {
|
|
|
|
|
ownedExposureListeners = await workspaceRuntimeExposureDeps.broker.list();
|
|
|
|
|
} catch {
|
|
|
|
|
ownedExposureListeners = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let reconciled = 0;
|
|
|
|
|
let adopted = 0;
|
|
|
|
|
let stopped = 0;
|
|
|
|
|
for (const row of rows) {
|
|
|
|
|
if (row.status === "stopped" && row.exposure && row.exposure.state !== "removed") {
|
|
|
|
|
await cleanupPersistedExposureRows(db, [{
|
|
|
|
|
id: row.id,
|
|
|
|
|
exposure: row.exposure,
|
|
|
|
|
exposureHandle: row.exposureHandle,
|
|
|
|
|
}]);
|
|
|
|
|
reconciled += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const rowExposureListeners = ownedExposureListeners?.filter((listener) => listener.runtimeId === row.id) ?? [];
|
|
|
|
|
const exposureMappingMatches = !row.exposure || (
|
|
|
|
|
row.exposure.state === "ready"
|
|
|
|
|
&& Boolean(row.exposureHandle)
|
|
|
|
|
&& rowExposureListeners.length === row.exposure.listeners.length
|
|
|
|
|
&& row.exposure.listeners.every((expected) => rowExposureListeners.some((actual) => (
|
|
|
|
|
actual.port === expected.targetPort && actual.purpose === expected.purpose
|
|
|
|
|
)))
|
|
|
|
|
);
|
|
|
|
|
const exposureHealthMatches = !row.exposure || (
|
|
|
|
|
exposureMappingMatches
|
|
|
|
|
&& Boolean(row.exposure.publicUrl)
|
|
|
|
|
&& await workspaceRuntimeExposureDeps.probeHealth(
|
|
|
|
|
new URL("/api/health", row.exposure.publicUrl!).toString(),
|
|
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
let adoptedRecord = await findLocalServiceRegistryRecordByRuntimeServiceId({
|
|
|
|
|
runtimeServiceId: row.id,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
@@ -5820,12 +6208,16 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
cwd: row.cwd,
|
|
|
|
|
envFingerprint: row.reuseKey ?? "",
|
|
|
|
|
port: row.port ?? null,
|
|
|
|
|
url: row.url ?? null,
|
|
|
|
|
url: row.backendUrl ?? row.url ?? null,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
if (adoptedRecord) {
|
|
|
|
|
const adoptedUrl = adoptedRecord.url ?? row.url ?? null;
|
|
|
|
|
if (!(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))) {
|
|
|
|
|
const adoptedUrl = adoptedRecord.url ?? row.backendUrl ?? row.url ?? null;
|
|
|
|
|
if (
|
|
|
|
|
!exposureHealthMatches
|
|
|
|
|
|| !(await isRuntimeServiceUrlHealthy(adoptedUrl, { serviceName: row.serviceName, command: row.command }))
|
|
|
|
|
) {
|
|
|
|
|
await terminateLocalService(adoptedRecord);
|
|
|
|
|
await removeLocalServiceRegistryRecord(adoptedRecord.serviceKey);
|
|
|
|
|
} else {
|
|
|
|
|
const record: RuntimeServiceRecord = {
|
|
|
|
@@ -5844,7 +6236,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
command: row.command ?? null,
|
|
|
|
|
cwd: row.cwd ?? null,
|
|
|
|
|
port: adoptedRecord.port ?? row.port ?? null,
|
|
|
|
|
url: adoptedRecord.url ?? row.url ?? null,
|
|
|
|
|
url: row.exposure?.publicUrl ?? adoptedRecord.url ?? row.url ?? null,
|
|
|
|
|
provider: "local_process",
|
|
|
|
|
providerRef: String(adoptedRecord.pid),
|
|
|
|
|
ownerAgentId: row.ownerAgentId ?? null,
|
|
|
|
@@ -5854,6 +6246,7 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
stoppedAt: null,
|
|
|
|
|
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
|
|
|
|
|
healthStatus: "healthy",
|
|
|
|
|
exposure: row.exposure ?? null,
|
|
|
|
|
reused: true,
|
|
|
|
|
db,
|
|
|
|
|
child: null,
|
|
|
|
@@ -5863,6 +6256,9 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
serviceKey: adoptedRecord.serviceKey,
|
|
|
|
|
profileKind: "workspace-runtime",
|
|
|
|
|
processGroupId: adoptedRecord.processGroupId ?? null,
|
|
|
|
|
exposureHandle: row.exposureHandle ?? null,
|
|
|
|
|
backendUrl: adoptedUrl,
|
|
|
|
|
exposureConfig: null,
|
|
|
|
|
};
|
|
|
|
|
registerRuntimeService(db, record);
|
|
|
|
|
await touchLocalServiceRegistryRecord(adoptedRecord.serviceKey, {
|
|
|
|
@@ -5881,11 +6277,32 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const now = new Date();
|
|
|
|
|
let stoppedExposure = row.exposure ?? null;
|
|
|
|
|
let stoppedExposureHandle = row.exposureHandle ?? null;
|
|
|
|
|
if (stoppedExposure) {
|
|
|
|
|
const cleanup = await deprovisionExposure(workspaceRuntimeExposureDeps, {
|
|
|
|
|
runtimeId: row.id,
|
|
|
|
|
handle: stoppedExposureHandle,
|
|
|
|
|
ports: stoppedExposure.listeners.map((listener) => listener.targetPort),
|
|
|
|
|
});
|
|
|
|
|
for (const port of cleanup.quarantinedPorts) quarantinedRuntimeExposurePorts.add(port);
|
|
|
|
|
stoppedExposure = {
|
|
|
|
|
...stoppedExposure,
|
|
|
|
|
state: cleanup.status.state,
|
|
|
|
|
publicUrl: null,
|
|
|
|
|
lastError: cleanup.status.lastError,
|
|
|
|
|
updatedAt: now.toISOString(),
|
|
|
|
|
};
|
|
|
|
|
if (cleanup.status.state === "removed") stoppedExposureHandle = null;
|
|
|
|
|
}
|
|
|
|
|
await db
|
|
|
|
|
.update(workspaceRuntimeServices)
|
|
|
|
|
.set({
|
|
|
|
|
status: "stopped",
|
|
|
|
|
healthStatus: "unknown",
|
|
|
|
|
url: stoppedExposure ? null : row.url,
|
|
|
|
|
exposure: stoppedExposure,
|
|
|
|
|
exposureHandle: stoppedExposureHandle,
|
|
|
|
|
stoppedAt: now,
|
|
|
|
|
lastUsedAt: now,
|
|
|
|
|
updatedAt: now,
|
|
|
|
@@ -6073,6 +6490,9 @@ export async function persistAdapterManagedRuntimeServices(input: {
|
|
|
|
|
startedAt,
|
|
|
|
|
stoppedAt: ref.stoppedAt ? new Date(ref.stoppedAt) : null,
|
|
|
|
|
stopPolicy: ref.stopPolicy,
|
|
|
|
|
exposure: null,
|
|
|
|
|
exposureHandle: null,
|
|
|
|
|
backendUrl: null,
|
|
|
|
|
healthStatus: ref.healthStatus,
|
|
|
|
|
createdAt,
|
|
|
|
|
updatedAt: new Date(),
|
|
|
|
@@ -6102,6 +6522,9 @@ export async function persistAdapterManagedRuntimeServices(input: {
|
|
|
|
|
startedAt,
|
|
|
|
|
stoppedAt: ref.stoppedAt ? new Date(ref.stoppedAt) : null,
|
|
|
|
|
stopPolicy: ref.stopPolicy,
|
|
|
|
|
exposure: null,
|
|
|
|
|
exposureHandle: null,
|
|
|
|
|
backendUrl: null,
|
|
|
|
|
healthStatus: ref.healthStatus,
|
|
|
|
|
updatedAt: new Date(),
|
|
|
|
|
},
|
|
|
|
|