fix(runtime): repair database runtime projections

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-14 07:17:17 +00:00
1 parent 54b4e0c2fb
commit db2893438d
16 files changed
+1155 -92

No files matched your search

+5
View File
@@ -844,6 +844,7 @@ export type {
ExecutionWorkspaceCloseReadiness,
ExecutionWorkspaceCloseReadinessState,
ExecutionWorkspaceDeliveryState,
EffectiveWorkspaceRuntimeConfig,
WorkspaceOverviewItem,
WorkspaceOverviewLinkedIssue,
WorkspaceOverviewPrimaryService,
@@ -852,6 +853,10 @@ export type {
WorkspaceCommandDefinition,
WorkspaceCommandKind,
WorkspaceRuntimeControlTarget,
WorkspaceRuntimeActualState,
WorkspaceRuntimeConfigSource,
WorkspaceRuntimeFailureDetails,
WorkspaceRuntimeFailureEvidence,
WorkspaceRuntimeService,
WorkspaceRuntimeServiceStateMap,
WorkspaceOperation,
+5
View File
@@ -366,6 +366,7 @@ export type {
ExecutionWorkspaceCloseReadiness,
ExecutionWorkspaceCloseReadinessState,
ExecutionWorkspaceDeliveryState,
EffectiveWorkspaceRuntimeConfig,
WorkspaceOverviewItem,
WorkspaceOverviewLinkedIssue,
WorkspaceOverviewPrimaryService,
@@ -374,6 +375,10 @@ export type {
WorkspaceCommandDefinition,
WorkspaceCommandKind,
WorkspaceRuntimeControlTarget,
WorkspaceRuntimeActualState,
WorkspaceRuntimeConfigSource,
WorkspaceRuntimeFailureDetails,
WorkspaceRuntimeFailureEvidence,
WorkspaceRuntimeService,
WorkspaceRuntimeServiceStateMap,
WorkspaceRuntimeDesiredState,
+42 -1
View File
@@ -58,6 +58,41 @@ export type ExecutionWorkspaceCloseActionKind =
export type WorkspaceRuntimeDesiredState = "running" | "stopped" | "manual";
export type WorkspaceRuntimeServiceStateMap = Record<string, WorkspaceRuntimeDesiredState>;
export type WorkspaceCommandKind = "service" | "job";
export type WorkspaceRuntimeActualState =
| "provisioning"
| "starting"
| "running"
| "stopped"
| "failed";
export interface WorkspaceRuntimeConfigSource {
type: "execution_workspace" | "project_workspace";
id: string;
}
export interface EffectiveWorkspaceRuntimeConfig {
workspaceRuntime: Record<string, unknown>;
source: WorkspaceRuntimeConfigSource;
desiredState: WorkspaceRuntimeDesiredState | null;
serviceStates: WorkspaceRuntimeServiceStateMap | null;
}
export interface WorkspaceRuntimeFailureDetails {
port?: number;
attemptedPortCount?: number;
conflictingExecutionWorkspaceId?: string;
conflictingProjectWorkspaceId?: string;
}
export interface WorkspaceRuntimeFailureEvidence {
operationId: string;
operationLogPath: string;
code: string;
message: string;
remediation: string;
details: WorkspaceRuntimeFailureDetails | null;
failedAt: Date;
}
export interface WorkspaceCommandSource {
type: "paperclip";
@@ -273,6 +308,7 @@ export interface ExecutionWorkspace {
cleanupEligibleAt: Date | null;
cleanupReason: string | null;
config: ExecutionWorkspaceConfig | null;
effectiveRuntimeConfig: EffectiveWorkspaceRuntimeConfig | null;
metadata: Record<string, unknown> | null;
runtimeServices?: WorkspaceRuntimeService[];
createdAt: Date;
@@ -289,7 +325,10 @@ export interface WorkspaceRuntimeService {
scopeType: "project_workspace" | "execution_workspace" | "run" | "agent";
scopeId: string | null;
serviceName: string;
status: "provisioning" | "starting" | "running" | "stopped" | "failed";
/** @deprecated Prefer actualState when comparing runtime intent with observed state. */
status: WorkspaceRuntimeActualState;
actualState: WorkspaceRuntimeActualState;
desiredState: WorkspaceRuntimeDesiredState | null;
lifecycle: "shared" | "ephemeral";
reuseKey: string | null;
command: string | null;
@@ -306,6 +345,8 @@ export interface WorkspaceRuntimeService {
stopPolicy: Record<string, unknown> | null;
healthStatus: "unknown" | "healthy" | "unhealthy";
configIndex?: number | null;
workspaceCommandId?: string | null;
latestFailure: WorkspaceRuntimeFailureEvidence | null;
createdAt: Date;
updatedAt: Date;
}
@@ -110,6 +110,8 @@ export const workspaceRuntimeServiceSchema = z.object({
scopeId: z.string().nullable(),
serviceName: z.string(),
status: z.enum(["provisioning", "starting", "running", "stopped", "failed"]),
actualState: z.enum(["provisioning", "starting", "running", "stopped", "failed"]),
desiredState: z.enum(["running", "stopped", "manual"]).nullable(),
lifecycle: z.enum(["shared", "ephemeral"]),
reuseKey: z.string().nullable(),
command: z.string().nullable(),
@@ -126,6 +128,21 @@ export const workspaceRuntimeServiceSchema = z.object({
stopPolicy: z.record(z.string(), z.unknown()).nullable(),
healthStatus: z.enum(["unknown", "healthy", "unhealthy"]),
configIndex: z.number().int().nonnegative().nullable().optional(),
workspaceCommandId: z.string().nullable().optional(),
latestFailure: z.object({
operationId: z.string().uuid(),
operationLogPath: z.string(),
code: z.string(),
message: z.string(),
remediation: z.string(),
details: z.object({
port: z.number().int().optional(),
attemptedPortCount: z.number().int().nonnegative().optional(),
conflictingExecutionWorkspaceId: z.string().uuid().optional(),
conflictingProjectWorkspaceId: z.string().uuid().optional(),
}).strict().nullable(),
failedAt: z.coerce.date(),
}).strict().nullable(),
createdAt: z.coerce.date(),
updatedAt: z.coerce.date(),
}).strict();
@@ -2,7 +2,9 @@ import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { errorHandler } from "../middleware/index.js";
import { conflict } from "../errors.js";
import { executionWorkspaceRoutes } from "../routes/execution-workspaces.js";
import { attachWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js";
const mockExecutionWorkspaceService = vi.hoisted(() => ({
list: vi.fn(),
@@ -61,6 +63,10 @@ vi.mock("../services/workspace-runtime.js", async (importActual) => {
};
});
vi.mock("../routes/workspace-runtime-authz.js", () => ({
assertCanManageExecutionWorkspaceRuntimeServices: vi.fn(async () => undefined),
}));
function createApp(actor: Record<string, unknown> = {
type: "board",
userId: "local-board",
@@ -107,6 +113,9 @@ describe.sequential("execution workspace routes", () => {
]);
mockExecutionWorkspaceService.getById.mockResolvedValue(null);
mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue(null);
mockWorkspaceOperationService.createRecorder.mockReturnValue({
recordOperation: vi.fn(),
});
mockHeartbeatService.wakeup.mockResolvedValue(null);
});
@@ -161,6 +170,141 @@ describe.sequential("execution workspace routes", () => {
expect(mockExecutionWorkspaceService.listOverview).not.toHaveBeenCalled();
});
it("returns effective inherited runtime config and the current failed service row", async () => {
mockExecutionWorkspaceService.getById.mockResolvedValue({
id: "workspace-1",
companyId: "company-1",
projectWorkspaceId: "project-workspace-1",
effectiveRuntimeConfig: {
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
source: { type: "project_workspace", id: "project-workspace-1" },
desiredState: "running",
serviceStates: null,
},
runtimeServices: [{
id: "service-1",
serviceName: "web",
status: "failed",
actualState: "failed",
desiredState: "running",
latestFailure: {
operationId: "operation-1",
operationLogPath: "/api/workspace-operations/operation-1/log",
code: "workspace_runtime_start_failed",
message: "Workspace runtime service failed to start.",
remediation: "Review the workspace operation log and retry.",
details: null,
failedAt: new Date("2026-08-11T12:00:00.000Z"),
},
}],
});
const res = await request(createApp()).get("/api/execution-workspaces/workspace-1");
expect(res.status).toBe(200);
expect(res.body.effectiveRuntimeConfig.source).toEqual({
type: "project_workspace",
id: "project-workspace-1",
});
expect(res.body.runtimeServices).toEqual([
expect.objectContaining({
id: "service-1",
actualState: "failed",
desiredState: "running",
latestFailure: expect.objectContaining({ operationId: "operation-1" }),
}),
]);
});
it("does not expose workspace detail across company scope", async () => {
mockExecutionWorkspaceService.getById.mockResolvedValue({
id: "workspace-1",
companyId: "company-1",
});
const res = await request(createApp({
type: "board",
userId: "other-board",
companyIds: ["company-2"],
source: "session",
isInstanceAdmin: false,
})).get("/api/execution-workspaces/workspace-1");
expect(res.status).toBe(404);
});
it("returns a safe operation reference when runtime start fails", async () => {
mockExecutionWorkspaceService.getById.mockResolvedValue({
id: "workspace-1",
companyId: "company-1",
projectId: null,
projectWorkspaceId: null,
sourceIssueId: null,
name: "Runtime failure workspace",
mode: "isolated_workspace",
strategyType: "git_worktree",
cwd: "/tmp/runtime-failure-workspace",
repoUrl: null,
baseRef: "master",
branchName: "runtime-failure",
providerType: "git_worktree",
providerRef: "/tmp/runtime-failure-workspace",
config: {
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
desiredState: "running",
serviceStates: null,
},
effectiveRuntimeConfig: {
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
source: { type: "execution_workspace", id: "workspace-1" },
desiredState: "running",
serviceStates: null,
},
runtimeServices: [],
metadata: null,
});
const operationError = attachWorkspaceOperationFailureEvidence(
conflict("No safe automatically allocated runtime service port is available.", {
code: "workspace_runtime_port_allocation_exhausted",
attemptedPortCount: 10,
cwd: "/secret/workspace/path",
remediation: "Configure a different runtime service port.",
}),
{
operationId: "operation-1",
operationLogPath: "/api/workspace-operations/operation-1/log",
code: "workspace_runtime_port_allocation_exhausted",
message: "No safe automatically allocated runtime service port is available.",
remediation: "Configure a different runtime service port.",
details: { attemptedPortCount: 10 },
failedAt: "2026-08-11T12:00:00.000Z",
},
);
mockWorkspaceOperationService.createRecorder.mockReturnValue({
recordOperation: vi.fn(async () => {
throw operationError;
}),
});
const res = await request(createApp())
.post("/api/execution-workspaces/workspace-1/runtime-services/start")
.send({ workspaceCommandId: "service:web" });
expect(res.status).toBe(409);
expect(res.body).toMatchObject({
error: "No safe automatically allocated runtime service port is available.",
details: {
code: "workspace_runtime_port_allocation_exhausted",
attemptedPortCount: 10,
remediation: "Configure a different runtime service port.",
operationId: "operation-1",
operationLogPath: "/api/workspace-operations/operation-1/log",
failedAt: "2026-08-11T12:00:00.000Z",
},
});
expect(JSON.stringify(res.body)).not.toContain("/secret/workspace/path");
});
it.each([
["forward", { mode: "forward" }],
["override", { mode: "override", reason: "operator break-glass" }],
@@ -21,6 +21,7 @@ import {
issues,
projectWorkspaces,
projects,
workspaceOperations,
workspaceRuntimeServices,
} from "@paperclipai/db";
import {
@@ -39,6 +40,11 @@ import {
readMetadataReopenPendingConsumptionSince,
} from "../services/execution-workspaces.ts";
import { issueService } from "../services/issues.ts";
import { conflict } from "../errors.ts";
import {
getWorkspaceOperationFailureEvidence,
workspaceOperationService,
} from "../services/workspace-operations.ts";
import {
startRuntimeServicesForWorkspaceControl,
stopRuntimeServicesForExecutionWorkspace,
@@ -287,6 +293,73 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
await tempDb?.cleanup();
});
it("persists safe structured evidence and attaches its exact operation reference", async () => {
const companyId = randomUUID();
const logDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-operation-log-"));
tempDirs.add(logDir);
const previousLogBasePath = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = logDir;
await db.insert(companies).values({
id: companyId,
name: "Paperclip",
issuePrefix: "PAP",
requireBoardApprovalForNewAgents: false,
});
let thrown: unknown;
try {
await workspaceOperationService(db).createRecorder({ companyId }).recordOperation({
phase: "workspace_provision",
command: "pnpm dev",
metadata: { action: "start", serviceIndex: 0, workspaceCommandId: "service:web" },
run: async () => {
throw conflict("No safe runtime port is available.", {
code: "workspace_runtime_port_allocation_exhausted",
port: 3100,
cwd: "/secret/workspace/path",
remediation: "Configure a different runtime service port.",
});
},
});
} catch (error) {
thrown = error;
} finally {
if (previousLogBasePath === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = previousLogBasePath;
}
const evidence = getWorkspaceOperationFailureEvidence(thrown);
expect(evidence).toMatchObject({
operationLogPath: `/api/workspace-operations/${evidence?.operationId}/log`,
code: "workspace_runtime_port_allocation_exhausted",
message: "No safe runtime port is available.",
remediation: "Configure a different runtime service port.",
details: { port: 3100 },
});
expect(evidence?.details).not.toHaveProperty("cwd");
const [operation] = await db
.select()
.from(workspaceOperations)
.where(eq(workspaceOperations.id, evidence!.operationId));
expect(operation).toMatchObject({
status: "failed",
metadata: expect.objectContaining({
action: "start",
serviceIndex: 0,
workspaceCommandId: "service:web",
failureEvidence: expect.objectContaining({
operationId: evidence!.operationId,
operationLogPath: evidence!.operationLogPath,
code: "workspace_runtime_port_allocation_exhausted",
details: { port: 3100 },
}),
}),
});
expect(JSON.stringify(operation?.metadata)).not.toContain("/secret/workspace/path");
});
async function seedTerminalWorkspace(options: {
mergedPr?: boolean;
activeRun?: boolean;
@@ -3922,16 +3995,31 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
]);
const [workspace] = await svc.list(companyId);
expect(workspace?.effectiveRuntimeConfig).toEqual({
workspaceRuntime: {
services: [
{ name: "web", command: "pnpm dev" },
{ name: "worker", command: "pnpm worker", reuseScope: "execution_workspace" },
],
},
source: { type: "project_workspace", id: projectWorkspaceId },
desiredState: "stopped",
serviceStates: null,
});
expect(workspace?.runtimeServices).toEqual([
expect.objectContaining({
id: currentWebServiceId,
serviceName: "web",
configIndex: 0,
actualState: "stopped",
desiredState: "stopped",
}),
expect.objectContaining({
id: currentWorkerServiceId,
serviceName: "worker",
configIndex: 1,
actualState: "running",
desiredState: "stopped",
}),
]);
@@ -3948,6 +4036,182 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
});
});
it("projects one failed configured row with inherited config and company-scoped failure evidence", async () => {
const companyId = randomUUID();
const otherCompanyId = randomUUID();
const projectId = randomUUID();
const projectWorkspaceId = randomUUID();
const executionWorkspaceId = randomUUID();
const failedServiceId = randomUUID();
const failedOperationId = randomUUID();
const failedAt = new Date("2026-08-11T14:00:00.000Z");
await db.insert(companies).values([
{
id: companyId,
name: "Paperclip",
issuePrefix: "PAP",
requireBoardApprovalForNewAgents: false,
},
{
id: otherCompanyId,
name: "OtherCo",
issuePrefix: "OTH",
requireBoardApprovalForNewAgents: false,
},
]);
await db.insert(projects).values({
id: projectId,
companyId,
name: "Inherited runtime projection",
status: "in_progress",
});
await db.insert(projectWorkspaces).values({
id: projectWorkspaceId,
companyId,
projectId,
name: "Primary",
sourceType: "local_path",
isPrimary: true,
cwd: "/tmp/inherited-runtime-projection",
metadata: {
runtimeConfig: {
workspaceRuntime: {
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
},
desiredState: "stopped",
},
},
});
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
projectWorkspaceId,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Inherited isolated workspace",
status: "idle",
providerType: "git_worktree",
cwd: "/tmp/inherited-runtime-projection/isolated",
metadata: {
config: {
desiredState: "running",
serviceStates: { "0": "manual" },
},
},
});
await db.insert(workspaceRuntimeServices).values([
{
id: randomUUID(),
companyId,
projectId,
projectWorkspaceId,
executionWorkspaceId,
scopeType: "execution_workspace",
scopeId: executionWorkspaceId,
serviceName: "web",
status: "stopped",
lifecycle: "shared",
reuseKey: "inherited-web",
command: "pnpm dev",
cwd: "/tmp/inherited-runtime-projection/isolated",
provider: "local_process",
healthStatus: "unknown",
updatedAt: new Date("2026-08-10T14:00:00.000Z"),
},
{
id: failedServiceId,
companyId,
projectId,
projectWorkspaceId,
executionWorkspaceId,
scopeType: "execution_workspace",
scopeId: executionWorkspaceId,
serviceName: "web",
status: "failed",
lifecycle: "shared",
reuseKey: "inherited-web",
command: "pnpm dev",
cwd: "/tmp/inherited-runtime-projection/isolated",
provider: "local_process",
healthStatus: "unhealthy",
updatedAt: failedAt,
},
]);
await db.insert(workspaceOperations).values([
{
id: failedOperationId,
companyId,
executionWorkspaceId,
phase: "workspace_provision",
command: "pnpm dev",
status: "failed",
metadata: {
action: "start",
serviceIndex: 0,
workspaceCommandId: "service:web",
failureEvidence: {
code: "workspace_runtime_start_failed",
message: "Workspace runtime service failed to start.",
remediation: "Review the workspace operation log and retry.",
details: { port: 3100 },
},
},
startedAt: failedAt,
finishedAt: failedAt,
},
{
companyId: otherCompanyId,
executionWorkspaceId,
phase: "workspace_provision",
command: "pnpm dev",
status: "failed",
metadata: {
action: "start",
serviceIndex: 0,
workspaceCommandId: "service:web",
failureEvidence: {
code: "other_company_failure",
message: "Must not leak.",
remediation: "Must not leak.",
},
},
startedAt: new Date("2026-08-11T15:00:00.000Z"),
finishedAt: new Date("2026-08-11T15:00:00.000Z"),
},
]);
const workspace = await svc.getById(executionWorkspaceId);
expect(workspace?.effectiveRuntimeConfig).toEqual({
workspaceRuntime: {
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
},
source: { type: "project_workspace", id: projectWorkspaceId },
desiredState: "running",
serviceStates: { "0": "manual" },
});
expect(workspace?.runtimeServices).toEqual([
expect.objectContaining({
id: failedServiceId,
status: "failed",
actualState: "failed",
desiredState: "manual",
workspaceCommandId: "service:web",
latestFailure: {
operationId: failedOperationId,
operationLogPath: `/api/workspace-operations/${failedOperationId}/log`,
code: "workspace_runtime_start_failed",
message: "Workspace runtime service failed to start.",
remediation: "Review the workspace operation log and retry.",
details: { port: 3100 },
failedAt,
},
}),
]);
});
it("returns a bounded company-scoped workspace overview with service and linked issue summaries", async () => {
const companyId = randomUUID();
const otherCompanyId = randomUUID();
@@ -4012,7 +4276,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
metadata: {
config: {
workspaceRuntime: {
services: [{ name: "web", command: "pnpm dev" }],
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
},
},
},
@@ -4148,7 +4412,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
projectUrlKey: "workspaces",
projectName: "Workspaces",
branchName: "paperclip/a",
serviceCount: 2,
serviceCount: 1,
runningServiceCount: 1,
primaryServiceUrl: "http://localhost:3100",
primaryServiceUrlRunning: true,
@@ -298,6 +298,8 @@ describe.sequential("issue goal context routes", () => {
scopeType: "execution_workspace",
scopeId: workspaceId,
serviceName: "Dev server",
actualState: "stopped",
desiredState: "running",
lifecycle: "shared",
reuseKey: "dev-server",
command: "pnpm dev",
@@ -313,6 +315,8 @@ describe.sequential("issue goal context routes", () => {
stoppedAt: null,
healthStatus: "healthy",
configIndex: 0,
workspaceCommandId: "service:dev-server",
latestFailure: null,
metadata: { huge: "runtime metadata should not be embedded" },
};
mockIssueService.getById.mockResolvedValueOnce({
@@ -402,10 +406,21 @@ describe.sequential("issue goal context routes", () => {
desiredState: null,
serviceStates: null,
},
effectiveRuntimeConfig: {
workspaceRuntime: { services: [{ name: "Dev server", command: "pnpm dev" }] },
source: { type: "execution_workspace", id: workspaceId },
desiredState: "running",
serviceStates: null,
},
metadata: { huge: "execution workspace metadata should not be embedded" },
runtimeServices: [
{ ...runtimeServiceBase, id: "service-running", status: "running" },
{ ...runtimeServiceBase, id: "service-stopped", status: "stopped" },
{
...runtimeServiceBase,
id: "service-failed",
status: "failed",
actualState: "failed",
healthStatus: "unhealthy",
},
],
createdAt: new Date("2026-03-24T12:00:00Z"),
updatedAt: new Date("2026-03-24T12:00:00Z"),
@@ -422,10 +437,16 @@ describe.sequential("issue goal context routes", () => {
expect(res.body.currentExecutionWorkspace.metadata).toBeNull();
expect(res.body.currentExecutionWorkspace.runtimeServices).toHaveLength(1);
expect(res.body.currentExecutionWorkspace.runtimeServices[0]).toMatchObject({
id: "service-running",
status: "running",
id: "service-failed",
status: "failed",
actualState: "failed",
desiredState: "running",
url: "http://localhost:3100",
});
expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({
type: "execution_workspace",
id: workspaceId,
});
expect(res.body.currentExecutionWorkspace.runtimeServices[0]).not.toHaveProperty("metadata");
});
@@ -512,13 +533,31 @@ describe.sequential("issue goal context routes", () => {
mode: "isolated_workspace",
status: "active",
cwd: "/tmp/pap-581",
effectiveRuntimeConfig: {
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
source: { type: "project_workspace", id: "project-workspace-1" },
desiredState: "running",
serviceStates: null,
},
runtimeServices: [
{
id: "service-1",
serviceName: "web",
status: "running",
url: "http://127.0.0.1:5173",
healthStatus: "healthy",
status: "failed",
actualState: "failed",
desiredState: "running",
url: null,
healthStatus: "unhealthy",
workspaceCommandId: "service:web",
latestFailure: {
operationId: "operation-1",
operationLogPath: "/api/workspace-operations/operation-1/log",
code: "workspace_runtime_start_failed",
message: "Workspace runtime service failed to start.",
remediation: "Review the workspace operation log and retry.",
details: null,
failedAt: new Date("2026-08-11T12:00:00.000Z"),
},
},
],
});
@@ -535,9 +574,15 @@ describe.sequential("issue goal context routes", () => {
runtimeServices: [
expect.objectContaining({
serviceName: "web",
url: "http://127.0.0.1:5173",
actualState: "failed",
desiredState: "running",
latestFailure: expect.objectContaining({ operationId: "operation-1" }),
}),
],
}));
expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({
type: "project_workspace",
id: "project-workspace-1",
});
});
});
@@ -5981,6 +5981,70 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
}
}, 20_000);
it("keeps one configured row failed after repeated readiness failures", async () => {
const fixture = await createRuntimeFixture();
const cleanupRuntimeHome = await createRuntimeHome();
const workspace = fixture.workspaces[0]!;
const port = await findFreePort();
const command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`;
const config = fixedPortRuntimeConfig(port, command);
try {
await expect(startRuntimeServicesForWorkspaceControl({
db,
actor: fixture.actor,
issue: null,
workspace: fixture.realizedWorkspace(workspace),
executionWorkspaceId: workspace.id,
config,
adapterEnv: {},
})).rejects.toThrow(/failed to start runtime service|exited before readiness/i);
const firstRows = await db
.select()
.from(workspaceRuntimeServices)
.where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id));
expect(firstRows).toEqual([
expect.objectContaining({
serviceName: "web",
command,
status: "failed",
healthStatus: "unhealthy",
}),
]);
await expect(startRuntimeServicesForWorkspaceControl({
db,
actor: fixture.actor,
issue: null,
workspace: fixture.realizedWorkspace(workspace),
executionWorkspaceId: workspace.id,
config,
adapterEnv: {},
})).rejects.toThrow(/failed to start runtime service|exited before readiness/i);
const retryRows = await db
.select()
.from(workspaceRuntimeServices)
.where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id));
expect(retryRows).toEqual([
expect.objectContaining({
id: firstRows[0]!.id,
status: "failed",
healthStatus: "unhealthy",
}),
]);
} finally {
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: workspace.id,
workspaceCwd: workspace.cwd,
}).catch(() => undefined);
await cleanupRuntimeHome();
await fixture.cleanup();
}
}, 20_000);
it("returns a bounded structured conflict and exposes only same-company workspace references", async () => {
const fixture = await createRuntimeFixture({
workspaceModes: ["isolated_workspace", "isolated_workspace"],
+22
View File
@@ -38,9 +38,29 @@ import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-ru
import { appendWithCap } from "../adapters/utils.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { HttpError } from "../errors.js";
import { getWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js";
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
function toWorkspaceRuntimeOperationHttpError(error: unknown) {
const evidence = getWorkspaceOperationFailureEvidence(error);
if (!evidence) return error;
return new HttpError(
error instanceof HttpError ? error.status : 500,
error instanceof HttpError ? error.message : evidence.message,
{
code: evidence.code,
remediation: evidence.remediation,
operationId: evidence.operationId,
operationLogPath: evidence.operationLogPath,
failedAt: evidence.failedAt,
...(evidence.details ?? {}),
},
);
}
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
const router = Router();
const svc = executionWorkspaceService(db);
@@ -456,6 +476,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
},
};
},
}).catch((error) => {
throw toWorkspaceRuntimeOperationHttpError(error);
});
const workspace = await svc.getById(id);
+6 -5
View File
@@ -5303,6 +5303,8 @@ export function issueRoutes(
scopeId: service.scopeId,
serviceName: service.serviceName,
status: service.status,
actualState: service.actualState,
desiredState: service.desiredState,
lifecycle: service.lifecycle,
reuseKey: service.reuseKey,
command: service.command,
@@ -5318,6 +5320,8 @@ export function issueRoutes(
stoppedAt: service.stoppedAt,
healthStatus: service.healthStatus,
configIndex: service.configIndex ?? null,
workspaceCommandId: service.workspaceCommandId ?? null,
latestFailure: service.latestFailure,
};
}
@@ -5358,12 +5362,9 @@ export function issueRoutes(
serviceStates: workspace.config.serviceStates,
}
: null,
effectiveRuntimeConfig: workspace.effectiveRuntimeConfig,
metadata: null,
runtimeServices: (workspace.runtimeServices ?? [])
.filter((service) =>
service.status === "provisioning" || service.status === "starting" || service.status === "running"
)
.map(compactIssueRuntimeService),
runtimeServices: (workspace.runtimeServices ?? []).map(compactIssueRuntimeService),
createdAt: workspace.createdAt,
updatedAt: workspace.updatedAt,
};
+286 -48
View File
@@ -13,9 +13,11 @@ import {
issues,
projects,
projectWorkspaces,
workspaceOperations,
workspaceRuntimeServices,
} from "@paperclipai/db";
import type {
EffectiveWorkspaceRuntimeConfig,
ExecutionWorkspace,
ExecutionWorkspaceDeliveryState,
ExecutionWorkspaceSummary,
@@ -27,6 +29,7 @@ import type {
WorkspaceOverviewItem,
WorkspaceOverviewLinkedIssue,
WorkspaceRuntimeDesiredState,
WorkspaceRuntimeFailureEvidence,
WorkspaceRuntimeService,
WorkspaceOverviewPrimaryService,
WorkspaceOverviewQuery,
@@ -61,10 +64,22 @@ import {
type ExecutionWorkspaceRow = typeof executionWorkspaces.$inferSelect;
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
type RuntimeServiceReadDb = Pick<Db, "select">;
type RuntimeServiceReadDb = Pick<Db, "select" | "selectDistinctOn">;
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & {
configIndex?: number | null;
workspaceCommandId?: string | null;
desiredState?: WorkspaceRuntimeDesiredState | null;
latestFailure?: WorkspaceRuntimeFailureEvidence | null;
};
type ProjectedWorkspaceRuntimeFailure = {
serviceIndex: number | null;
workspaceCommandId: string | null;
evidence: WorkspaceRuntimeFailureEvidence;
};
const execFileAsync = promisify(execFile);
const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]);
const ACTIVE_RUNTIME_SERVICE_STATUSES = new Set(["provisioning", "starting", "running"]);
const WORKSPACE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence";
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
export const ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON = "issue_terminal";
@@ -627,7 +642,9 @@ function assertBranchReconcileRuntimeServicesStopped(input: {
inspection: ExecutionWorkspaceBranchReconcileInspection;
runtimeServices: WorkspaceRuntimeService[];
}) {
const activeRuntimeServices = input.runtimeServices.filter((service) => service.status !== "stopped");
const activeRuntimeServices = input.runtimeServices.filter((service) =>
ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState)
);
if (activeRuntimeServices.length > 0) {
throw unprocessable("Execution workspace branch reconciliation requires all runtime services to be stopped", {
inspection: input.inspection,
@@ -952,8 +969,9 @@ export function mergeExecutionWorkspaceConfig(
}
function toRuntimeService(
row: WorkspaceRuntimeServiceRow & { configIndex?: number | null },
row: ProjectedWorkspaceRuntimeServiceRow,
): WorkspaceRuntimeService {
const actualState = row.status as WorkspaceRuntimeService["actualState"];
return {
id: row.id,
companyId: row.companyId,
@@ -964,7 +982,9 @@ function toRuntimeService(
scopeType: row.scopeType as WorkspaceRuntimeService["scopeType"],
scopeId: row.scopeId ?? null,
serviceName: row.serviceName,
status: row.status as WorkspaceRuntimeService["status"],
status: actualState,
actualState,
desiredState: row.desiredState ?? null,
lifecycle: row.lifecycle as WorkspaceRuntimeService["lifecycle"],
reuseKey: row.reuseKey ?? null,
command: row.command ?? null,
@@ -981,6 +1001,8 @@ function toRuntimeService(
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"],
configIndex: row.configIndex ?? null,
workspaceCommandId: row.workspaceCommandId ?? null,
latestFailure: row.latestFailure ?? null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
@@ -990,7 +1012,9 @@ function toExecutionWorkspace(
row: ExecutionWorkspaceRow,
runtimeServices: WorkspaceRuntimeService[] = [],
deliveryState: ExecutionWorkspaceDeliveryState = "unknown",
effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null,
): ExecutionWorkspace {
const config = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
return {
id: row.id,
companyId: row.companyId,
@@ -1014,7 +1038,17 @@ function toExecutionWorkspace(
closedAt: row.closedAt ?? null,
cleanupEligibleAt: row.cleanupEligibleAt ?? null,
cleanupReason: row.cleanupReason ?? null,
config: readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null),
config,
effectiveRuntimeConfig: effectiveRuntimeConfig === undefined
? config?.workspaceRuntime
? {
workspaceRuntime: config.workspaceRuntime,
source: { type: "execution_workspace", id: row.id },
desiredState: config.desiredState,
serviceStates: config.serviceStates ?? null,
}
: null
: effectiveRuntimeConfig,
metadata: (row.metadata as Record<string, unknown> | null) ?? null,
runtimeServices,
createdAt: row.createdAt,
@@ -1075,6 +1109,11 @@ function usesInheritedProjectRuntimeServices(row: ExecutionWorkspaceRow) {
return !readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime;
}
function inheritsProjectRuntimeConfig(row: ExecutionWorkspaceRow) {
if (!row.projectWorkspaceId) return false;
return !readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime;
}
function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow) {
const inheritedProjectWorkspaceId = usesInheritedProjectRuntimeServices(row) ? row.projectWorkspaceId : null;
const activeServiceConditions = inheritedProjectWorkspaceId
@@ -1087,22 +1126,124 @@ function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow
),
eq(workspaceRuntimeServices.executionWorkspaceId, row.id),
),
ne(workspaceRuntimeServices.status, "stopped"),
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]),
)
: and(
eq(workspaceRuntimeServices.companyId, row.companyId),
eq(workspaceRuntimeServices.executionWorkspaceId, row.id),
ne(workspaceRuntimeServices.status, "stopped"),
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]),
);
return sql`not exists (select 1 from ${workspaceRuntimeServices} where ${activeServiceConditions})`;
}
function readWorkspaceRuntimeFailureEvidence(
operation: {
id: string;
metadata: Record<string, unknown> | null;
startedAt: Date;
finishedAt: Date | null;
},
): WorkspaceRuntimeFailureEvidence {
const raw = isRecord(operation.metadata?.failureEvidence)
? operation.metadata.failureEvidence
: null;
const rawDetails = isRecord(raw?.details) ? raw.details : null;
const details: NonNullable<WorkspaceRuntimeFailureEvidence["details"]> = {};
if (typeof rawDetails?.port === "number" && Number.isInteger(rawDetails.port)) details.port = rawDetails.port;
if (typeof rawDetails?.attemptedPortCount === "number" && Number.isInteger(rawDetails.attemptedPortCount)) {
details.attemptedPortCount = rawDetails.attemptedPortCount;
}
if (typeof rawDetails?.conflictingExecutionWorkspaceId === "string") {
details.conflictingExecutionWorkspaceId = rawDetails.conflictingExecutionWorkspaceId;
}
if (typeof rawDetails?.conflictingProjectWorkspaceId === "string") {
details.conflictingProjectWorkspaceId = rawDetails.conflictingProjectWorkspaceId;
}
return {
operationId: operation.id,
operationLogPath: `/api/workspace-operations/${operation.id}/log`,
code: typeof raw?.code === "string" ? raw.code : "workspace_runtime_start_failed",
message: typeof raw?.message === "string" ? raw.message : "Workspace runtime service failed to start.",
remediation: typeof raw?.remediation === "string"
? raw.remediation
: "Review the workspace operation log, correct the runtime configuration or dependency, and retry.",
details: Object.keys(details).length > 0 ? details : null,
failedAt: operation.finishedAt ?? operation.startedAt,
};
}
async function loadLatestRuntimeFailuresByExecutionWorkspace(
db: RuntimeServiceReadDb,
companyId: string,
runtimeRowsByWorkspaceId: Map<string, ProjectedWorkspaceRuntimeServiceRow[]>,
) {
const configuredRows = [...runtimeRowsByWorkspaceId.entries()].flatMap(([executionWorkspaceId, rows]) =>
rows.flatMap((row) => row.configIndex === undefined || row.configIndex === null
? []
: [{ executionWorkspaceId, configIndex: row.configIndex }]),
);
if (configuredRows.length === 0) {
return new Map<string, ProjectedWorkspaceRuntimeFailure[]>();
}
const executionWorkspaceIds = [...new Set(configuredRows.map((row) => row.executionWorkspaceId))];
const serviceIndexes = [...new Set(configuredRows.map((row) => String(row.configIndex)))];
const serviceIndexExpression = sql<string | null>`${workspaceOperations.metadata}->>'serviceIndex'`;
const actionExpression = sql<string | null>`${workspaceOperations.metadata}->>'action'`;
const operations = await db
.selectDistinctOn(
[workspaceOperations.executionWorkspaceId, serviceIndexExpression],
{
id: workspaceOperations.id,
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
serviceIndex: serviceIndexExpression,
workspaceCommandId: sql<string | null>`${workspaceOperations.metadata}->>'workspaceCommandId'`,
metadata: workspaceOperations.metadata,
startedAt: workspaceOperations.startedAt,
finishedAt: workspaceOperations.finishedAt,
},
)
.from(workspaceOperations)
.where(
and(
eq(workspaceOperations.companyId, companyId),
inArray(workspaceOperations.executionWorkspaceId, executionWorkspaceIds),
eq(workspaceOperations.status, "failed"),
inArray(actionExpression, ["start", "restart"]),
or(inArray(serviceIndexExpression, serviceIndexes), isNull(serviceIndexExpression)),
),
)
.orderBy(
workspaceOperations.executionWorkspaceId,
serviceIndexExpression,
desc(workspaceOperations.startedAt),
desc(workspaceOperations.createdAt),
);
const result = new Map<string, ProjectedWorkspaceRuntimeFailure[]>();
for (const operation of operations) {
if (!operation.executionWorkspaceId) continue;
const serviceIndex = operation.serviceIndex === null
? null
: Number.parseInt(operation.serviceIndex, 10);
if (serviceIndex !== null && !Number.isInteger(serviceIndex)) continue;
const existing = result.get(operation.executionWorkspaceId) ?? [];
existing.push({
serviceIndex,
workspaceCommandId: operation.workspaceCommandId,
evidence: readWorkspaceRuntimeFailureEvidence(operation),
});
result.set(operation.executionWorkspaceId, existing);
}
return result;
}
async function loadEffectiveRuntimeServicesByExecutionWorkspace(
db: RuntimeServiceReadDb,
companyId: string,
rows: ExecutionWorkspaceRow[],
) {
const inheritedRows = rows.filter((row) => usesInheritedProjectRuntimeServices(row));
const inheritedRows = rows.filter((row) => inheritsProjectRuntimeConfig(row));
const projectWorkspaceIds = inheritedRows
.map((row) => row.projectWorkspaceId)
.filter((value): value is string => Boolean(value));
@@ -1136,8 +1277,7 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace(
const projectRuntimeConfigByWorkspaceId = new Map(
projectWorkspaceRows.map((row) => [
row.id,
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime
?? null,
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null),
]),
);
const effectiveProjectRuntimeServices = new Map(
@@ -1145,35 +1285,113 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace(
projectWorkspaceId,
selectConfiguredRuntimeServiceRows(
projectRuntimeServices.get(projectWorkspaceId) ?? [],
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId) ?? null,
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.workspaceRuntime ?? null,
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.desiredState ?? null,
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.serviceStates ?? null,
),
]),
);
return new Map(
rows.map((row) => {
if (!usesInheritedProjectRuntimeServices(row)) {
return [row.id, executionRuntimeServices.get(row.id) ?? []] as const;
}
const runtimeRowsByWorkspaceId = new Map<string, ProjectedWorkspaceRuntimeServiceRow[]>();
const effectiveConfigByWorkspaceId = new Map<string, EffectiveWorkspaceRuntimeConfig | null>();
for (const row of rows) {
const localConfig = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
const projectConfig = row.projectWorkspaceId
? projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId) ?? null
: null;
const workspaceRuntime = localConfig?.workspaceRuntime ?? projectConfig?.workspaceRuntime ?? null;
const desiredState = localConfig?.desiredState ?? projectConfig?.desiredState ?? null;
const serviceStates = localConfig?.serviceStates ?? projectConfig?.serviceStates ?? null;
effectiveConfigByWorkspaceId.set(
row.id,
workspaceRuntime
? {
workspaceRuntime,
source: localConfig?.workspaceRuntime
? { type: "execution_workspace", id: row.id }
: { type: "project_workspace", id: row.projectWorkspaceId! },
desiredState,
serviceStates,
}
: null,
);
const workspaceRuntime = projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId!) ?? null;
const executionScopedRows = selectConfiguredRuntimeServiceRows(
(executionRuntimeServices.get(row.id) ?? []).filter(
(runtimeService) => runtimeService.scopeType !== "project_workspace",
if (!workspaceRuntime) {
runtimeRowsByWorkspaceId.set(
row.id,
usesInheritedProjectRuntimeServices(row)
? []
: (executionRuntimeServices.get(row.id) ?? []).map((runtimeService) => ({
...runtimeService,
desiredState,
})),
);
continue;
}
if (!usesInheritedProjectRuntimeServices(row)) {
runtimeRowsByWorkspaceId.set(
row.id,
selectConfiguredRuntimeServiceRows(
executionRuntimeServices.get(row.id) ?? [],
workspaceRuntime,
desiredState,
serviceStates,
),
workspaceRuntime,
);
const effectiveRows = [
...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []),
...executionScopedRows,
].sort(
(left, right) =>
(left.configIndex ?? Number.MAX_SAFE_INTEGER) -
(right.configIndex ?? Number.MAX_SAFE_INTEGER),
);
return [row.id, effectiveRows] as const;
}),
continue;
}
const executionScopedRows = selectConfiguredRuntimeServiceRows(
(executionRuntimeServices.get(row.id) ?? []).filter(
(runtimeService) => runtimeService.scopeType !== "project_workspace",
),
workspaceRuntime,
desiredState,
serviceStates,
);
const effectiveRows = [
...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []).map((runtimeService) => ({
...runtimeService,
desiredState:
(runtimeService.configIndex === null || runtimeService.configIndex === undefined
? null
: serviceStates?.[String(runtimeService.configIndex)])
?? desiredState,
})),
...executionScopedRows,
].sort(
(left, right) =>
(left.configIndex ?? Number.MAX_SAFE_INTEGER) -
(right.configIndex ?? Number.MAX_SAFE_INTEGER),
);
runtimeRowsByWorkspaceId.set(row.id, effectiveRows);
}
const failuresByWorkspaceId = await loadLatestRuntimeFailuresByExecutionWorkspace(
db,
companyId,
runtimeRowsByWorkspaceId,
);
return new Map(rows.map((row) => {
const failures = failuresByWorkspaceId.get(row.id) ?? [];
const runtimeServices = (runtimeRowsByWorkspaceId.get(row.id) ?? []).map((runtimeService) => {
if (runtimeService.configIndex === undefined || runtimeService.configIndex === null) return runtimeService;
const candidates = failures.filter((failure) =>
(failure.serviceIndex === runtimeService.configIndex || failure.serviceIndex === null)
&& (!failure.workspaceCommandId || failure.workspaceCommandId === runtimeService.workspaceCommandId),
);
const latestFailure = candidates
.sort((left, right) => right.evidence.failedAt.getTime() - left.evidence.failedAt.getTime())[0]
?.evidence ?? null;
return { ...runtimeService, latestFailure };
});
return [row.id, {
runtimeServices,
effectiveRuntimeConfig: effectiveConfigByWorkspaceId.get(row.id) ?? null,
}] as const;
}));
}
type WorkspaceOverviewPageRow = ExecutionWorkspaceRow & {
@@ -1376,11 +1594,15 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
}
async function hydrateWorkspace(row: ExecutionWorkspaceRow, runtimeServices: WorkspaceRuntimeService[] = []) {
const workspace = toExecutionWorkspace(row, runtimeServices);
async function hydrateWorkspace(
row: ExecutionWorkspaceRow,
runtimeServices: WorkspaceRuntimeService[] = [],
effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null,
) {
const workspace = toExecutionWorkspace(row, runtimeServices, "unknown", effectiveRuntimeConfig);
const { git } = await inspectGitCloseReadiness(workspace);
const assessment = await assessDelivery(row, git);
return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState);
return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState, effectiveRuntimeConfig);
}
async function workspaceHasActiveRun(workspace: Pick<ExecutionWorkspaceRow, "id" | "companyId" | "sourceIssueId">) {
@@ -1903,7 +2125,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
}
const items: WorkspaceOverviewItem[] = pageRows.map((row) => {
const runtimeServices = (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService);
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService);
const runningServiceCount = runtimeServices.filter((service) => service.status === "running").length;
const primaryService = selectPrimaryOverviewService(runtimeServices);
const config = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
@@ -1971,12 +2194,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
.where(and(...conditions))
.orderBy(desc(executionWorkspaces.lastUsedAt), desc(executionWorkspaces.createdAt));
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, companyId, rows);
return Promise.all(rows.map((row) =>
hydrateWorkspace(
return Promise.all(rows.map((row) => {
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
return hydrateWorkspace(
row,
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
),
));
(runtimeProjection?.runtimeServices ?? []).map(toRuntimeService),
runtimeProjection?.effectiveRuntimeConfig ?? null,
);
}));
},
listSummaries: async (companyId: string, filters?: {
@@ -2130,9 +2355,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
.then((rows) => rows[0] ?? null);
if (!row) return null;
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, row.companyId, [row]);
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
return hydrateWorkspace(
row,
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
(runtimeProjection?.runtimeServices ?? []).map(toRuntimeService),
runtimeProjection?.effectiveRuntimeConfig ?? null,
);
},
@@ -2145,7 +2372,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
if (!workspace) return null;
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, workspace.companyId, [workspace]);
const runtimeServices = (runtimeServicesByWorkspaceId.get(workspace.id) ?? []).map(toRuntimeService);
const runtimeProjection = runtimeServicesByWorkspaceId.get(workspace.id);
const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService);
const linkedIssues = await db
.select({
@@ -2240,7 +2468,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
warnings.push("This shared workspace session points at project workspace infrastructure. Archiving it only removes the session record.");
}
if (runtimeServices.some((service) => service.status !== "stopped")) {
if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) {
warnings.push(
runtimeServices.length === 1
? "Closing this workspace will stop 1 attached runtime service."
@@ -2291,7 +2519,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
},
];
if (runtimeServices.some((service) => service.status !== "stopped")) {
if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) {
plannedActions.push({
kind: "stop_runtime_services",
label: runtimeServices.length === 1 ? "Stop attached runtime service" : "Stop attached runtime services",
@@ -3225,7 +3453,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
);
assertBranchReconcileRuntimeServicesStopped({
inspection,
runtimeServices: (runtimeServicesByWorkspaceId.get(existing.id) ?? []).map(toRuntimeService),
runtimeServices: (
runtimeServicesByWorkspaceId.get(existing.id)?.runtimeServices ?? []
).map(toRuntimeService),
});
// The git rescue has to happen before the DB transaction because the
// transaction may be retried/rolled back, while git side effects cannot.
@@ -3303,8 +3533,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
lockedRow.companyId,
[lockedRow],
);
const lockedRuntimeServices = (lockedRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService);
const lockedWorkspace = toExecutionWorkspace(lockedRow, lockedRuntimeServices);
const lockedRuntimeProjection = lockedRuntimeServicesByWorkspaceId.get(lockedRow.id);
const lockedRuntimeServices = (lockedRuntimeProjection?.runtimeServices ?? []).map(toRuntimeService);
const lockedWorkspace = toExecutionWorkspace(
lockedRow,
lockedRuntimeServices,
"unknown",
lockedRuntimeProjection?.effectiveRuntimeConfig ?? null,
);
if (!lockedWorkspace.sourceIssueId) {
throw unprocessable("Execution workspace needs a source issue before Paperclip can audit branch reconciliation");
}
@@ -3352,7 +3588,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
lockedRow.companyId,
[lockedRow],
);
const latestRuntimeServices = (latestRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService);
const latestRuntimeServices = (
latestRuntimeServicesByWorkspaceId.get(lockedRow.id)?.runtimeServices ?? []
).map(toRuntimeService);
assertBranchReconcileWorkspaceIsSafe({
workspaceStatus: lockedWorkspace.status,
inspection,
+38 -6
View File
@@ -24,11 +24,15 @@ import {
type ProjectManagedByPlugin,
type ProjectWorkspaceRuntimeConfig,
type ProjectWorkspace,
type WorkspaceRuntimeDesiredState,
type WorkspaceRuntimeService,
type PluginManagedProjectDeclaration,
type PluginManagedProjectResolution,
} from "@paperclipai/shared";
import { listCurrentRuntimeServicesForProjectWorkspaces } from "./workspace-runtime-read-model.js";
import {
listCurrentRuntimeServicesForProjectWorkspaces,
selectConfiguredRuntimeServiceRows,
} from "./workspace-runtime-read-model.js";
import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-policy.js";
import { mergeProjectWorkspaceRuntimeConfig, readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js";
import { resolveManagedProjectWorkspaceDir } from "../home-paths.js";
@@ -36,6 +40,11 @@ import { resolveManagedProjectWorkspaceDir } from "../home-paths.js";
type ProjectRow = typeof projects.$inferSelect;
type ProjectWorkspaceRow = typeof projectWorkspaces.$inferSelect;
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & {
configIndex?: number | null;
workspaceCommandId?: string | null;
desiredState?: WorkspaceRuntimeDesiredState | null;
};
const REPO_ONLY_CWD_SENTINEL = "/__paperclip_repo_only__";
type CreateWorkspaceInput = {
name?: string | null;
@@ -117,7 +126,7 @@ async function attachGoals(db: Db, rows: ProjectRow[]): Promise<ProjectWithGoals
});
}
function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeService {
function toRuntimeService(row: ProjectedWorkspaceRuntimeServiceRow): WorkspaceRuntimeService {
return {
id: row.id,
companyId: row.companyId,
@@ -129,6 +138,8 @@ function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeServ
scopeId: row.scopeId ?? null,
serviceName: row.serviceName,
status: row.status as WorkspaceRuntimeService["status"],
actualState: row.status as WorkspaceRuntimeService["actualState"],
desiredState: row.desiredState ?? null,
lifecycle: row.lifecycle as WorkspaceRuntimeService["lifecycle"],
reuseKey: row.reuseKey ?? null,
command: row.command ?? null,
@@ -144,11 +155,29 @@ function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeServ
stoppedAt: row.stoppedAt ?? null,
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"],
configIndex: row.configIndex ?? null,
workspaceCommandId: row.workspaceCommandId ?? null,
latestFailure: null,
createdAt: row.createdAt,
updatedAt: row.updatedAt,
};
}
function projectWorkspaceRuntimeServices(
workspace: ProjectWorkspaceRow,
rows: WorkspaceRuntimeServiceRow[],
) {
const runtimeConfig = readProjectWorkspaceRuntimeConfig(
(workspace.metadata as Record<string, unknown> | null) ?? null,
);
return selectConfiguredRuntimeServiceRows(
rows,
runtimeConfig?.workspaceRuntime ?? null,
runtimeConfig?.desiredState ?? null,
runtimeConfig?.serviceStates ?? null,
).map(toRuntimeService);
}
function toWorkspace(
row: ProjectWorkspaceRow,
runtimeServices: WorkspaceRuntimeService[] = [],
@@ -246,9 +275,12 @@ async function attachWorkspaces(db: Db, rows: ProjectWithGoals[]): Promise<Proje
workspaceRows.map((workspace) => workspace.id),
);
const sharedRuntimeServicesByWorkspaceId = new Map(
Array.from(runtimeServicesByWorkspaceId.entries()).map(([workspaceId, services]) => [
workspaceId,
services.map(toRuntimeService),
workspaceRows.map((workspace) => [
workspace.id,
projectWorkspaceRuntimeServices(
workspace,
runtimeServicesByWorkspaceId.get(workspace.id) ?? [],
),
]),
);
@@ -891,7 +923,7 @@ export function projectService(db: Db) {
return rows.map((row) =>
toWorkspace(
row,
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
projectWorkspaceRuntimeServices(row, runtimeServicesByWorkspaceId.get(row.id) ?? []),
),
);
},
+108 -2
View File
@@ -3,13 +3,109 @@ import type { Db } from "@paperclipai/db";
import { workspaceOperations } from "@paperclipai/db";
import type { WorkspaceOperation, WorkspaceOperationPhase, WorkspaceOperationStatus } from "@paperclipai/shared";
import { asc, desc, eq, inArray, isNull, or, and } from "drizzle-orm";
import { notFound } from "../errors.js";
import { HttpError, notFound } from "../errors.js";
import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
import { instanceSettingsService } from "./instance-settings.js";
import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js";
type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
export interface WorkspaceOperationFailureEvidence {
operationId: string;
operationLogPath: string;
code: string;
message: string;
remediation: string;
details: {
port?: number;
attemptedPortCount?: number;
conflictingExecutionWorkspaceId?: string;
conflictingProjectWorkspaceId?: string;
} | null;
failedAt: string;
}
const WORKSPACE_OPERATION_FAILURE_PROPERTY = "paperclipWorkspaceOperationFailure";
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function readSafeFailureDetails(value: unknown): WorkspaceOperationFailureEvidence["details"] {
if (!isRecord(value)) return null;
const details: NonNullable<WorkspaceOperationFailureEvidence["details"]> = {};
if (typeof value.port === "number" && Number.isInteger(value.port)) details.port = value.port;
if (typeof value.attemptedPortCount === "number" && Number.isInteger(value.attemptedPortCount)) {
details.attemptedPortCount = value.attemptedPortCount;
}
if (typeof value.conflictingExecutionWorkspaceId === "string") {
details.conflictingExecutionWorkspaceId = value.conflictingExecutionWorkspaceId;
}
if (typeof value.conflictingProjectWorkspaceId === "string") {
details.conflictingProjectWorkspaceId = value.conflictingProjectWorkspaceId;
}
return Object.keys(details).length > 0 ? details : null;
}
function buildWorkspaceOperationFailureEvidence(input: {
operationId: string;
failedAt: Date;
error: unknown;
metadata: Record<string, unknown> | null | undefined;
}): WorkspaceOperationFailureEvidence {
const httpDetails = input.error instanceof HttpError && isRecord(input.error.details)
? input.error.details
: null;
const action = typeof input.metadata?.action === "string" ? input.metadata.action : null;
const isRuntimeStart = action === "start" || action === "restart";
const code = typeof httpDetails?.code === "string"
? httpDetails.code
: isRuntimeStart
? "workspace_runtime_start_failed"
: "workspace_operation_failed";
const message = input.error instanceof HttpError
? input.error.message
: isRuntimeStart
? "Workspace runtime service failed to start."
: "Workspace operation failed.";
const remediation = typeof httpDetails?.remediation === "string"
? httpDetails.remediation
: "Review the workspace operation log, correct the runtime configuration or dependency, and retry.";
return {
operationId: input.operationId,
operationLogPath: `/api/workspace-operations/${input.operationId}/log`,
code,
message,
remediation,
details: readSafeFailureDetails(httpDetails),
failedAt: input.failedAt.toISOString(),
};
}
export function attachWorkspaceOperationFailureEvidence(
error: unknown,
evidence: WorkspaceOperationFailureEvidence,
) {
const target = error instanceof Error ? error : new Error(String(error));
Object.defineProperty(target, WORKSPACE_OPERATION_FAILURE_PROPERTY, {
configurable: true,
enumerable: false,
value: evidence,
writable: true,
});
return target;
}
export function getWorkspaceOperationFailureEvidence(
error: unknown,
): WorkspaceOperationFailureEvidence | null {
if (!(error instanceof Error)) return null;
const value = (error as unknown as Record<string, unknown>)[WORKSPACE_OPERATION_FAILURE_PROPERTY];
if (!isRecord(value)) return null;
return value as unknown as WorkspaceOperationFailureEvidence;
}
function toWorkspaceOperation(row: WorkspaceOperationRow): WorkspaceOperation {
return {
id: row.id,
@@ -187,6 +283,12 @@ export function workspaceOperationService(db: Db) {
await append("stderr", error instanceof Error ? error.message : String(error));
const finalized = await logStore.finalize(handle).catch(() => null);
const finishedAt = new Date();
const failureEvidence = buildWorkspaceOperationFailureEvidence({
operationId: id,
failedAt: finishedAt,
error,
metadata: recordInput.metadata,
});
await db
.update(workspaceOperations)
.set({
@@ -197,11 +299,15 @@ export function workspaceOperationService(db: Db) {
logBytes: finalized?.bytes ?? null,
logSha256: finalized?.sha256 ?? null,
logCompressed: finalized?.compressed ?? false,
metadata: redactCurrentUserValue(
combineMetadata(recordInput.metadata, { failureEvidence }),
currentUserRedactionOptions,
) as Record<string, unknown> | null,
finishedAt,
updatedAt: finishedAt,
})
.where(eq(workspaceOperations.id, id));
throw error;
throw attachWorkspaceOperationFailureEvidence(error, failureEvidence);
}
},
};
@@ -53,7 +53,7 @@ describe("selectConfiguredRuntimeServiceRows", () => {
expect(selectConfiguredRuntimeServiceRows(historicalRows, null)).toEqual([]);
});
it("selects the newest current row for each configured service and annotates its config index", () => {
it("selects the newest configured row including failed state and annotates effective intent", () => {
const oldWeb = runtimeServiceRow({
serviceName: "web",
command: "pnpm dev",
@@ -62,6 +62,8 @@ describe("selectConfiguredRuntimeServiceRows", () => {
const currentWeb = runtimeServiceRow({
serviceName: "web",
command: "pnpm dev",
status: "failed",
healthStatus: "unhealthy",
updatedAt: new Date("2026-07-30T10:00:00.000Z"),
});
const removedWorker = runtimeServiceRow({
@@ -73,13 +75,18 @@ describe("selectConfiguredRuntimeServiceRows", () => {
const selected = selectConfiguredRuntimeServiceRows(
[removedWorker, currentWeb, oldWeb],
{ services: [{ name: "web", command: "pnpm dev" }] },
"stopped",
{ "0": "running" },
);
expect(selected).toEqual([
expect.objectContaining({
id: currentWeb.id,
serviceName: "web",
status: "failed",
configIndex: 0,
workspaceCommandId: "service:web",
desiredState: "running",
}),
]);
});
@@ -4,6 +4,7 @@ import {
listWorkspaceServiceCommandDefinitions,
matchWorkspaceRuntimeServiceToCommand,
} from "@paperclipai/shared";
import type { WorkspaceRuntimeDesiredState, WorkspaceRuntimeServiceStateMap } from "@paperclipai/shared";
import { and, desc, eq, inArray } from "drizzle-orm";
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
@@ -34,12 +35,20 @@ export function selectCurrentRuntimeServiceRows(rows: WorkspaceRuntimeServiceRow
export function selectConfiguredRuntimeServiceRows(
rows: WorkspaceRuntimeServiceRow[],
workspaceRuntime: Record<string, unknown> | null | undefined,
desiredState: WorkspaceRuntimeDesiredState | null = null,
serviceStates: WorkspaceRuntimeServiceStateMap | null = null,
) {
const availableRows = selectCurrentRuntimeServiceRows(rows).map((row) => ({
...row,
configIndex: null as number | null,
workspaceCommandId: null as string | null,
desiredState: null as WorkspaceRuntimeDesiredState | null,
}));
const selectedRows: Array<WorkspaceRuntimeServiceRow & { configIndex: number | null }> = [];
const selectedRows: Array<WorkspaceRuntimeServiceRow & {
configIndex: number | null;
workspaceCommandId: string | null;
desiredState: WorkspaceRuntimeDesiredState | null;
}> = [];
for (const command of listWorkspaceServiceCommandDefinitions(workspaceRuntime)) {
const reuseScope = command.rawConfig.reuseScope;
@@ -60,6 +69,10 @@ export function selectConfiguredRuntimeServiceRows(
selectedRows.push({
...matchedRow,
configIndex: command.serviceIndex,
workspaceCommandId: command.id,
desiredState:
(command.serviceIndex === null ? null : serviceStates?.[String(command.serviceIndex)])
?? desiredState,
});
availableRows.splice(availableRows.indexOf(matchedRow), 1);
}
+77 -18
View File
@@ -186,6 +186,24 @@ const runtimeProvisionByWorkspace = new Map<string, Promise<void>>();
const DEFAULT_EXECUTE_PROCESS_OUTPUT_BYTES = 256 * 1024;
export const WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS = 32;
const ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES = ["provisioning", "starting", "running"] as const;
const RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY = "paperclipRuntimeServiceFailureRecord";
function attachRuntimeServiceFailureRecord(error: unknown, record: RuntimeServiceRecord) {
const target = error instanceof Error ? error : new Error(String(error));
Object.defineProperty(target, RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY, {
configurable: true,
enumerable: false,
value: record,
writable: true,
});
return target;
}
function getRuntimeServiceFailureRecord(error: unknown): RuntimeServiceRecord | null {
if (!(error instanceof Error)) return null;
const value = (error as unknown as Record<string, unknown>)[RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY];
return value && typeof value === "object" ? value as RuntimeServiceRecord : null;
}
class RuntimeServicePortBindCollision extends Error {
readonly port: number;
@@ -905,7 +923,12 @@ async function findActiveRuntimeServiceBlockingDirtyQuarantine(input: {
scopeType: workspaceRuntimeServices.scopeType,
})
.from(workspaceRuntimeServices)
.where(and(serviceScopeCondition, ne(workspaceRuntimeServices.status, "stopped")))
.where(
and(
serviceScopeCondition,
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES]),
),
)
.orderBy(desc(workspaceRuntimeServices.updatedAt), desc(workspaceRuntimeServices.createdAt))
.limit(1);
return service ?? null;
@@ -4153,7 +4176,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: {
eq(workspaceRuntimeServices.companyId, input.companyId),
eq(workspaceRuntimeServices.reuseKey, input.reuseKey),
eq(workspaceRuntimeServices.provider, "local_process"),
eq(workspaceRuntimeServices.status, "stopped"),
inArray(workspaceRuntimeServices.status, ["stopped", "failed"]),
),
)
.orderBy(desc(workspaceRuntimeServices.updatedAt))
@@ -4175,7 +4198,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: {
and(
eq(workspaceRuntimeServices.companyId, input.companyId),
eq(workspaceRuntimeServices.provider, "local_process"),
eq(workspaceRuntimeServices.status, "stopped"),
inArray(workspaceRuntimeServices.status, ["stopped", "failed"]),
eq(workspaceRuntimeServices.scopeType, input.scopeType),
scopeIdCondition,
eq(workspaceRuntimeServices.serviceName, input.serviceName),
@@ -4770,7 +4793,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
processGroupId: child.pid,
});
}
record.status = "stopped";
record.status = "failed";
record.healthStatus = "unhealthy";
record.lastUsedAt = new Date().toISOString();
record.stoppedAt = new Date().toISOString();
@@ -4856,28 +4879,54 @@ async function startLocalRuntimeService(
);
const retryBindCollisions = fixedPortFallbackEnabled || portType === "auto";
const deferReadiness = Boolean(options?.deferReadiness && !fixedPortFallbackEnabled);
const identity = resolveRuntimeServiceReuseIdentity({
service: input.service,
workspace: input.workspace,
agent: input.agent,
issue: input.issue,
adapterEnv: input.adapterEnv,
scopeType: input.scopeType,
scopeId: input.scopeId,
});
if (!identity.command) throw new Error(`Runtime service "${identity.serviceName}" is missing command`);
let currentAttemptRecord = provisioningRecord ?? createProvisioningRuntimeServiceRecord(input, identity);
if (!provisioningRecord) {
currentAttemptRecord.status = "starting";
await persistRuntimeServiceRecord(input.db, currentAttemptRecord);
}
let retryRuntimeServiceId = input.runtimeServiceId;
try {
for (let attempt = 0; attempt < WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS; attempt += 1) {
try {
const previousAttemptRecord = currentAttemptRecord;
started = await spawnLocalRuntimeService({
...input,
excludedPorts,
runtimeServiceId: provisioningRecord?.id ?? input.runtimeServiceId,
runtimeServiceId: provisioningRecord?.id ?? retryRuntimeServiceId,
});
retryRuntimeServiceId = started.record.id;
currentAttemptRecord = started.record;
if (runtimeProvisionCommand) {
await persistRuntimeServiceRecord(input.db, started.record);
}
if (provisioningRecord && started.record.id !== provisioningRecord.id && input.db) {
if (started.record.id !== previousAttemptRecord.id && input.db) {
await input.db
.delete(workspaceRuntimeServices)
.where(eq(workspaceRuntimeServices.id, provisioningRecord.id));
.where(eq(workspaceRuntimeServices.id, previousAttemptRecord.id));
}
if (!deferReadiness) {
await started.readiness;
}
return started;
} catch (error) {
const failedRecord = started?.record ?? currentAttemptRecord;
const nowIso = new Date().toISOString();
failedRecord.status = "failed";
failedRecord.healthStatus = "unhealthy";
failedRecord.lastUsedAt = nowIso;
failedRecord.stoppedAt = nowIso;
await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined);
if (!(error instanceof RuntimeServicePortBindCollision) || !retryBindCollisions) throw error;
excludedPorts.add(error.port);
started = null;
@@ -4899,15 +4948,15 @@ async function startLocalRuntimeService(
remediation: "Retry the start or configure a different runtime service port.",
});
} catch (error) {
if (!started && provisioningRecord && provisioningRecord.status === "starting") {
if (!started && currentAttemptRecord.status !== "failed") {
const nowIso = new Date().toISOString();
provisioningRecord.status = "failed";
provisioningRecord.healthStatus = "unhealthy";
provisioningRecord.lastUsedAt = nowIso;
provisioningRecord.stoppedAt = nowIso;
await persistRuntimeServiceRecord(input.db, provisioningRecord).catch(() => undefined);
currentAttemptRecord.status = "failed";
currentAttemptRecord.healthStatus = "unhealthy";
currentAttemptRecord.lastUsedAt = nowIso;
currentAttemptRecord.stoppedAt = nowIso;
await persistRuntimeServiceRecord(input.db, currentAttemptRecord).catch(() => undefined);
}
throw error;
throw attachRuntimeServiceFailureRecord(error, currentAttemptRecord);
}
}
@@ -4921,12 +4970,13 @@ function scheduleIdleStop(record: RuntimeServiceRecord) {
}, idleSeconds * 1000);
}
async function stopRuntimeService(serviceId: string) {
async function stopRuntimeService(serviceId: string, options?: { preserveFailure?: boolean }) {
const record = runtimeServicesById.get(serviceId);
if (!record) return;
clearIdleTimer(record);
record.status = "stopped";
record.healthStatus = "unknown";
const preserveFailure = options?.preserveFailure === true && record.status === "failed";
record.status = preserveFailure ? "failed" : "stopped";
record.healthStatus = preserveFailure ? "unhealthy" : "unknown";
record.lastUsedAt = new Date().toISOString();
record.stoppedAt = new Date().toISOString();
runtimeServicesById.delete(serviceId);
@@ -5488,7 +5538,16 @@ export async function startRuntimeServicesForWorkspaceControl(
});
} catch (error) {
for (const serviceId of startBatch.startedServiceIds) {
await stopRuntimeService(serviceId).catch(() => undefined);
await stopRuntimeService(serviceId, { preserveFailure: true }).catch(() => undefined);
}
const failedRecord = getRuntimeServiceFailureRecord(error);
if (failedRecord) {
const nowIso = new Date().toISOString();
failedRecord.status = "failed";
failedRecord.healthStatus = "unhealthy";
failedRecord.lastUsedAt = nowIso;
failedRecord.stoppedAt = nowIso;
await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined);
}
if (preparedProvisioning && startBatch.startedServiceIds.length === 0) {
const nowIso = new Date().toISOString();