diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index eb81657f74c..ab0f7a30b51 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -844,6 +844,7 @@ export type { ExecutionWorkspaceCloseReadiness, ExecutionWorkspaceCloseReadinessState, ExecutionWorkspaceDeliveryState, + EffectiveWorkspaceRuntimeConfig, WorkspaceOverviewItem, WorkspaceOverviewLinkedIssue, WorkspaceOverviewPrimaryService, @@ -852,6 +853,10 @@ export type { WorkspaceCommandDefinition, WorkspaceCommandKind, WorkspaceRuntimeControlTarget, + WorkspaceRuntimeActualState, + WorkspaceRuntimeConfigSource, + WorkspaceRuntimeFailureDetails, + WorkspaceRuntimeFailureEvidence, WorkspaceRuntimeService, WorkspaceRuntimeServiceStateMap, WorkspaceOperation, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 52ff2ed3a62..83c3d921317 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -366,6 +366,7 @@ export type { ExecutionWorkspaceCloseReadiness, ExecutionWorkspaceCloseReadinessState, ExecutionWorkspaceDeliveryState, + EffectiveWorkspaceRuntimeConfig, WorkspaceOverviewItem, WorkspaceOverviewLinkedIssue, WorkspaceOverviewPrimaryService, @@ -374,6 +375,10 @@ export type { WorkspaceCommandDefinition, WorkspaceCommandKind, WorkspaceRuntimeControlTarget, + WorkspaceRuntimeActualState, + WorkspaceRuntimeConfigSource, + WorkspaceRuntimeFailureDetails, + WorkspaceRuntimeFailureEvidence, WorkspaceRuntimeService, WorkspaceRuntimeServiceStateMap, WorkspaceRuntimeDesiredState, diff --git a/packages/shared/src/types/workspace-runtime.ts b/packages/shared/src/types/workspace-runtime.ts index f2b79f8981c..346c9dcd5e9 100644 --- a/packages/shared/src/types/workspace-runtime.ts +++ b/packages/shared/src/types/workspace-runtime.ts @@ -58,6 +58,41 @@ export type ExecutionWorkspaceCloseActionKind = export type WorkspaceRuntimeDesiredState = "running" | "stopped" | "manual"; export type WorkspaceRuntimeServiceStateMap = Record; export type WorkspaceCommandKind = "service" | "job"; +export type WorkspaceRuntimeActualState = + | "provisioning" + | "starting" + | "running" + | "stopped" + | "failed"; + +export interface WorkspaceRuntimeConfigSource { + type: "execution_workspace" | "project_workspace"; + id: string; +} + +export interface EffectiveWorkspaceRuntimeConfig { + workspaceRuntime: Record; + source: WorkspaceRuntimeConfigSource; + desiredState: WorkspaceRuntimeDesiredState | null; + serviceStates: WorkspaceRuntimeServiceStateMap | null; +} + +export interface WorkspaceRuntimeFailureDetails { + port?: number; + attemptedPortCount?: number; + conflictingExecutionWorkspaceId?: string; + conflictingProjectWorkspaceId?: string; +} + +export interface WorkspaceRuntimeFailureEvidence { + operationId: string; + operationLogPath: string; + code: string; + message: string; + remediation: string; + details: WorkspaceRuntimeFailureDetails | null; + failedAt: Date; +} export interface WorkspaceCommandSource { type: "paperclip"; @@ -273,6 +308,7 @@ export interface ExecutionWorkspace { cleanupEligibleAt: Date | null; cleanupReason: string | null; config: ExecutionWorkspaceConfig | null; + effectiveRuntimeConfig: EffectiveWorkspaceRuntimeConfig | null; metadata: Record | null; runtimeServices?: WorkspaceRuntimeService[]; createdAt: Date; @@ -289,7 +325,10 @@ export interface WorkspaceRuntimeService { scopeType: "project_workspace" | "execution_workspace" | "run" | "agent"; scopeId: string | null; serviceName: string; - status: "provisioning" | "starting" | "running" | "stopped" | "failed"; + /** @deprecated Prefer actualState when comparing runtime intent with observed state. */ + status: WorkspaceRuntimeActualState; + actualState: WorkspaceRuntimeActualState; + desiredState: WorkspaceRuntimeDesiredState | null; lifecycle: "shared" | "ephemeral"; reuseKey: string | null; command: string | null; @@ -306,6 +345,8 @@ export interface WorkspaceRuntimeService { stopPolicy: Record | null; healthStatus: "unknown" | "healthy" | "unhealthy"; configIndex?: number | null; + workspaceCommandId?: string | null; + latestFailure: WorkspaceRuntimeFailureEvidence | null; createdAt: Date; updatedAt: Date; } diff --git a/packages/shared/src/validators/execution-workspace.ts b/packages/shared/src/validators/execution-workspace.ts index fc03413aebb..704a4e9fb97 100644 --- a/packages/shared/src/validators/execution-workspace.ts +++ b/packages/shared/src/validators/execution-workspace.ts @@ -110,6 +110,8 @@ export const workspaceRuntimeServiceSchema = z.object({ scopeId: z.string().nullable(), serviceName: z.string(), status: z.enum(["provisioning", "starting", "running", "stopped", "failed"]), + actualState: z.enum(["provisioning", "starting", "running", "stopped", "failed"]), + desiredState: z.enum(["running", "stopped", "manual"]).nullable(), lifecycle: z.enum(["shared", "ephemeral"]), reuseKey: z.string().nullable(), command: z.string().nullable(), @@ -126,6 +128,21 @@ export const workspaceRuntimeServiceSchema = z.object({ stopPolicy: z.record(z.string(), z.unknown()).nullable(), healthStatus: z.enum(["unknown", "healthy", "unhealthy"]), configIndex: z.number().int().nonnegative().nullable().optional(), + workspaceCommandId: z.string().nullable().optional(), + latestFailure: z.object({ + operationId: z.string().uuid(), + operationLogPath: z.string(), + code: z.string(), + message: z.string(), + remediation: z.string(), + details: z.object({ + port: z.number().int().optional(), + attemptedPortCount: z.number().int().nonnegative().optional(), + conflictingExecutionWorkspaceId: z.string().uuid().optional(), + conflictingProjectWorkspaceId: z.string().uuid().optional(), + }).strict().nullable(), + failedAt: z.coerce.date(), + }).strict().nullable(), createdAt: z.coerce.date(), updatedAt: z.coerce.date(), }).strict(); diff --git a/server/src/__tests__/execution-workspaces-routes.test.ts b/server/src/__tests__/execution-workspaces-routes.test.ts index d7f9913ce50..8dd14590fd0 100644 --- a/server/src/__tests__/execution-workspaces-routes.test.ts +++ b/server/src/__tests__/execution-workspaces-routes.test.ts @@ -2,7 +2,9 @@ import express from "express"; import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { errorHandler } from "../middleware/index.js"; +import { conflict } from "../errors.js"; import { executionWorkspaceRoutes } from "../routes/execution-workspaces.js"; +import { attachWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js"; const mockExecutionWorkspaceService = vi.hoisted(() => ({ list: vi.fn(), @@ -61,6 +63,10 @@ vi.mock("../services/workspace-runtime.js", async (importActual) => { }; }); +vi.mock("../routes/workspace-runtime-authz.js", () => ({ + assertCanManageExecutionWorkspaceRuntimeServices: vi.fn(async () => undefined), +})); + function createApp(actor: Record = { type: "board", userId: "local-board", @@ -107,6 +113,9 @@ describe.sequential("execution workspace routes", () => { ]); mockExecutionWorkspaceService.getById.mockResolvedValue(null); mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue(null); + mockWorkspaceOperationService.createRecorder.mockReturnValue({ + recordOperation: vi.fn(), + }); mockHeartbeatService.wakeup.mockResolvedValue(null); }); @@ -161,6 +170,141 @@ describe.sequential("execution workspace routes", () => { expect(mockExecutionWorkspaceService.listOverview).not.toHaveBeenCalled(); }); + it("returns effective inherited runtime config and the current failed service row", async () => { + mockExecutionWorkspaceService.getById.mockResolvedValue({ + id: "workspace-1", + companyId: "company-1", + projectWorkspaceId: "project-workspace-1", + effectiveRuntimeConfig: { + workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] }, + source: { type: "project_workspace", id: "project-workspace-1" }, + desiredState: "running", + serviceStates: null, + }, + runtimeServices: [{ + id: "service-1", + serviceName: "web", + status: "failed", + actualState: "failed", + desiredState: "running", + latestFailure: { + operationId: "operation-1", + operationLogPath: "/api/workspace-operations/operation-1/log", + code: "workspace_runtime_start_failed", + message: "Workspace runtime service failed to start.", + remediation: "Review the workspace operation log and retry.", + details: null, + failedAt: new Date("2026-08-11T12:00:00.000Z"), + }, + }], + }); + + const res = await request(createApp()).get("/api/execution-workspaces/workspace-1"); + + expect(res.status).toBe(200); + expect(res.body.effectiveRuntimeConfig.source).toEqual({ + type: "project_workspace", + id: "project-workspace-1", + }); + expect(res.body.runtimeServices).toEqual([ + expect.objectContaining({ + id: "service-1", + actualState: "failed", + desiredState: "running", + latestFailure: expect.objectContaining({ operationId: "operation-1" }), + }), + ]); + }); + + it("does not expose workspace detail across company scope", async () => { + mockExecutionWorkspaceService.getById.mockResolvedValue({ + id: "workspace-1", + companyId: "company-1", + }); + + const res = await request(createApp({ + type: "board", + userId: "other-board", + companyIds: ["company-2"], + source: "session", + isInstanceAdmin: false, + })).get("/api/execution-workspaces/workspace-1"); + + expect(res.status).toBe(404); + }); + + it("returns a safe operation reference when runtime start fails", async () => { + mockExecutionWorkspaceService.getById.mockResolvedValue({ + id: "workspace-1", + companyId: "company-1", + projectId: null, + projectWorkspaceId: null, + sourceIssueId: null, + name: "Runtime failure workspace", + mode: "isolated_workspace", + strategyType: "git_worktree", + cwd: "/tmp/runtime-failure-workspace", + repoUrl: null, + baseRef: "master", + branchName: "runtime-failure", + providerType: "git_worktree", + providerRef: "/tmp/runtime-failure-workspace", + config: { + workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] }, + desiredState: "running", + serviceStates: null, + }, + effectiveRuntimeConfig: { + workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] }, + source: { type: "execution_workspace", id: "workspace-1" }, + desiredState: "running", + serviceStates: null, + }, + runtimeServices: [], + metadata: null, + }); + const operationError = attachWorkspaceOperationFailureEvidence( + conflict("No safe automatically allocated runtime service port is available.", { + code: "workspace_runtime_port_allocation_exhausted", + attemptedPortCount: 10, + cwd: "/secret/workspace/path", + remediation: "Configure a different runtime service port.", + }), + { + operationId: "operation-1", + operationLogPath: "/api/workspace-operations/operation-1/log", + code: "workspace_runtime_port_allocation_exhausted", + message: "No safe automatically allocated runtime service port is available.", + remediation: "Configure a different runtime service port.", + details: { attemptedPortCount: 10 }, + failedAt: "2026-08-11T12:00:00.000Z", + }, + ); + mockWorkspaceOperationService.createRecorder.mockReturnValue({ + recordOperation: vi.fn(async () => { + throw operationError; + }), + }); + + const res = await request(createApp()) + .post("/api/execution-workspaces/workspace-1/runtime-services/start") + .send({ workspaceCommandId: "service:web" }); + + expect(res.status).toBe(409); + expect(res.body).toMatchObject({ + error: "No safe automatically allocated runtime service port is available.", + details: { + code: "workspace_runtime_port_allocation_exhausted", + attemptedPortCount: 10, + remediation: "Configure a different runtime service port.", + operationId: "operation-1", + operationLogPath: "/api/workspace-operations/operation-1/log", + failedAt: "2026-08-11T12:00:00.000Z", + }, + }); + expect(JSON.stringify(res.body)).not.toContain("/secret/workspace/path"); + }); + it.each([ ["forward", { mode: "forward" }], ["override", { mode: "override", reason: "operator break-glass" }], diff --git a/server/src/__tests__/execution-workspaces-service.test.ts b/server/src/__tests__/execution-workspaces-service.test.ts index 5c5a239897b..f64ce055bac 100644 --- a/server/src/__tests__/execution-workspaces-service.test.ts +++ b/server/src/__tests__/execution-workspaces-service.test.ts @@ -21,6 +21,7 @@ import { issues, projectWorkspaces, projects, + workspaceOperations, workspaceRuntimeServices, } from "@paperclipai/db"; import { @@ -39,6 +40,11 @@ import { readMetadataReopenPendingConsumptionSince, } from "../services/execution-workspaces.ts"; import { issueService } from "../services/issues.ts"; +import { conflict } from "../errors.ts"; +import { + getWorkspaceOperationFailureEvidence, + workspaceOperationService, +} from "../services/workspace-operations.ts"; import { startRuntimeServicesForWorkspaceControl, stopRuntimeServicesForExecutionWorkspace, @@ -287,6 +293,73 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { await tempDb?.cleanup(); }); + it("persists safe structured evidence and attaches its exact operation reference", async () => { + const companyId = randomUUID(); + const logDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-operation-log-")); + tempDirs.add(logDir); + const previousLogBasePath = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; + process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = logDir; + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }); + + let thrown: unknown; + try { + await workspaceOperationService(db).createRecorder({ companyId }).recordOperation({ + phase: "workspace_provision", + command: "pnpm dev", + metadata: { action: "start", serviceIndex: 0, workspaceCommandId: "service:web" }, + run: async () => { + throw conflict("No safe runtime port is available.", { + code: "workspace_runtime_port_allocation_exhausted", + port: 3100, + cwd: "/secret/workspace/path", + remediation: "Configure a different runtime service port.", + }); + }, + }); + } catch (error) { + thrown = error; + } finally { + if (previousLogBasePath === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; + else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = previousLogBasePath; + } + + const evidence = getWorkspaceOperationFailureEvidence(thrown); + expect(evidence).toMatchObject({ + operationLogPath: `/api/workspace-operations/${evidence?.operationId}/log`, + code: "workspace_runtime_port_allocation_exhausted", + message: "No safe runtime port is available.", + remediation: "Configure a different runtime service port.", + details: { port: 3100 }, + }); + expect(evidence?.details).not.toHaveProperty("cwd"); + + const [operation] = await db + .select() + .from(workspaceOperations) + .where(eq(workspaceOperations.id, evidence!.operationId)); + expect(operation).toMatchObject({ + status: "failed", + metadata: expect.objectContaining({ + action: "start", + serviceIndex: 0, + workspaceCommandId: "service:web", + failureEvidence: expect.objectContaining({ + operationId: evidence!.operationId, + operationLogPath: evidence!.operationLogPath, + code: "workspace_runtime_port_allocation_exhausted", + details: { port: 3100 }, + }), + }), + }); + expect(JSON.stringify(operation?.metadata)).not.toContain("/secret/workspace/path"); + }); + async function seedTerminalWorkspace(options: { mergedPr?: boolean; activeRun?: boolean; @@ -3922,16 +3995,31 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { ]); const [workspace] = await svc.list(companyId); + expect(workspace?.effectiveRuntimeConfig).toEqual({ + workspaceRuntime: { + services: [ + { name: "web", command: "pnpm dev" }, + { name: "worker", command: "pnpm worker", reuseScope: "execution_workspace" }, + ], + }, + source: { type: "project_workspace", id: projectWorkspaceId }, + desiredState: "stopped", + serviceStates: null, + }); expect(workspace?.runtimeServices).toEqual([ expect.objectContaining({ id: currentWebServiceId, serviceName: "web", configIndex: 0, + actualState: "stopped", + desiredState: "stopped", }), expect.objectContaining({ id: currentWorkerServiceId, serviceName: "worker", configIndex: 1, + actualState: "running", + desiredState: "stopped", }), ]); @@ -3948,6 +4036,182 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { }); }); + it("projects one failed configured row with inherited config and company-scoped failure evidence", async () => { + const companyId = randomUUID(); + const otherCompanyId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const failedServiceId = randomUUID(); + const failedOperationId = randomUUID(); + const failedAt = new Date("2026-08-11T14:00:00.000Z"); + + await db.insert(companies).values([ + { + id: companyId, + name: "Paperclip", + issuePrefix: "PAP", + requireBoardApprovalForNewAgents: false, + }, + { + id: otherCompanyId, + name: "OtherCo", + issuePrefix: "OTH", + requireBoardApprovalForNewAgents: false, + }, + ]); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Inherited runtime projection", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + sourceType: "local_path", + isPrimary: true, + cwd: "/tmp/inherited-runtime-projection", + metadata: { + runtimeConfig: { + workspaceRuntime: { + services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }], + }, + desiredState: "stopped", + }, + }, + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "Inherited isolated workspace", + status: "idle", + providerType: "git_worktree", + cwd: "/tmp/inherited-runtime-projection/isolated", + metadata: { + config: { + desiredState: "running", + serviceStates: { "0": "manual" }, + }, + }, + }); + await db.insert(workspaceRuntimeServices).values([ + { + id: randomUUID(), + companyId, + projectId, + projectWorkspaceId, + executionWorkspaceId, + scopeType: "execution_workspace", + scopeId: executionWorkspaceId, + serviceName: "web", + status: "stopped", + lifecycle: "shared", + reuseKey: "inherited-web", + command: "pnpm dev", + cwd: "/tmp/inherited-runtime-projection/isolated", + provider: "local_process", + healthStatus: "unknown", + updatedAt: new Date("2026-08-10T14:00:00.000Z"), + }, + { + id: failedServiceId, + companyId, + projectId, + projectWorkspaceId, + executionWorkspaceId, + scopeType: "execution_workspace", + scopeId: executionWorkspaceId, + serviceName: "web", + status: "failed", + lifecycle: "shared", + reuseKey: "inherited-web", + command: "pnpm dev", + cwd: "/tmp/inherited-runtime-projection/isolated", + provider: "local_process", + healthStatus: "unhealthy", + updatedAt: failedAt, + }, + ]); + await db.insert(workspaceOperations).values([ + { + id: failedOperationId, + companyId, + executionWorkspaceId, + phase: "workspace_provision", + command: "pnpm dev", + status: "failed", + metadata: { + action: "start", + serviceIndex: 0, + workspaceCommandId: "service:web", + failureEvidence: { + code: "workspace_runtime_start_failed", + message: "Workspace runtime service failed to start.", + remediation: "Review the workspace operation log and retry.", + details: { port: 3100 }, + }, + }, + startedAt: failedAt, + finishedAt: failedAt, + }, + { + companyId: otherCompanyId, + executionWorkspaceId, + phase: "workspace_provision", + command: "pnpm dev", + status: "failed", + metadata: { + action: "start", + serviceIndex: 0, + workspaceCommandId: "service:web", + failureEvidence: { + code: "other_company_failure", + message: "Must not leak.", + remediation: "Must not leak.", + }, + }, + startedAt: new Date("2026-08-11T15:00:00.000Z"), + finishedAt: new Date("2026-08-11T15:00:00.000Z"), + }, + ]); + + const workspace = await svc.getById(executionWorkspaceId); + + expect(workspace?.effectiveRuntimeConfig).toEqual({ + workspaceRuntime: { + services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }], + }, + source: { type: "project_workspace", id: projectWorkspaceId }, + desiredState: "running", + serviceStates: { "0": "manual" }, + }); + expect(workspace?.runtimeServices).toEqual([ + expect.objectContaining({ + id: failedServiceId, + status: "failed", + actualState: "failed", + desiredState: "manual", + workspaceCommandId: "service:web", + latestFailure: { + operationId: failedOperationId, + operationLogPath: `/api/workspace-operations/${failedOperationId}/log`, + code: "workspace_runtime_start_failed", + message: "Workspace runtime service failed to start.", + remediation: "Review the workspace operation log and retry.", + details: { port: 3100 }, + failedAt, + }, + }), + ]); + }); + it("returns a bounded company-scoped workspace overview with service and linked issue summaries", async () => { const companyId = randomUUID(); const otherCompanyId = randomUUID(); @@ -4012,7 +4276,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { metadata: { config: { workspaceRuntime: { - services: [{ name: "web", command: "pnpm dev" }], + services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }], }, }, }, @@ -4148,7 +4412,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => { projectUrlKey: "workspaces", projectName: "Workspaces", branchName: "paperclip/a", - serviceCount: 2, + serviceCount: 1, runningServiceCount: 1, primaryServiceUrl: "http://localhost:3100", primaryServiceUrlRunning: true, diff --git a/server/src/__tests__/issues-goal-context-routes.test.ts b/server/src/__tests__/issues-goal-context-routes.test.ts index 55a064ebcd6..810e32095d2 100644 --- a/server/src/__tests__/issues-goal-context-routes.test.ts +++ b/server/src/__tests__/issues-goal-context-routes.test.ts @@ -298,6 +298,8 @@ describe.sequential("issue goal context routes", () => { scopeType: "execution_workspace", scopeId: workspaceId, serviceName: "Dev server", + actualState: "stopped", + desiredState: "running", lifecycle: "shared", reuseKey: "dev-server", command: "pnpm dev", @@ -313,6 +315,8 @@ describe.sequential("issue goal context routes", () => { stoppedAt: null, healthStatus: "healthy", configIndex: 0, + workspaceCommandId: "service:dev-server", + latestFailure: null, metadata: { huge: "runtime metadata should not be embedded" }, }; mockIssueService.getById.mockResolvedValueOnce({ @@ -402,10 +406,21 @@ describe.sequential("issue goal context routes", () => { desiredState: null, serviceStates: null, }, + effectiveRuntimeConfig: { + workspaceRuntime: { services: [{ name: "Dev server", command: "pnpm dev" }] }, + source: { type: "execution_workspace", id: workspaceId }, + desiredState: "running", + serviceStates: null, + }, metadata: { huge: "execution workspace metadata should not be embedded" }, runtimeServices: [ - { ...runtimeServiceBase, id: "service-running", status: "running" }, - { ...runtimeServiceBase, id: "service-stopped", status: "stopped" }, + { + ...runtimeServiceBase, + id: "service-failed", + status: "failed", + actualState: "failed", + healthStatus: "unhealthy", + }, ], createdAt: new Date("2026-03-24T12:00:00Z"), updatedAt: new Date("2026-03-24T12:00:00Z"), @@ -422,10 +437,16 @@ describe.sequential("issue goal context routes", () => { expect(res.body.currentExecutionWorkspace.metadata).toBeNull(); expect(res.body.currentExecutionWorkspace.runtimeServices).toHaveLength(1); expect(res.body.currentExecutionWorkspace.runtimeServices[0]).toMatchObject({ - id: "service-running", - status: "running", + id: "service-failed", + status: "failed", + actualState: "failed", + desiredState: "running", url: "http://localhost:3100", }); + expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({ + type: "execution_workspace", + id: workspaceId, + }); expect(res.body.currentExecutionWorkspace.runtimeServices[0]).not.toHaveProperty("metadata"); }); @@ -512,13 +533,31 @@ describe.sequential("issue goal context routes", () => { mode: "isolated_workspace", status: "active", cwd: "/tmp/pap-581", + effectiveRuntimeConfig: { + workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] }, + source: { type: "project_workspace", id: "project-workspace-1" }, + desiredState: "running", + serviceStates: null, + }, runtimeServices: [ { id: "service-1", serviceName: "web", - status: "running", - url: "http://127.0.0.1:5173", - healthStatus: "healthy", + status: "failed", + actualState: "failed", + desiredState: "running", + url: null, + healthStatus: "unhealthy", + workspaceCommandId: "service:web", + latestFailure: { + operationId: "operation-1", + operationLogPath: "/api/workspace-operations/operation-1/log", + code: "workspace_runtime_start_failed", + message: "Workspace runtime service failed to start.", + remediation: "Review the workspace operation log and retry.", + details: null, + failedAt: new Date("2026-08-11T12:00:00.000Z"), + }, }, ], }); @@ -535,9 +574,15 @@ describe.sequential("issue goal context routes", () => { runtimeServices: [ expect.objectContaining({ serviceName: "web", - url: "http://127.0.0.1:5173", + actualState: "failed", + desiredState: "running", + latestFailure: expect.objectContaining({ operationId: "operation-1" }), }), ], })); + expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({ + type: "project_workspace", + id: "project-workspace-1", + }); }); }); diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index ec050549ac0..b3a5c36d322 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -5981,6 +5981,70 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () => } }, 20_000); + it("keeps one configured row failed after repeated readiness failures", async () => { + const fixture = await createRuntimeFixture(); + const cleanupRuntimeHome = await createRuntimeHome(); + const workspace = fixture.workspaces[0]!; + const port = await findFreePort(); + const command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`; + const config = fixedPortRuntimeConfig(port, command); + + try { + await expect(startRuntimeServicesForWorkspaceControl({ + db, + actor: fixture.actor, + issue: null, + workspace: fixture.realizedWorkspace(workspace), + executionWorkspaceId: workspace.id, + config, + adapterEnv: {}, + })).rejects.toThrow(/failed to start runtime service|exited before readiness/i); + + const firstRows = await db + .select() + .from(workspaceRuntimeServices) + .where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id)); + expect(firstRows).toEqual([ + expect.objectContaining({ + serviceName: "web", + command, + status: "failed", + healthStatus: "unhealthy", + }), + ]); + + await expect(startRuntimeServicesForWorkspaceControl({ + db, + actor: fixture.actor, + issue: null, + workspace: fixture.realizedWorkspace(workspace), + executionWorkspaceId: workspace.id, + config, + adapterEnv: {}, + })).rejects.toThrow(/failed to start runtime service|exited before readiness/i); + + const retryRows = await db + .select() + .from(workspaceRuntimeServices) + .where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id)); + expect(retryRows).toEqual([ + expect.objectContaining({ + id: firstRows[0]!.id, + status: "failed", + healthStatus: "unhealthy", + }), + ]); + } finally { + await stopRuntimeServicesForExecutionWorkspace({ + db, + executionWorkspaceId: workspace.id, + workspaceCwd: workspace.cwd, + }).catch(() => undefined); + await cleanupRuntimeHome(); + await fixture.cleanup(); + } + }, 20_000); + it("returns a bounded structured conflict and exposes only same-company workspace references", async () => { const fixture = await createRuntimeFixture({ workspaceModes: ["isolated_workspace", "isolated_workspace"], diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index 77f13d4eab0..ab80dea092b 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -38,9 +38,29 @@ import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-ru import { appendWithCap } from "../adapters/utils.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { HttpError } from "../errors.js"; +import { getWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js"; const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024; +function toWorkspaceRuntimeOperationHttpError(error: unknown) { + const evidence = getWorkspaceOperationFailureEvidence(error); + if (!evidence) return error; + + return new HttpError( + error instanceof HttpError ? error.status : 500, + error instanceof HttpError ? error.message : evidence.message, + { + code: evidence.code, + remediation: evidence.remediation, + operationId: evidence.operationId, + operationLogPath: evidence.operationLogPath, + failedAt: evidence.failedAt, + ...(evidence.details ?? {}), + }, + ); +} + export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) { const router = Router(); const svc = executionWorkspaceService(db); @@ -456,6 +476,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P }, }; }, + }).catch((error) => { + throw toWorkspaceRuntimeOperationHttpError(error); }); const workspace = await svc.getById(id); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 50ae72bda7b..fec354ada8f 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -5303,6 +5303,8 @@ export function issueRoutes( scopeId: service.scopeId, serviceName: service.serviceName, status: service.status, + actualState: service.actualState, + desiredState: service.desiredState, lifecycle: service.lifecycle, reuseKey: service.reuseKey, command: service.command, @@ -5318,6 +5320,8 @@ export function issueRoutes( stoppedAt: service.stoppedAt, healthStatus: service.healthStatus, configIndex: service.configIndex ?? null, + workspaceCommandId: service.workspaceCommandId ?? null, + latestFailure: service.latestFailure, }; } @@ -5358,12 +5362,9 @@ export function issueRoutes( serviceStates: workspace.config.serviceStates, } : null, + effectiveRuntimeConfig: workspace.effectiveRuntimeConfig, metadata: null, - runtimeServices: (workspace.runtimeServices ?? []) - .filter((service) => - service.status === "provisioning" || service.status === "starting" || service.status === "running" - ) - .map(compactIssueRuntimeService), + runtimeServices: (workspace.runtimeServices ?? []).map(compactIssueRuntimeService), createdAt: workspace.createdAt, updatedAt: workspace.updatedAt, }; diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4e32a69a37f..e864bc176c2 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -13,9 +13,11 @@ import { issues, projects, projectWorkspaces, + workspaceOperations, workspaceRuntimeServices, } from "@paperclipai/db"; import type { + EffectiveWorkspaceRuntimeConfig, ExecutionWorkspace, ExecutionWorkspaceDeliveryState, ExecutionWorkspaceSummary, @@ -27,6 +29,7 @@ import type { WorkspaceOverviewItem, WorkspaceOverviewLinkedIssue, WorkspaceRuntimeDesiredState, + WorkspaceRuntimeFailureEvidence, WorkspaceRuntimeService, WorkspaceOverviewPrimaryService, WorkspaceOverviewQuery, @@ -61,10 +64,22 @@ import { type ExecutionWorkspaceRow = typeof executionWorkspaces.$inferSelect; type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect; -type RuntimeServiceReadDb = Pick; +type RuntimeServiceReadDb = Pick; type DbTransaction = Parameters[0]>[0]; +type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & { + configIndex?: number | null; + workspaceCommandId?: string | null; + desiredState?: WorkspaceRuntimeDesiredState | null; + latestFailure?: WorkspaceRuntimeFailureEvidence | null; +}; +type ProjectedWorkspaceRuntimeFailure = { + serviceIndex: number | null; + workspaceCommandId: string | null; + evidence: WorkspaceRuntimeFailureEvidence; +}; const execFileAsync = promisify(execFile); const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]); +const ACTIVE_RUNTIME_SERVICE_STATUSES = new Set(["provisioning", "starting", "running"]); const WORKSPACE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence"; const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed"; export const ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON = "issue_terminal"; @@ -627,7 +642,9 @@ function assertBranchReconcileRuntimeServicesStopped(input: { inspection: ExecutionWorkspaceBranchReconcileInspection; runtimeServices: WorkspaceRuntimeService[]; }) { - const activeRuntimeServices = input.runtimeServices.filter((service) => service.status !== "stopped"); + const activeRuntimeServices = input.runtimeServices.filter((service) => + ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState) + ); if (activeRuntimeServices.length > 0) { throw unprocessable("Execution workspace branch reconciliation requires all runtime services to be stopped", { inspection: input.inspection, @@ -952,8 +969,9 @@ export function mergeExecutionWorkspaceConfig( } function toRuntimeService( - row: WorkspaceRuntimeServiceRow & { configIndex?: number | null }, + row: ProjectedWorkspaceRuntimeServiceRow, ): WorkspaceRuntimeService { + const actualState = row.status as WorkspaceRuntimeService["actualState"]; return { id: row.id, companyId: row.companyId, @@ -964,7 +982,9 @@ function toRuntimeService( scopeType: row.scopeType as WorkspaceRuntimeService["scopeType"], scopeId: row.scopeId ?? null, serviceName: row.serviceName, - status: row.status as WorkspaceRuntimeService["status"], + status: actualState, + actualState, + desiredState: row.desiredState ?? null, lifecycle: row.lifecycle as WorkspaceRuntimeService["lifecycle"], reuseKey: row.reuseKey ?? null, command: row.command ?? null, @@ -981,6 +1001,8 @@ function toRuntimeService( stopPolicy: (row.stopPolicy as Record | null) ?? null, healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"], configIndex: row.configIndex ?? null, + workspaceCommandId: row.workspaceCommandId ?? null, + latestFailure: row.latestFailure ?? null, createdAt: row.createdAt, updatedAt: row.updatedAt, }; @@ -990,7 +1012,9 @@ function toExecutionWorkspace( row: ExecutionWorkspaceRow, runtimeServices: WorkspaceRuntimeService[] = [], deliveryState: ExecutionWorkspaceDeliveryState = "unknown", + effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null, ): ExecutionWorkspace { + const config = readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null); return { id: row.id, companyId: row.companyId, @@ -1014,7 +1038,17 @@ function toExecutionWorkspace( closedAt: row.closedAt ?? null, cleanupEligibleAt: row.cleanupEligibleAt ?? null, cleanupReason: row.cleanupReason ?? null, - config: readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null), + config, + effectiveRuntimeConfig: effectiveRuntimeConfig === undefined + ? config?.workspaceRuntime + ? { + workspaceRuntime: config.workspaceRuntime, + source: { type: "execution_workspace", id: row.id }, + desiredState: config.desiredState, + serviceStates: config.serviceStates ?? null, + } + : null + : effectiveRuntimeConfig, metadata: (row.metadata as Record | null) ?? null, runtimeServices, createdAt: row.createdAt, @@ -1075,6 +1109,11 @@ function usesInheritedProjectRuntimeServices(row: ExecutionWorkspaceRow) { return !readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null)?.workspaceRuntime; } +function inheritsProjectRuntimeConfig(row: ExecutionWorkspaceRow) { + if (!row.projectWorkspaceId) return false; + return !readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null)?.workspaceRuntime; +} + function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow) { const inheritedProjectWorkspaceId = usesInheritedProjectRuntimeServices(row) ? row.projectWorkspaceId : null; const activeServiceConditions = inheritedProjectWorkspaceId @@ -1087,22 +1126,124 @@ function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow ), eq(workspaceRuntimeServices.executionWorkspaceId, row.id), ), - ne(workspaceRuntimeServices.status, "stopped"), + inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]), ) : and( eq(workspaceRuntimeServices.companyId, row.companyId), eq(workspaceRuntimeServices.executionWorkspaceId, row.id), - ne(workspaceRuntimeServices.status, "stopped"), + inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]), ); return sql`not exists (select 1 from ${workspaceRuntimeServices} where ${activeServiceConditions})`; } +function readWorkspaceRuntimeFailureEvidence( + operation: { + id: string; + metadata: Record | null; + startedAt: Date; + finishedAt: Date | null; + }, +): WorkspaceRuntimeFailureEvidence { + const raw = isRecord(operation.metadata?.failureEvidence) + ? operation.metadata.failureEvidence + : null; + const rawDetails = isRecord(raw?.details) ? raw.details : null; + const details: NonNullable = {}; + if (typeof rawDetails?.port === "number" && Number.isInteger(rawDetails.port)) details.port = rawDetails.port; + if (typeof rawDetails?.attemptedPortCount === "number" && Number.isInteger(rawDetails.attemptedPortCount)) { + details.attemptedPortCount = rawDetails.attemptedPortCount; + } + if (typeof rawDetails?.conflictingExecutionWorkspaceId === "string") { + details.conflictingExecutionWorkspaceId = rawDetails.conflictingExecutionWorkspaceId; + } + if (typeof rawDetails?.conflictingProjectWorkspaceId === "string") { + details.conflictingProjectWorkspaceId = rawDetails.conflictingProjectWorkspaceId; + } + return { + operationId: operation.id, + operationLogPath: `/api/workspace-operations/${operation.id}/log`, + code: typeof raw?.code === "string" ? raw.code : "workspace_runtime_start_failed", + message: typeof raw?.message === "string" ? raw.message : "Workspace runtime service failed to start.", + remediation: typeof raw?.remediation === "string" + ? raw.remediation + : "Review the workspace operation log, correct the runtime configuration or dependency, and retry.", + details: Object.keys(details).length > 0 ? details : null, + failedAt: operation.finishedAt ?? operation.startedAt, + }; +} + +async function loadLatestRuntimeFailuresByExecutionWorkspace( + db: RuntimeServiceReadDb, + companyId: string, + runtimeRowsByWorkspaceId: Map, +) { + const configuredRows = [...runtimeRowsByWorkspaceId.entries()].flatMap(([executionWorkspaceId, rows]) => + rows.flatMap((row) => row.configIndex === undefined || row.configIndex === null + ? [] + : [{ executionWorkspaceId, configIndex: row.configIndex }]), + ); + if (configuredRows.length === 0) { + return new Map(); + } + + const executionWorkspaceIds = [...new Set(configuredRows.map((row) => row.executionWorkspaceId))]; + const serviceIndexes = [...new Set(configuredRows.map((row) => String(row.configIndex)))]; + const serviceIndexExpression = sql`${workspaceOperations.metadata}->>'serviceIndex'`; + const actionExpression = sql`${workspaceOperations.metadata}->>'action'`; + const operations = await db + .selectDistinctOn( + [workspaceOperations.executionWorkspaceId, serviceIndexExpression], + { + id: workspaceOperations.id, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + serviceIndex: serviceIndexExpression, + workspaceCommandId: sql`${workspaceOperations.metadata}->>'workspaceCommandId'`, + metadata: workspaceOperations.metadata, + startedAt: workspaceOperations.startedAt, + finishedAt: workspaceOperations.finishedAt, + }, + ) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray(workspaceOperations.executionWorkspaceId, executionWorkspaceIds), + eq(workspaceOperations.status, "failed"), + inArray(actionExpression, ["start", "restart"]), + or(inArray(serviceIndexExpression, serviceIndexes), isNull(serviceIndexExpression)), + ), + ) + .orderBy( + workspaceOperations.executionWorkspaceId, + serviceIndexExpression, + desc(workspaceOperations.startedAt), + desc(workspaceOperations.createdAt), + ); + + const result = new Map(); + for (const operation of operations) { + if (!operation.executionWorkspaceId) continue; + const serviceIndex = operation.serviceIndex === null + ? null + : Number.parseInt(operation.serviceIndex, 10); + if (serviceIndex !== null && !Number.isInteger(serviceIndex)) continue; + const existing = result.get(operation.executionWorkspaceId) ?? []; + existing.push({ + serviceIndex, + workspaceCommandId: operation.workspaceCommandId, + evidence: readWorkspaceRuntimeFailureEvidence(operation), + }); + result.set(operation.executionWorkspaceId, existing); + } + return result; +} + async function loadEffectiveRuntimeServicesByExecutionWorkspace( db: RuntimeServiceReadDb, companyId: string, rows: ExecutionWorkspaceRow[], ) { - const inheritedRows = rows.filter((row) => usesInheritedProjectRuntimeServices(row)); + const inheritedRows = rows.filter((row) => inheritsProjectRuntimeConfig(row)); const projectWorkspaceIds = inheritedRows .map((row) => row.projectWorkspaceId) .filter((value): value is string => Boolean(value)); @@ -1136,8 +1277,7 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace( const projectRuntimeConfigByWorkspaceId = new Map( projectWorkspaceRows.map((row) => [ row.id, - readProjectWorkspaceRuntimeConfig((row.metadata as Record | null) ?? null)?.workspaceRuntime - ?? null, + readProjectWorkspaceRuntimeConfig((row.metadata as Record | null) ?? null), ]), ); const effectiveProjectRuntimeServices = new Map( @@ -1145,35 +1285,113 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace( projectWorkspaceId, selectConfiguredRuntimeServiceRows( projectRuntimeServices.get(projectWorkspaceId) ?? [], - projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId) ?? null, + projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.workspaceRuntime ?? null, + projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.desiredState ?? null, + projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.serviceStates ?? null, ), ]), ); - return new Map( - rows.map((row) => { - if (!usesInheritedProjectRuntimeServices(row)) { - return [row.id, executionRuntimeServices.get(row.id) ?? []] as const; - } + const runtimeRowsByWorkspaceId = new Map(); + const effectiveConfigByWorkspaceId = new Map(); + for (const row of rows) { + const localConfig = readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null); + const projectConfig = row.projectWorkspaceId + ? projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId) ?? null + : null; + const workspaceRuntime = localConfig?.workspaceRuntime ?? projectConfig?.workspaceRuntime ?? null; + const desiredState = localConfig?.desiredState ?? projectConfig?.desiredState ?? null; + const serviceStates = localConfig?.serviceStates ?? projectConfig?.serviceStates ?? null; + effectiveConfigByWorkspaceId.set( + row.id, + workspaceRuntime + ? { + workspaceRuntime, + source: localConfig?.workspaceRuntime + ? { type: "execution_workspace", id: row.id } + : { type: "project_workspace", id: row.projectWorkspaceId! }, + desiredState, + serviceStates, + } + : null, + ); - const workspaceRuntime = projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId!) ?? null; - const executionScopedRows = selectConfiguredRuntimeServiceRows( - (executionRuntimeServices.get(row.id) ?? []).filter( - (runtimeService) => runtimeService.scopeType !== "project_workspace", + if (!workspaceRuntime) { + runtimeRowsByWorkspaceId.set( + row.id, + usesInheritedProjectRuntimeServices(row) + ? [] + : (executionRuntimeServices.get(row.id) ?? []).map((runtimeService) => ({ + ...runtimeService, + desiredState, + })), + ); + continue; + } + + if (!usesInheritedProjectRuntimeServices(row)) { + runtimeRowsByWorkspaceId.set( + row.id, + selectConfiguredRuntimeServiceRows( + executionRuntimeServices.get(row.id) ?? [], + workspaceRuntime, + desiredState, + serviceStates, ), - workspaceRuntime, ); - const effectiveRows = [ - ...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []), - ...executionScopedRows, - ].sort( - (left, right) => - (left.configIndex ?? Number.MAX_SAFE_INTEGER) - - (right.configIndex ?? Number.MAX_SAFE_INTEGER), - ); - return [row.id, effectiveRows] as const; - }), + continue; + } + + const executionScopedRows = selectConfiguredRuntimeServiceRows( + (executionRuntimeServices.get(row.id) ?? []).filter( + (runtimeService) => runtimeService.scopeType !== "project_workspace", + ), + workspaceRuntime, + desiredState, + serviceStates, + ); + const effectiveRows = [ + ...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []).map((runtimeService) => ({ + ...runtimeService, + desiredState: + (runtimeService.configIndex === null || runtimeService.configIndex === undefined + ? null + : serviceStates?.[String(runtimeService.configIndex)]) + ?? desiredState, + })), + ...executionScopedRows, + ].sort( + (left, right) => + (left.configIndex ?? Number.MAX_SAFE_INTEGER) - + (right.configIndex ?? Number.MAX_SAFE_INTEGER), + ); + runtimeRowsByWorkspaceId.set(row.id, effectiveRows); + } + + const failuresByWorkspaceId = await loadLatestRuntimeFailuresByExecutionWorkspace( + db, + companyId, + runtimeRowsByWorkspaceId, ); + + return new Map(rows.map((row) => { + const failures = failuresByWorkspaceId.get(row.id) ?? []; + const runtimeServices = (runtimeRowsByWorkspaceId.get(row.id) ?? []).map((runtimeService) => { + if (runtimeService.configIndex === undefined || runtimeService.configIndex === null) return runtimeService; + const candidates = failures.filter((failure) => + (failure.serviceIndex === runtimeService.configIndex || failure.serviceIndex === null) + && (!failure.workspaceCommandId || failure.workspaceCommandId === runtimeService.workspaceCommandId), + ); + const latestFailure = candidates + .sort((left, right) => right.evidence.failedAt.getTime() - left.evidence.failedAt.getTime())[0] + ?.evidence ?? null; + return { ...runtimeService, latestFailure }; + }); + return [row.id, { + runtimeServices, + effectiveRuntimeConfig: effectiveConfigByWorkspaceId.get(row.id) ?? null, + }] as const; + })); } type WorkspaceOverviewPageRow = ExecutionWorkspaceRow & { @@ -1376,11 +1594,15 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } - async function hydrateWorkspace(row: ExecutionWorkspaceRow, runtimeServices: WorkspaceRuntimeService[] = []) { - const workspace = toExecutionWorkspace(row, runtimeServices); + async function hydrateWorkspace( + row: ExecutionWorkspaceRow, + runtimeServices: WorkspaceRuntimeService[] = [], + effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null, + ) { + const workspace = toExecutionWorkspace(row, runtimeServices, "unknown", effectiveRuntimeConfig); const { git } = await inspectGitCloseReadiness(workspace); const assessment = await assessDelivery(row, git); - return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState); + return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState, effectiveRuntimeConfig); } async function workspaceHasActiveRun(workspace: Pick) { @@ -1903,7 +2125,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } const items: WorkspaceOverviewItem[] = pageRows.map((row) => { - const runtimeServices = (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService); + const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id); + const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService); const runningServiceCount = runtimeServices.filter((service) => service.status === "running").length; const primaryService = selectPrimaryOverviewService(runtimeServices); const config = readExecutionWorkspaceConfig((row.metadata as Record | null) ?? null); @@ -1971,12 +2194,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic .where(and(...conditions)) .orderBy(desc(executionWorkspaces.lastUsedAt), desc(executionWorkspaces.createdAt)); const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, companyId, rows); - return Promise.all(rows.map((row) => - hydrateWorkspace( + return Promise.all(rows.map((row) => { + const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id); + return hydrateWorkspace( row, - (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService), - ), - )); + (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService), + runtimeProjection?.effectiveRuntimeConfig ?? null, + ); + })); }, listSummaries: async (companyId: string, filters?: { @@ -2130,9 +2355,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic .then((rows) => rows[0] ?? null); if (!row) return null; const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, row.companyId, [row]); + const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id); return hydrateWorkspace( row, - (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService), + (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService), + runtimeProjection?.effectiveRuntimeConfig ?? null, ); }, @@ -2145,7 +2372,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic if (!workspace) return null; const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, workspace.companyId, [workspace]); - const runtimeServices = (runtimeServicesByWorkspaceId.get(workspace.id) ?? []).map(toRuntimeService); + const runtimeProjection = runtimeServicesByWorkspaceId.get(workspace.id); + const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService); const linkedIssues = await db .select({ @@ -2240,7 +2468,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic warnings.push("This shared workspace session points at project workspace infrastructure. Archiving it only removes the session record."); } - if (runtimeServices.some((service) => service.status !== "stopped")) { + if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) { warnings.push( runtimeServices.length === 1 ? "Closing this workspace will stop 1 attached runtime service." @@ -2291,7 +2519,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic }, ]; - if (runtimeServices.some((service) => service.status !== "stopped")) { + if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) { plannedActions.push({ kind: "stop_runtime_services", label: runtimeServices.length === 1 ? "Stop attached runtime service" : "Stop attached runtime services", @@ -3225,7 +3453,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic ); assertBranchReconcileRuntimeServicesStopped({ inspection, - runtimeServices: (runtimeServicesByWorkspaceId.get(existing.id) ?? []).map(toRuntimeService), + runtimeServices: ( + runtimeServicesByWorkspaceId.get(existing.id)?.runtimeServices ?? [] + ).map(toRuntimeService), }); // The git rescue has to happen before the DB transaction because the // transaction may be retried/rolled back, while git side effects cannot. @@ -3303,8 +3533,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic lockedRow.companyId, [lockedRow], ); - const lockedRuntimeServices = (lockedRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService); - const lockedWorkspace = toExecutionWorkspace(lockedRow, lockedRuntimeServices); + const lockedRuntimeProjection = lockedRuntimeServicesByWorkspaceId.get(lockedRow.id); + const lockedRuntimeServices = (lockedRuntimeProjection?.runtimeServices ?? []).map(toRuntimeService); + const lockedWorkspace = toExecutionWorkspace( + lockedRow, + lockedRuntimeServices, + "unknown", + lockedRuntimeProjection?.effectiveRuntimeConfig ?? null, + ); if (!lockedWorkspace.sourceIssueId) { throw unprocessable("Execution workspace needs a source issue before Paperclip can audit branch reconciliation"); } @@ -3352,7 +3588,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic lockedRow.companyId, [lockedRow], ); - const latestRuntimeServices = (latestRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService); + const latestRuntimeServices = ( + latestRuntimeServicesByWorkspaceId.get(lockedRow.id)?.runtimeServices ?? [] + ).map(toRuntimeService); assertBranchReconcileWorkspaceIsSafe({ workspaceStatus: lockedWorkspace.status, inspection, diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 5ac1b50e4c1..93f6d6ec2b5 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -24,11 +24,15 @@ import { type ProjectManagedByPlugin, type ProjectWorkspaceRuntimeConfig, type ProjectWorkspace, + type WorkspaceRuntimeDesiredState, type WorkspaceRuntimeService, type PluginManagedProjectDeclaration, type PluginManagedProjectResolution, } from "@paperclipai/shared"; -import { listCurrentRuntimeServicesForProjectWorkspaces } from "./workspace-runtime-read-model.js"; +import { + listCurrentRuntimeServicesForProjectWorkspaces, + selectConfiguredRuntimeServiceRows, +} from "./workspace-runtime-read-model.js"; import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-policy.js"; import { mergeProjectWorkspaceRuntimeConfig, readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js"; import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; @@ -36,6 +40,11 @@ import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; type ProjectRow = typeof projects.$inferSelect; type ProjectWorkspaceRow = typeof projectWorkspaces.$inferSelect; type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect; +type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & { + configIndex?: number | null; + workspaceCommandId?: string | null; + desiredState?: WorkspaceRuntimeDesiredState | null; +}; const REPO_ONLY_CWD_SENTINEL = "/__paperclip_repo_only__"; type CreateWorkspaceInput = { name?: string | null; @@ -117,7 +126,7 @@ async function attachGoals(db: Db, rows: ProjectRow[]): Promise | null) ?? null, healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"], + configIndex: row.configIndex ?? null, + workspaceCommandId: row.workspaceCommandId ?? null, + latestFailure: null, createdAt: row.createdAt, updatedAt: row.updatedAt, }; } +function projectWorkspaceRuntimeServices( + workspace: ProjectWorkspaceRow, + rows: WorkspaceRuntimeServiceRow[], +) { + const runtimeConfig = readProjectWorkspaceRuntimeConfig( + (workspace.metadata as Record | null) ?? null, + ); + return selectConfiguredRuntimeServiceRows( + rows, + runtimeConfig?.workspaceRuntime ?? null, + runtimeConfig?.desiredState ?? null, + runtimeConfig?.serviceStates ?? null, + ).map(toRuntimeService); +} + function toWorkspace( row: ProjectWorkspaceRow, runtimeServices: WorkspaceRuntimeService[] = [], @@ -246,9 +275,12 @@ async function attachWorkspaces(db: Db, rows: ProjectWithGoals[]): Promise workspace.id), ); const sharedRuntimeServicesByWorkspaceId = new Map( - Array.from(runtimeServicesByWorkspaceId.entries()).map(([workspaceId, services]) => [ - workspaceId, - services.map(toRuntimeService), + workspaceRows.map((workspace) => [ + workspace.id, + projectWorkspaceRuntimeServices( + workspace, + runtimeServicesByWorkspaceId.get(workspace.id) ?? [], + ), ]), ); @@ -891,7 +923,7 @@ export function projectService(db: Db) { return rows.map((row) => toWorkspace( row, - (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService), + projectWorkspaceRuntimeServices(row, runtimeServicesByWorkspaceId.get(row.id) ?? []), ), ); }, diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index 1c0c13a824f..2ca77a6743f 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -3,13 +3,109 @@ import type { Db } from "@paperclipai/db"; import { workspaceOperations } from "@paperclipai/db"; import type { WorkspaceOperation, WorkspaceOperationPhase, WorkspaceOperationStatus } from "@paperclipai/shared"; import { asc, desc, eq, inArray, isNull, or, and } from "drizzle-orm"; -import { notFound } from "../errors.js"; +import { HttpError, notFound } from "../errors.js"; import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js"; import { instanceSettingsService } from "./instance-settings.js"; import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js"; type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect; +export interface WorkspaceOperationFailureEvidence { + operationId: string; + operationLogPath: string; + code: string; + message: string; + remediation: string; + details: { + port?: number; + attemptedPortCount?: number; + conflictingExecutionWorkspaceId?: string; + conflictingProjectWorkspaceId?: string; + } | null; + failedAt: string; +} + +const WORKSPACE_OPERATION_FAILURE_PROPERTY = "paperclipWorkspaceOperationFailure"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function readSafeFailureDetails(value: unknown): WorkspaceOperationFailureEvidence["details"] { + if (!isRecord(value)) return null; + const details: NonNullable = {}; + if (typeof value.port === "number" && Number.isInteger(value.port)) details.port = value.port; + if (typeof value.attemptedPortCount === "number" && Number.isInteger(value.attemptedPortCount)) { + details.attemptedPortCount = value.attemptedPortCount; + } + if (typeof value.conflictingExecutionWorkspaceId === "string") { + details.conflictingExecutionWorkspaceId = value.conflictingExecutionWorkspaceId; + } + if (typeof value.conflictingProjectWorkspaceId === "string") { + details.conflictingProjectWorkspaceId = value.conflictingProjectWorkspaceId; + } + return Object.keys(details).length > 0 ? details : null; +} + +function buildWorkspaceOperationFailureEvidence(input: { + operationId: string; + failedAt: Date; + error: unknown; + metadata: Record | null | undefined; +}): WorkspaceOperationFailureEvidence { + const httpDetails = input.error instanceof HttpError && isRecord(input.error.details) + ? input.error.details + : null; + const action = typeof input.metadata?.action === "string" ? input.metadata.action : null; + const isRuntimeStart = action === "start" || action === "restart"; + const code = typeof httpDetails?.code === "string" + ? httpDetails.code + : isRuntimeStart + ? "workspace_runtime_start_failed" + : "workspace_operation_failed"; + const message = input.error instanceof HttpError + ? input.error.message + : isRuntimeStart + ? "Workspace runtime service failed to start." + : "Workspace operation failed."; + const remediation = typeof httpDetails?.remediation === "string" + ? httpDetails.remediation + : "Review the workspace operation log, correct the runtime configuration or dependency, and retry."; + + return { + operationId: input.operationId, + operationLogPath: `/api/workspace-operations/${input.operationId}/log`, + code, + message, + remediation, + details: readSafeFailureDetails(httpDetails), + failedAt: input.failedAt.toISOString(), + }; +} + +export function attachWorkspaceOperationFailureEvidence( + error: unknown, + evidence: WorkspaceOperationFailureEvidence, +) { + const target = error instanceof Error ? error : new Error(String(error)); + Object.defineProperty(target, WORKSPACE_OPERATION_FAILURE_PROPERTY, { + configurable: true, + enumerable: false, + value: evidence, + writable: true, + }); + return target; +} + +export function getWorkspaceOperationFailureEvidence( + error: unknown, +): WorkspaceOperationFailureEvidence | null { + if (!(error instanceof Error)) return null; + const value = (error as unknown as Record)[WORKSPACE_OPERATION_FAILURE_PROPERTY]; + if (!isRecord(value)) return null; + return value as unknown as WorkspaceOperationFailureEvidence; +} + function toWorkspaceOperation(row: WorkspaceOperationRow): WorkspaceOperation { return { id: row.id, @@ -187,6 +283,12 @@ export function workspaceOperationService(db: Db) { await append("stderr", error instanceof Error ? error.message : String(error)); const finalized = await logStore.finalize(handle).catch(() => null); const finishedAt = new Date(); + const failureEvidence = buildWorkspaceOperationFailureEvidence({ + operationId: id, + failedAt: finishedAt, + error, + metadata: recordInput.metadata, + }); await db .update(workspaceOperations) .set({ @@ -197,11 +299,15 @@ export function workspaceOperationService(db: Db) { logBytes: finalized?.bytes ?? null, logSha256: finalized?.sha256 ?? null, logCompressed: finalized?.compressed ?? false, + metadata: redactCurrentUserValue( + combineMetadata(recordInput.metadata, { failureEvidence }), + currentUserRedactionOptions, + ) as Record | null, finishedAt, updatedAt: finishedAt, }) .where(eq(workspaceOperations.id, id)); - throw error; + throw attachWorkspaceOperationFailureEvidence(error, failureEvidence); } }, }; diff --git a/server/src/services/workspace-runtime-read-model.test.ts b/server/src/services/workspace-runtime-read-model.test.ts index 1de4250b079..34c45e7504b 100644 --- a/server/src/services/workspace-runtime-read-model.test.ts +++ b/server/src/services/workspace-runtime-read-model.test.ts @@ -53,7 +53,7 @@ describe("selectConfiguredRuntimeServiceRows", () => { expect(selectConfiguredRuntimeServiceRows(historicalRows, null)).toEqual([]); }); - it("selects the newest current row for each configured service and annotates its config index", () => { + it("selects the newest configured row including failed state and annotates effective intent", () => { const oldWeb = runtimeServiceRow({ serviceName: "web", command: "pnpm dev", @@ -62,6 +62,8 @@ describe("selectConfiguredRuntimeServiceRows", () => { const currentWeb = runtimeServiceRow({ serviceName: "web", command: "pnpm dev", + status: "failed", + healthStatus: "unhealthy", updatedAt: new Date("2026-07-30T10:00:00.000Z"), }); const removedWorker = runtimeServiceRow({ @@ -73,13 +75,18 @@ describe("selectConfiguredRuntimeServiceRows", () => { const selected = selectConfiguredRuntimeServiceRows( [removedWorker, currentWeb, oldWeb], { services: [{ name: "web", command: "pnpm dev" }] }, + "stopped", + { "0": "running" }, ); expect(selected).toEqual([ expect.objectContaining({ id: currentWeb.id, serviceName: "web", + status: "failed", configIndex: 0, + workspaceCommandId: "service:web", + desiredState: "running", }), ]); }); diff --git a/server/src/services/workspace-runtime-read-model.ts b/server/src/services/workspace-runtime-read-model.ts index 471211fa434..b6fccd66f76 100644 --- a/server/src/services/workspace-runtime-read-model.ts +++ b/server/src/services/workspace-runtime-read-model.ts @@ -4,6 +4,7 @@ import { listWorkspaceServiceCommandDefinitions, matchWorkspaceRuntimeServiceToCommand, } from "@paperclipai/shared"; +import type { WorkspaceRuntimeDesiredState, WorkspaceRuntimeServiceStateMap } from "@paperclipai/shared"; import { and, desc, eq, inArray } from "drizzle-orm"; type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect; @@ -34,12 +35,20 @@ export function selectCurrentRuntimeServiceRows(rows: WorkspaceRuntimeServiceRow export function selectConfiguredRuntimeServiceRows( rows: WorkspaceRuntimeServiceRow[], workspaceRuntime: Record | null | undefined, + desiredState: WorkspaceRuntimeDesiredState | null = null, + serviceStates: WorkspaceRuntimeServiceStateMap | null = null, ) { const availableRows = selectCurrentRuntimeServiceRows(rows).map((row) => ({ ...row, configIndex: null as number | null, + workspaceCommandId: null as string | null, + desiredState: null as WorkspaceRuntimeDesiredState | null, })); - const selectedRows: Array = []; + const selectedRows: Array = []; for (const command of listWorkspaceServiceCommandDefinitions(workspaceRuntime)) { const reuseScope = command.rawConfig.reuseScope; @@ -60,6 +69,10 @@ export function selectConfiguredRuntimeServiceRows( selectedRows.push({ ...matchedRow, configIndex: command.serviceIndex, + workspaceCommandId: command.id, + desiredState: + (command.serviceIndex === null ? null : serviceStates?.[String(command.serviceIndex)]) + ?? desiredState, }); availableRows.splice(availableRows.indexOf(matchedRow), 1); } diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index f98fd019d1b..06a25749735 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -186,6 +186,24 @@ const runtimeProvisionByWorkspace = new Map>(); const DEFAULT_EXECUTE_PROCESS_OUTPUT_BYTES = 256 * 1024; export const WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS = 32; const ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES = ["provisioning", "starting", "running"] as const; +const RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY = "paperclipRuntimeServiceFailureRecord"; + +function attachRuntimeServiceFailureRecord(error: unknown, record: RuntimeServiceRecord) { + const target = error instanceof Error ? error : new Error(String(error)); + Object.defineProperty(target, RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY, { + configurable: true, + enumerable: false, + value: record, + writable: true, + }); + return target; +} + +function getRuntimeServiceFailureRecord(error: unknown): RuntimeServiceRecord | null { + if (!(error instanceof Error)) return null; + const value = (error as unknown as Record)[RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY]; + return value && typeof value === "object" ? value as RuntimeServiceRecord : null; +} class RuntimeServicePortBindCollision extends Error { readonly port: number; @@ -905,7 +923,12 @@ async function findActiveRuntimeServiceBlockingDirtyQuarantine(input: { scopeType: workspaceRuntimeServices.scopeType, }) .from(workspaceRuntimeServices) - .where(and(serviceScopeCondition, ne(workspaceRuntimeServices.status, "stopped"))) + .where( + and( + serviceScopeCondition, + inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES]), + ), + ) .orderBy(desc(workspaceRuntimeServices.updatedAt), desc(workspaceRuntimeServices.createdAt)) .limit(1); return service ?? null; @@ -4153,7 +4176,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: { eq(workspaceRuntimeServices.companyId, input.companyId), eq(workspaceRuntimeServices.reuseKey, input.reuseKey), eq(workspaceRuntimeServices.provider, "local_process"), - eq(workspaceRuntimeServices.status, "stopped"), + inArray(workspaceRuntimeServices.status, ["stopped", "failed"]), ), ) .orderBy(desc(workspaceRuntimeServices.updatedAt)) @@ -4175,7 +4198,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: { and( eq(workspaceRuntimeServices.companyId, input.companyId), eq(workspaceRuntimeServices.provider, "local_process"), - eq(workspaceRuntimeServices.status, "stopped"), + inArray(workspaceRuntimeServices.status, ["stopped", "failed"]), eq(workspaceRuntimeServices.scopeType, input.scopeType), scopeIdCondition, eq(workspaceRuntimeServices.serviceName, input.serviceName), @@ -4770,7 +4793,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P processGroupId: child.pid, }); } - record.status = "stopped"; + record.status = "failed"; record.healthStatus = "unhealthy"; record.lastUsedAt = new Date().toISOString(); record.stoppedAt = new Date().toISOString(); @@ -4856,28 +4879,54 @@ async function startLocalRuntimeService( ); const retryBindCollisions = fixedPortFallbackEnabled || portType === "auto"; const deferReadiness = Boolean(options?.deferReadiness && !fixedPortFallbackEnabled); + const identity = resolveRuntimeServiceReuseIdentity({ + service: input.service, + workspace: input.workspace, + agent: input.agent, + issue: input.issue, + adapterEnv: input.adapterEnv, + scopeType: input.scopeType, + scopeId: input.scopeId, + }); + if (!identity.command) throw new Error(`Runtime service "${identity.serviceName}" is missing command`); + let currentAttemptRecord = provisioningRecord ?? createProvisioningRuntimeServiceRecord(input, identity); + if (!provisioningRecord) { + currentAttemptRecord.status = "starting"; + await persistRuntimeServiceRecord(input.db, currentAttemptRecord); + } + let retryRuntimeServiceId = input.runtimeServiceId; try { for (let attempt = 0; attempt < WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS; attempt += 1) { try { + const previousAttemptRecord = currentAttemptRecord; started = await spawnLocalRuntimeService({ ...input, excludedPorts, - runtimeServiceId: provisioningRecord?.id ?? input.runtimeServiceId, + runtimeServiceId: provisioningRecord?.id ?? retryRuntimeServiceId, }); + retryRuntimeServiceId = started.record.id; + currentAttemptRecord = started.record; if (runtimeProvisionCommand) { await persistRuntimeServiceRecord(input.db, started.record); } - if (provisioningRecord && started.record.id !== provisioningRecord.id && input.db) { + if (started.record.id !== previousAttemptRecord.id && input.db) { await input.db .delete(workspaceRuntimeServices) - .where(eq(workspaceRuntimeServices.id, provisioningRecord.id)); + .where(eq(workspaceRuntimeServices.id, previousAttemptRecord.id)); } if (!deferReadiness) { await started.readiness; } return started; } catch (error) { + const failedRecord = started?.record ?? currentAttemptRecord; + const nowIso = new Date().toISOString(); + failedRecord.status = "failed"; + failedRecord.healthStatus = "unhealthy"; + failedRecord.lastUsedAt = nowIso; + failedRecord.stoppedAt = nowIso; + await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined); if (!(error instanceof RuntimeServicePortBindCollision) || !retryBindCollisions) throw error; excludedPorts.add(error.port); started = null; @@ -4899,15 +4948,15 @@ async function startLocalRuntimeService( remediation: "Retry the start or configure a different runtime service port.", }); } catch (error) { - if (!started && provisioningRecord && provisioningRecord.status === "starting") { + if (!started && currentAttemptRecord.status !== "failed") { const nowIso = new Date().toISOString(); - provisioningRecord.status = "failed"; - provisioningRecord.healthStatus = "unhealthy"; - provisioningRecord.lastUsedAt = nowIso; - provisioningRecord.stoppedAt = nowIso; - await persistRuntimeServiceRecord(input.db, provisioningRecord).catch(() => undefined); + currentAttemptRecord.status = "failed"; + currentAttemptRecord.healthStatus = "unhealthy"; + currentAttemptRecord.lastUsedAt = nowIso; + currentAttemptRecord.stoppedAt = nowIso; + await persistRuntimeServiceRecord(input.db, currentAttemptRecord).catch(() => undefined); } - throw error; + throw attachRuntimeServiceFailureRecord(error, currentAttemptRecord); } } @@ -4921,12 +4970,13 @@ function scheduleIdleStop(record: RuntimeServiceRecord) { }, idleSeconds * 1000); } -async function stopRuntimeService(serviceId: string) { +async function stopRuntimeService(serviceId: string, options?: { preserveFailure?: boolean }) { const record = runtimeServicesById.get(serviceId); if (!record) return; clearIdleTimer(record); - record.status = "stopped"; - record.healthStatus = "unknown"; + const preserveFailure = options?.preserveFailure === true && record.status === "failed"; + record.status = preserveFailure ? "failed" : "stopped"; + record.healthStatus = preserveFailure ? "unhealthy" : "unknown"; record.lastUsedAt = new Date().toISOString(); record.stoppedAt = new Date().toISOString(); runtimeServicesById.delete(serviceId); @@ -5488,7 +5538,16 @@ export async function startRuntimeServicesForWorkspaceControl( }); } catch (error) { for (const serviceId of startBatch.startedServiceIds) { - await stopRuntimeService(serviceId).catch(() => undefined); + await stopRuntimeService(serviceId, { preserveFailure: true }).catch(() => undefined); + } + const failedRecord = getRuntimeServiceFailureRecord(error); + if (failedRecord) { + const nowIso = new Date().toISOString(); + failedRecord.status = "failed"; + failedRecord.healthStatus = "unhealthy"; + failedRecord.lastUsedAt = nowIso; + failedRecord.stoppedAt = nowIso; + await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined); } if (preparedProvisioning && startBatch.startedServiceIds.length === 0) { const nowIso = new Date().toISOString();