mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-27 11:20:42 +01:00
fix(runtime): repair database runtime projections
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
54b4e0c2fb
commit
db2893438d
16 files changed
+1155
-92
No files matched your search
@@ -844,6 +844,7 @@ export type {
|
||||
ExecutionWorkspaceCloseReadiness,
|
||||
ExecutionWorkspaceCloseReadinessState,
|
||||
ExecutionWorkspaceDeliveryState,
|
||||
EffectiveWorkspaceRuntimeConfig,
|
||||
WorkspaceOverviewItem,
|
||||
WorkspaceOverviewLinkedIssue,
|
||||
WorkspaceOverviewPrimaryService,
|
||||
@@ -852,6 +853,10 @@ export type {
|
||||
WorkspaceCommandDefinition,
|
||||
WorkspaceCommandKind,
|
||||
WorkspaceRuntimeControlTarget,
|
||||
WorkspaceRuntimeActualState,
|
||||
WorkspaceRuntimeConfigSource,
|
||||
WorkspaceRuntimeFailureDetails,
|
||||
WorkspaceRuntimeFailureEvidence,
|
||||
WorkspaceRuntimeService,
|
||||
WorkspaceRuntimeServiceStateMap,
|
||||
WorkspaceOperation,
|
||||
|
||||
@@ -366,6 +366,7 @@ export type {
|
||||
ExecutionWorkspaceCloseReadiness,
|
||||
ExecutionWorkspaceCloseReadinessState,
|
||||
ExecutionWorkspaceDeliveryState,
|
||||
EffectiveWorkspaceRuntimeConfig,
|
||||
WorkspaceOverviewItem,
|
||||
WorkspaceOverviewLinkedIssue,
|
||||
WorkspaceOverviewPrimaryService,
|
||||
@@ -374,6 +375,10 @@ export type {
|
||||
WorkspaceCommandDefinition,
|
||||
WorkspaceCommandKind,
|
||||
WorkspaceRuntimeControlTarget,
|
||||
WorkspaceRuntimeActualState,
|
||||
WorkspaceRuntimeConfigSource,
|
||||
WorkspaceRuntimeFailureDetails,
|
||||
WorkspaceRuntimeFailureEvidence,
|
||||
WorkspaceRuntimeService,
|
||||
WorkspaceRuntimeServiceStateMap,
|
||||
WorkspaceRuntimeDesiredState,
|
||||
|
||||
@@ -58,6 +58,41 @@ export type ExecutionWorkspaceCloseActionKind =
|
||||
export type WorkspaceRuntimeDesiredState = "running" | "stopped" | "manual";
|
||||
export type WorkspaceRuntimeServiceStateMap = Record<string, WorkspaceRuntimeDesiredState>;
|
||||
export type WorkspaceCommandKind = "service" | "job";
|
||||
export type WorkspaceRuntimeActualState =
|
||||
| "provisioning"
|
||||
| "starting"
|
||||
| "running"
|
||||
| "stopped"
|
||||
| "failed";
|
||||
|
||||
export interface WorkspaceRuntimeConfigSource {
|
||||
type: "execution_workspace" | "project_workspace";
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface EffectiveWorkspaceRuntimeConfig {
|
||||
workspaceRuntime: Record<string, unknown>;
|
||||
source: WorkspaceRuntimeConfigSource;
|
||||
desiredState: WorkspaceRuntimeDesiredState | null;
|
||||
serviceStates: WorkspaceRuntimeServiceStateMap | null;
|
||||
}
|
||||
|
||||
export interface WorkspaceRuntimeFailureDetails {
|
||||
port?: number;
|
||||
attemptedPortCount?: number;
|
||||
conflictingExecutionWorkspaceId?: string;
|
||||
conflictingProjectWorkspaceId?: string;
|
||||
}
|
||||
|
||||
export interface WorkspaceRuntimeFailureEvidence {
|
||||
operationId: string;
|
||||
operationLogPath: string;
|
||||
code: string;
|
||||
message: string;
|
||||
remediation: string;
|
||||
details: WorkspaceRuntimeFailureDetails | null;
|
||||
failedAt: Date;
|
||||
}
|
||||
|
||||
export interface WorkspaceCommandSource {
|
||||
type: "paperclip";
|
||||
@@ -273,6 +308,7 @@ export interface ExecutionWorkspace {
|
||||
cleanupEligibleAt: Date | null;
|
||||
cleanupReason: string | null;
|
||||
config: ExecutionWorkspaceConfig | null;
|
||||
effectiveRuntimeConfig: EffectiveWorkspaceRuntimeConfig | null;
|
||||
metadata: Record<string, unknown> | null;
|
||||
runtimeServices?: WorkspaceRuntimeService[];
|
||||
createdAt: Date;
|
||||
@@ -289,7 +325,10 @@ export interface WorkspaceRuntimeService {
|
||||
scopeType: "project_workspace" | "execution_workspace" | "run" | "agent";
|
||||
scopeId: string | null;
|
||||
serviceName: string;
|
||||
status: "provisioning" | "starting" | "running" | "stopped" | "failed";
|
||||
/** @deprecated Prefer actualState when comparing runtime intent with observed state. */
|
||||
status: WorkspaceRuntimeActualState;
|
||||
actualState: WorkspaceRuntimeActualState;
|
||||
desiredState: WorkspaceRuntimeDesiredState | null;
|
||||
lifecycle: "shared" | "ephemeral";
|
||||
reuseKey: string | null;
|
||||
command: string | null;
|
||||
@@ -306,6 +345,8 @@ export interface WorkspaceRuntimeService {
|
||||
stopPolicy: Record<string, unknown> | null;
|
||||
healthStatus: "unknown" | "healthy" | "unhealthy";
|
||||
configIndex?: number | null;
|
||||
workspaceCommandId?: string | null;
|
||||
latestFailure: WorkspaceRuntimeFailureEvidence | null;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
@@ -110,6 +110,8 @@ export const workspaceRuntimeServiceSchema = z.object({
|
||||
scopeId: z.string().nullable(),
|
||||
serviceName: z.string(),
|
||||
status: z.enum(["provisioning", "starting", "running", "stopped", "failed"]),
|
||||
actualState: z.enum(["provisioning", "starting", "running", "stopped", "failed"]),
|
||||
desiredState: z.enum(["running", "stopped", "manual"]).nullable(),
|
||||
lifecycle: z.enum(["shared", "ephemeral"]),
|
||||
reuseKey: z.string().nullable(),
|
||||
command: z.string().nullable(),
|
||||
@@ -126,6 +128,21 @@ export const workspaceRuntimeServiceSchema = z.object({
|
||||
stopPolicy: z.record(z.string(), z.unknown()).nullable(),
|
||||
healthStatus: z.enum(["unknown", "healthy", "unhealthy"]),
|
||||
configIndex: z.number().int().nonnegative().nullable().optional(),
|
||||
workspaceCommandId: z.string().nullable().optional(),
|
||||
latestFailure: z.object({
|
||||
operationId: z.string().uuid(),
|
||||
operationLogPath: z.string(),
|
||||
code: z.string(),
|
||||
message: z.string(),
|
||||
remediation: z.string(),
|
||||
details: z.object({
|
||||
port: z.number().int().optional(),
|
||||
attemptedPortCount: z.number().int().nonnegative().optional(),
|
||||
conflictingExecutionWorkspaceId: z.string().uuid().optional(),
|
||||
conflictingProjectWorkspaceId: z.string().uuid().optional(),
|
||||
}).strict().nullable(),
|
||||
failedAt: z.coerce.date(),
|
||||
}).strict().nullable(),
|
||||
createdAt: z.coerce.date(),
|
||||
updatedAt: z.coerce.date(),
|
||||
}).strict();
|
||||
|
||||
@@ -2,7 +2,9 @@ import express from "express";
|
||||
import request from "supertest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { errorHandler } from "../middleware/index.js";
|
||||
import { conflict } from "../errors.js";
|
||||
import { executionWorkspaceRoutes } from "../routes/execution-workspaces.js";
|
||||
import { attachWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js";
|
||||
|
||||
const mockExecutionWorkspaceService = vi.hoisted(() => ({
|
||||
list: vi.fn(),
|
||||
@@ -61,6 +63,10 @@ vi.mock("../services/workspace-runtime.js", async (importActual) => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../routes/workspace-runtime-authz.js", () => ({
|
||||
assertCanManageExecutionWorkspaceRuntimeServices: vi.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
function createApp(actor: Record<string, unknown> = {
|
||||
type: "board",
|
||||
userId: "local-board",
|
||||
@@ -107,6 +113,9 @@ describe.sequential("execution workspace routes", () => {
|
||||
]);
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue(null);
|
||||
mockExecutionWorkspaceService.reconcileExecutionWorkspaceBranch.mockResolvedValue(null);
|
||||
mockWorkspaceOperationService.createRecorder.mockReturnValue({
|
||||
recordOperation: vi.fn(),
|
||||
});
|
||||
mockHeartbeatService.wakeup.mockResolvedValue(null);
|
||||
});
|
||||
|
||||
@@ -161,6 +170,141 @@ describe.sequential("execution workspace routes", () => {
|
||||
expect(mockExecutionWorkspaceService.listOverview).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns effective inherited runtime config and the current failed service row", async () => {
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue({
|
||||
id: "workspace-1",
|
||||
companyId: "company-1",
|
||||
projectWorkspaceId: "project-workspace-1",
|
||||
effectiveRuntimeConfig: {
|
||||
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
|
||||
source: { type: "project_workspace", id: "project-workspace-1" },
|
||||
desiredState: "running",
|
||||
serviceStates: null,
|
||||
},
|
||||
runtimeServices: [{
|
||||
id: "service-1",
|
||||
serviceName: "web",
|
||||
status: "failed",
|
||||
actualState: "failed",
|
||||
desiredState: "running",
|
||||
latestFailure: {
|
||||
operationId: "operation-1",
|
||||
operationLogPath: "/api/workspace-operations/operation-1/log",
|
||||
code: "workspace_runtime_start_failed",
|
||||
message: "Workspace runtime service failed to start.",
|
||||
remediation: "Review the workspace operation log and retry.",
|
||||
details: null,
|
||||
failedAt: new Date("2026-08-11T12:00:00.000Z"),
|
||||
},
|
||||
}],
|
||||
});
|
||||
|
||||
const res = await request(createApp()).get("/api/execution-workspaces/workspace-1");
|
||||
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body.effectiveRuntimeConfig.source).toEqual({
|
||||
type: "project_workspace",
|
||||
id: "project-workspace-1",
|
||||
});
|
||||
expect(res.body.runtimeServices).toEqual([
|
||||
expect.objectContaining({
|
||||
id: "service-1",
|
||||
actualState: "failed",
|
||||
desiredState: "running",
|
||||
latestFailure: expect.objectContaining({ operationId: "operation-1" }),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not expose workspace detail across company scope", async () => {
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue({
|
||||
id: "workspace-1",
|
||||
companyId: "company-1",
|
||||
});
|
||||
|
||||
const res = await request(createApp({
|
||||
type: "board",
|
||||
userId: "other-board",
|
||||
companyIds: ["company-2"],
|
||||
source: "session",
|
||||
isInstanceAdmin: false,
|
||||
})).get("/api/execution-workspaces/workspace-1");
|
||||
|
||||
expect(res.status).toBe(404);
|
||||
});
|
||||
|
||||
it("returns a safe operation reference when runtime start fails", async () => {
|
||||
mockExecutionWorkspaceService.getById.mockResolvedValue({
|
||||
id: "workspace-1",
|
||||
companyId: "company-1",
|
||||
projectId: null,
|
||||
projectWorkspaceId: null,
|
||||
sourceIssueId: null,
|
||||
name: "Runtime failure workspace",
|
||||
mode: "isolated_workspace",
|
||||
strategyType: "git_worktree",
|
||||
cwd: "/tmp/runtime-failure-workspace",
|
||||
repoUrl: null,
|
||||
baseRef: "master",
|
||||
branchName: "runtime-failure",
|
||||
providerType: "git_worktree",
|
||||
providerRef: "/tmp/runtime-failure-workspace",
|
||||
config: {
|
||||
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
|
||||
desiredState: "running",
|
||||
serviceStates: null,
|
||||
},
|
||||
effectiveRuntimeConfig: {
|
||||
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
|
||||
source: { type: "execution_workspace", id: "workspace-1" },
|
||||
desiredState: "running",
|
||||
serviceStates: null,
|
||||
},
|
||||
runtimeServices: [],
|
||||
metadata: null,
|
||||
});
|
||||
const operationError = attachWorkspaceOperationFailureEvidence(
|
||||
conflict("No safe automatically allocated runtime service port is available.", {
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
attemptedPortCount: 10,
|
||||
cwd: "/secret/workspace/path",
|
||||
remediation: "Configure a different runtime service port.",
|
||||
}),
|
||||
{
|
||||
operationId: "operation-1",
|
||||
operationLogPath: "/api/workspace-operations/operation-1/log",
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
message: "No safe automatically allocated runtime service port is available.",
|
||||
remediation: "Configure a different runtime service port.",
|
||||
details: { attemptedPortCount: 10 },
|
||||
failedAt: "2026-08-11T12:00:00.000Z",
|
||||
},
|
||||
);
|
||||
mockWorkspaceOperationService.createRecorder.mockReturnValue({
|
||||
recordOperation: vi.fn(async () => {
|
||||
throw operationError;
|
||||
}),
|
||||
});
|
||||
|
||||
const res = await request(createApp())
|
||||
.post("/api/execution-workspaces/workspace-1/runtime-services/start")
|
||||
.send({ workspaceCommandId: "service:web" });
|
||||
|
||||
expect(res.status).toBe(409);
|
||||
expect(res.body).toMatchObject({
|
||||
error: "No safe automatically allocated runtime service port is available.",
|
||||
details: {
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
attemptedPortCount: 10,
|
||||
remediation: "Configure a different runtime service port.",
|
||||
operationId: "operation-1",
|
||||
operationLogPath: "/api/workspace-operations/operation-1/log",
|
||||
failedAt: "2026-08-11T12:00:00.000Z",
|
||||
},
|
||||
});
|
||||
expect(JSON.stringify(res.body)).not.toContain("/secret/workspace/path");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["forward", { mode: "forward" }],
|
||||
["override", { mode: "override", reason: "operator break-glass" }],
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
issues,
|
||||
projectWorkspaces,
|
||||
projects,
|
||||
workspaceOperations,
|
||||
workspaceRuntimeServices,
|
||||
} from "@paperclipai/db";
|
||||
import {
|
||||
@@ -39,6 +40,11 @@ import {
|
||||
readMetadataReopenPendingConsumptionSince,
|
||||
} from "../services/execution-workspaces.ts";
|
||||
import { issueService } from "../services/issues.ts";
|
||||
import { conflict } from "../errors.ts";
|
||||
import {
|
||||
getWorkspaceOperationFailureEvidence,
|
||||
workspaceOperationService,
|
||||
} from "../services/workspace-operations.ts";
|
||||
import {
|
||||
startRuntimeServicesForWorkspaceControl,
|
||||
stopRuntimeServicesForExecutionWorkspace,
|
||||
@@ -287,6 +293,73 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
it("persists safe structured evidence and attaches its exact operation reference", async () => {
|
||||
const companyId = randomUUID();
|
||||
const logDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-workspace-operation-log-"));
|
||||
tempDirs.add(logDir);
|
||||
const previousLogBasePath = process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
|
||||
process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = logDir;
|
||||
|
||||
await db.insert(companies).values({
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: "PAP",
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
});
|
||||
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await workspaceOperationService(db).createRecorder({ companyId }).recordOperation({
|
||||
phase: "workspace_provision",
|
||||
command: "pnpm dev",
|
||||
metadata: { action: "start", serviceIndex: 0, workspaceCommandId: "service:web" },
|
||||
run: async () => {
|
||||
throw conflict("No safe runtime port is available.", {
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
port: 3100,
|
||||
cwd: "/secret/workspace/path",
|
||||
remediation: "Configure a different runtime service port.",
|
||||
});
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
thrown = error;
|
||||
} finally {
|
||||
if (previousLogBasePath === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH;
|
||||
else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = previousLogBasePath;
|
||||
}
|
||||
|
||||
const evidence = getWorkspaceOperationFailureEvidence(thrown);
|
||||
expect(evidence).toMatchObject({
|
||||
operationLogPath: `/api/workspace-operations/${evidence?.operationId}/log`,
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
message: "No safe runtime port is available.",
|
||||
remediation: "Configure a different runtime service port.",
|
||||
details: { port: 3100 },
|
||||
});
|
||||
expect(evidence?.details).not.toHaveProperty("cwd");
|
||||
|
||||
const [operation] = await db
|
||||
.select()
|
||||
.from(workspaceOperations)
|
||||
.where(eq(workspaceOperations.id, evidence!.operationId));
|
||||
expect(operation).toMatchObject({
|
||||
status: "failed",
|
||||
metadata: expect.objectContaining({
|
||||
action: "start",
|
||||
serviceIndex: 0,
|
||||
workspaceCommandId: "service:web",
|
||||
failureEvidence: expect.objectContaining({
|
||||
operationId: evidence!.operationId,
|
||||
operationLogPath: evidence!.operationLogPath,
|
||||
code: "workspace_runtime_port_allocation_exhausted",
|
||||
details: { port: 3100 },
|
||||
}),
|
||||
}),
|
||||
});
|
||||
expect(JSON.stringify(operation?.metadata)).not.toContain("/secret/workspace/path");
|
||||
});
|
||||
|
||||
async function seedTerminalWorkspace(options: {
|
||||
mergedPr?: boolean;
|
||||
activeRun?: boolean;
|
||||
@@ -3922,16 +3995,31 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
]);
|
||||
|
||||
const [workspace] = await svc.list(companyId);
|
||||
expect(workspace?.effectiveRuntimeConfig).toEqual({
|
||||
workspaceRuntime: {
|
||||
services: [
|
||||
{ name: "web", command: "pnpm dev" },
|
||||
{ name: "worker", command: "pnpm worker", reuseScope: "execution_workspace" },
|
||||
],
|
||||
},
|
||||
source: { type: "project_workspace", id: projectWorkspaceId },
|
||||
desiredState: "stopped",
|
||||
serviceStates: null,
|
||||
});
|
||||
expect(workspace?.runtimeServices).toEqual([
|
||||
expect.objectContaining({
|
||||
id: currentWebServiceId,
|
||||
serviceName: "web",
|
||||
configIndex: 0,
|
||||
actualState: "stopped",
|
||||
desiredState: "stopped",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: currentWorkerServiceId,
|
||||
serviceName: "worker",
|
||||
configIndex: 1,
|
||||
actualState: "running",
|
||||
desiredState: "stopped",
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -3948,6 +4036,182 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("projects one failed configured row with inherited config and company-scoped failure evidence", async () => {
|
||||
const companyId = randomUUID();
|
||||
const otherCompanyId = randomUUID();
|
||||
const projectId = randomUUID();
|
||||
const projectWorkspaceId = randomUUID();
|
||||
const executionWorkspaceId = randomUUID();
|
||||
const failedServiceId = randomUUID();
|
||||
const failedOperationId = randomUUID();
|
||||
const failedAt = new Date("2026-08-11T14:00:00.000Z");
|
||||
|
||||
await db.insert(companies).values([
|
||||
{
|
||||
id: companyId,
|
||||
name: "Paperclip",
|
||||
issuePrefix: "PAP",
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
},
|
||||
{
|
||||
id: otherCompanyId,
|
||||
name: "OtherCo",
|
||||
issuePrefix: "OTH",
|
||||
requireBoardApprovalForNewAgents: false,
|
||||
},
|
||||
]);
|
||||
await db.insert(projects).values({
|
||||
id: projectId,
|
||||
companyId,
|
||||
name: "Inherited runtime projection",
|
||||
status: "in_progress",
|
||||
});
|
||||
await db.insert(projectWorkspaces).values({
|
||||
id: projectWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
name: "Primary",
|
||||
sourceType: "local_path",
|
||||
isPrimary: true,
|
||||
cwd: "/tmp/inherited-runtime-projection",
|
||||
metadata: {
|
||||
runtimeConfig: {
|
||||
workspaceRuntime: {
|
||||
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
|
||||
},
|
||||
desiredState: "stopped",
|
||||
},
|
||||
},
|
||||
});
|
||||
await db.insert(executionWorkspaces).values({
|
||||
id: executionWorkspaceId,
|
||||
companyId,
|
||||
projectId,
|
||||
projectWorkspaceId,
|
||||
mode: "isolated_workspace",
|
||||
strategyType: "git_worktree",
|
||||
name: "Inherited isolated workspace",
|
||||
status: "idle",
|
||||
providerType: "git_worktree",
|
||||
cwd: "/tmp/inherited-runtime-projection/isolated",
|
||||
metadata: {
|
||||
config: {
|
||||
desiredState: "running",
|
||||
serviceStates: { "0": "manual" },
|
||||
},
|
||||
},
|
||||
});
|
||||
await db.insert(workspaceRuntimeServices).values([
|
||||
{
|
||||
id: randomUUID(),
|
||||
companyId,
|
||||
projectId,
|
||||
projectWorkspaceId,
|
||||
executionWorkspaceId,
|
||||
scopeType: "execution_workspace",
|
||||
scopeId: executionWorkspaceId,
|
||||
serviceName: "web",
|
||||
status: "stopped",
|
||||
lifecycle: "shared",
|
||||
reuseKey: "inherited-web",
|
||||
command: "pnpm dev",
|
||||
cwd: "/tmp/inherited-runtime-projection/isolated",
|
||||
provider: "local_process",
|
||||
healthStatus: "unknown",
|
||||
updatedAt: new Date("2026-08-10T14:00:00.000Z"),
|
||||
},
|
||||
{
|
||||
id: failedServiceId,
|
||||
companyId,
|
||||
projectId,
|
||||
projectWorkspaceId,
|
||||
executionWorkspaceId,
|
||||
scopeType: "execution_workspace",
|
||||
scopeId: executionWorkspaceId,
|
||||
serviceName: "web",
|
||||
status: "failed",
|
||||
lifecycle: "shared",
|
||||
reuseKey: "inherited-web",
|
||||
command: "pnpm dev",
|
||||
cwd: "/tmp/inherited-runtime-projection/isolated",
|
||||
provider: "local_process",
|
||||
healthStatus: "unhealthy",
|
||||
updatedAt: failedAt,
|
||||
},
|
||||
]);
|
||||
await db.insert(workspaceOperations).values([
|
||||
{
|
||||
id: failedOperationId,
|
||||
companyId,
|
||||
executionWorkspaceId,
|
||||
phase: "workspace_provision",
|
||||
command: "pnpm dev",
|
||||
status: "failed",
|
||||
metadata: {
|
||||
action: "start",
|
||||
serviceIndex: 0,
|
||||
workspaceCommandId: "service:web",
|
||||
failureEvidence: {
|
||||
code: "workspace_runtime_start_failed",
|
||||
message: "Workspace runtime service failed to start.",
|
||||
remediation: "Review the workspace operation log and retry.",
|
||||
details: { port: 3100 },
|
||||
},
|
||||
},
|
||||
startedAt: failedAt,
|
||||
finishedAt: failedAt,
|
||||
},
|
||||
{
|
||||
companyId: otherCompanyId,
|
||||
executionWorkspaceId,
|
||||
phase: "workspace_provision",
|
||||
command: "pnpm dev",
|
||||
status: "failed",
|
||||
metadata: {
|
||||
action: "start",
|
||||
serviceIndex: 0,
|
||||
workspaceCommandId: "service:web",
|
||||
failureEvidence: {
|
||||
code: "other_company_failure",
|
||||
message: "Must not leak.",
|
||||
remediation: "Must not leak.",
|
||||
},
|
||||
},
|
||||
startedAt: new Date("2026-08-11T15:00:00.000Z"),
|
||||
finishedAt: new Date("2026-08-11T15:00:00.000Z"),
|
||||
},
|
||||
]);
|
||||
|
||||
const workspace = await svc.getById(executionWorkspaceId);
|
||||
|
||||
expect(workspace?.effectiveRuntimeConfig).toEqual({
|
||||
workspaceRuntime: {
|
||||
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
|
||||
},
|
||||
source: { type: "project_workspace", id: projectWorkspaceId },
|
||||
desiredState: "running",
|
||||
serviceStates: { "0": "manual" },
|
||||
});
|
||||
expect(workspace?.runtimeServices).toEqual([
|
||||
expect.objectContaining({
|
||||
id: failedServiceId,
|
||||
status: "failed",
|
||||
actualState: "failed",
|
||||
desiredState: "manual",
|
||||
workspaceCommandId: "service:web",
|
||||
latestFailure: {
|
||||
operationId: failedOperationId,
|
||||
operationLogPath: `/api/workspace-operations/${failedOperationId}/log`,
|
||||
code: "workspace_runtime_start_failed",
|
||||
message: "Workspace runtime service failed to start.",
|
||||
remediation: "Review the workspace operation log and retry.",
|
||||
details: { port: 3100 },
|
||||
failedAt,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns a bounded company-scoped workspace overview with service and linked issue summaries", async () => {
|
||||
const companyId = randomUUID();
|
||||
const otherCompanyId = randomUUID();
|
||||
@@ -4012,7 +4276,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
metadata: {
|
||||
config: {
|
||||
workspaceRuntime: {
|
||||
services: [{ name: "web", command: "pnpm dev" }],
|
||||
services: [{ name: "web", command: "pnpm dev", reuseScope: "execution_workspace" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -4148,7 +4412,7 @@ describeEmbeddedPostgres("executionWorkspaceService.getCloseReadiness", () => {
|
||||
projectUrlKey: "workspaces",
|
||||
projectName: "Workspaces",
|
||||
branchName: "paperclip/a",
|
||||
serviceCount: 2,
|
||||
serviceCount: 1,
|
||||
runningServiceCount: 1,
|
||||
primaryServiceUrl: "http://localhost:3100",
|
||||
primaryServiceUrlRunning: true,
|
||||
|
||||
@@ -298,6 +298,8 @@ describe.sequential("issue goal context routes", () => {
|
||||
scopeType: "execution_workspace",
|
||||
scopeId: workspaceId,
|
||||
serviceName: "Dev server",
|
||||
actualState: "stopped",
|
||||
desiredState: "running",
|
||||
lifecycle: "shared",
|
||||
reuseKey: "dev-server",
|
||||
command: "pnpm dev",
|
||||
@@ -313,6 +315,8 @@ describe.sequential("issue goal context routes", () => {
|
||||
stoppedAt: null,
|
||||
healthStatus: "healthy",
|
||||
configIndex: 0,
|
||||
workspaceCommandId: "service:dev-server",
|
||||
latestFailure: null,
|
||||
metadata: { huge: "runtime metadata should not be embedded" },
|
||||
};
|
||||
mockIssueService.getById.mockResolvedValueOnce({
|
||||
@@ -402,10 +406,21 @@ describe.sequential("issue goal context routes", () => {
|
||||
desiredState: null,
|
||||
serviceStates: null,
|
||||
},
|
||||
effectiveRuntimeConfig: {
|
||||
workspaceRuntime: { services: [{ name: "Dev server", command: "pnpm dev" }] },
|
||||
source: { type: "execution_workspace", id: workspaceId },
|
||||
desiredState: "running",
|
||||
serviceStates: null,
|
||||
},
|
||||
metadata: { huge: "execution workspace metadata should not be embedded" },
|
||||
runtimeServices: [
|
||||
{ ...runtimeServiceBase, id: "service-running", status: "running" },
|
||||
{ ...runtimeServiceBase, id: "service-stopped", status: "stopped" },
|
||||
{
|
||||
...runtimeServiceBase,
|
||||
id: "service-failed",
|
||||
status: "failed",
|
||||
actualState: "failed",
|
||||
healthStatus: "unhealthy",
|
||||
},
|
||||
],
|
||||
createdAt: new Date("2026-03-24T12:00:00Z"),
|
||||
updatedAt: new Date("2026-03-24T12:00:00Z"),
|
||||
@@ -422,10 +437,16 @@ describe.sequential("issue goal context routes", () => {
|
||||
expect(res.body.currentExecutionWorkspace.metadata).toBeNull();
|
||||
expect(res.body.currentExecutionWorkspace.runtimeServices).toHaveLength(1);
|
||||
expect(res.body.currentExecutionWorkspace.runtimeServices[0]).toMatchObject({
|
||||
id: "service-running",
|
||||
status: "running",
|
||||
id: "service-failed",
|
||||
status: "failed",
|
||||
actualState: "failed",
|
||||
desiredState: "running",
|
||||
url: "http://localhost:3100",
|
||||
});
|
||||
expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({
|
||||
type: "execution_workspace",
|
||||
id: workspaceId,
|
||||
});
|
||||
expect(res.body.currentExecutionWorkspace.runtimeServices[0]).not.toHaveProperty("metadata");
|
||||
});
|
||||
|
||||
@@ -512,13 +533,31 @@ describe.sequential("issue goal context routes", () => {
|
||||
mode: "isolated_workspace",
|
||||
status: "active",
|
||||
cwd: "/tmp/pap-581",
|
||||
effectiveRuntimeConfig: {
|
||||
workspaceRuntime: { services: [{ name: "web", command: "pnpm dev" }] },
|
||||
source: { type: "project_workspace", id: "project-workspace-1" },
|
||||
desiredState: "running",
|
||||
serviceStates: null,
|
||||
},
|
||||
runtimeServices: [
|
||||
{
|
||||
id: "service-1",
|
||||
serviceName: "web",
|
||||
status: "running",
|
||||
url: "http://127.0.0.1:5173",
|
||||
healthStatus: "healthy",
|
||||
status: "failed",
|
||||
actualState: "failed",
|
||||
desiredState: "running",
|
||||
url: null,
|
||||
healthStatus: "unhealthy",
|
||||
workspaceCommandId: "service:web",
|
||||
latestFailure: {
|
||||
operationId: "operation-1",
|
||||
operationLogPath: "/api/workspace-operations/operation-1/log",
|
||||
code: "workspace_runtime_start_failed",
|
||||
message: "Workspace runtime service failed to start.",
|
||||
remediation: "Review the workspace operation log and retry.",
|
||||
details: null,
|
||||
failedAt: new Date("2026-08-11T12:00:00.000Z"),
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -535,9 +574,15 @@ describe.sequential("issue goal context routes", () => {
|
||||
runtimeServices: [
|
||||
expect.objectContaining({
|
||||
serviceName: "web",
|
||||
url: "http://127.0.0.1:5173",
|
||||
actualState: "failed",
|
||||
desiredState: "running",
|
||||
latestFailure: expect.objectContaining({ operationId: "operation-1" }),
|
||||
}),
|
||||
],
|
||||
}));
|
||||
expect(res.body.currentExecutionWorkspace.effectiveRuntimeConfig.source).toEqual({
|
||||
type: "project_workspace",
|
||||
id: "project-workspace-1",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -5981,6 +5981,70 @@ describeEmbeddedPostgres("workspace runtime service control persistence", () =>
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("keeps one configured row failed after repeated readiness failures", async () => {
|
||||
const fixture = await createRuntimeFixture();
|
||||
const cleanupRuntimeHome = await createRuntimeHome();
|
||||
const workspace = fixture.workspaces[0]!;
|
||||
const port = await findFreePort();
|
||||
const command = `${JSON.stringify(process.execPath)} -e ${JSON.stringify("process.exit(1)")}`;
|
||||
const config = fixedPortRuntimeConfig(port, command);
|
||||
|
||||
try {
|
||||
await expect(startRuntimeServicesForWorkspaceControl({
|
||||
db,
|
||||
actor: fixture.actor,
|
||||
issue: null,
|
||||
workspace: fixture.realizedWorkspace(workspace),
|
||||
executionWorkspaceId: workspace.id,
|
||||
config,
|
||||
adapterEnv: {},
|
||||
})).rejects.toThrow(/failed to start runtime service|exited before readiness/i);
|
||||
|
||||
const firstRows = await db
|
||||
.select()
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id));
|
||||
expect(firstRows).toEqual([
|
||||
expect.objectContaining({
|
||||
serviceName: "web",
|
||||
command,
|
||||
status: "failed",
|
||||
healthStatus: "unhealthy",
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(startRuntimeServicesForWorkspaceControl({
|
||||
db,
|
||||
actor: fixture.actor,
|
||||
issue: null,
|
||||
workspace: fixture.realizedWorkspace(workspace),
|
||||
executionWorkspaceId: workspace.id,
|
||||
config,
|
||||
adapterEnv: {},
|
||||
})).rejects.toThrow(/failed to start runtime service|exited before readiness/i);
|
||||
|
||||
const retryRows = await db
|
||||
.select()
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(eq(workspaceRuntimeServices.executionWorkspaceId, workspace.id));
|
||||
expect(retryRows).toEqual([
|
||||
expect.objectContaining({
|
||||
id: firstRows[0]!.id,
|
||||
status: "failed",
|
||||
healthStatus: "unhealthy",
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await stopRuntimeServicesForExecutionWorkspace({
|
||||
db,
|
||||
executionWorkspaceId: workspace.id,
|
||||
workspaceCwd: workspace.cwd,
|
||||
}).catch(() => undefined);
|
||||
await cleanupRuntimeHome();
|
||||
await fixture.cleanup();
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("returns a bounded structured conflict and exposes only same-company workspace references", async () => {
|
||||
const fixture = await createRuntimeFixture({
|
||||
workspaceModes: ["isolated_workspace", "isolated_workspace"],
|
||||
|
||||
@@ -38,9 +38,29 @@ import { assertCanManageExecutionWorkspaceRuntimeServices } from "./workspace-ru
|
||||
import { appendWithCap } from "../adapters/utils.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { HttpError } from "../errors.js";
|
||||
import { getWorkspaceOperationFailureEvidence } from "../services/workspace-operations.js";
|
||||
|
||||
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
|
||||
|
||||
function toWorkspaceRuntimeOperationHttpError(error: unknown) {
|
||||
const evidence = getWorkspaceOperationFailureEvidence(error);
|
||||
if (!evidence) return error;
|
||||
|
||||
return new HttpError(
|
||||
error instanceof HttpError ? error.status : 500,
|
||||
error instanceof HttpError ? error.message : evidence.message,
|
||||
{
|
||||
code: evidence.code,
|
||||
remediation: evidence.remediation,
|
||||
operationId: evidence.operationId,
|
||||
operationLogPath: evidence.operationLogPath,
|
||||
failedAt: evidence.failedAt,
|
||||
...(evidence.details ?? {}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: PluginWorkerManager } = {}) {
|
||||
const router = Router();
|
||||
const svc = executionWorkspaceService(db);
|
||||
@@ -456,6 +476,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
},
|
||||
};
|
||||
},
|
||||
}).catch((error) => {
|
||||
throw toWorkspaceRuntimeOperationHttpError(error);
|
||||
});
|
||||
|
||||
const workspace = await svc.getById(id);
|
||||
|
||||
@@ -5303,6 +5303,8 @@ export function issueRoutes(
|
||||
scopeId: service.scopeId,
|
||||
serviceName: service.serviceName,
|
||||
status: service.status,
|
||||
actualState: service.actualState,
|
||||
desiredState: service.desiredState,
|
||||
lifecycle: service.lifecycle,
|
||||
reuseKey: service.reuseKey,
|
||||
command: service.command,
|
||||
@@ -5318,6 +5320,8 @@ export function issueRoutes(
|
||||
stoppedAt: service.stoppedAt,
|
||||
healthStatus: service.healthStatus,
|
||||
configIndex: service.configIndex ?? null,
|
||||
workspaceCommandId: service.workspaceCommandId ?? null,
|
||||
latestFailure: service.latestFailure,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -5358,12 +5362,9 @@ export function issueRoutes(
|
||||
serviceStates: workspace.config.serviceStates,
|
||||
}
|
||||
: null,
|
||||
effectiveRuntimeConfig: workspace.effectiveRuntimeConfig,
|
||||
metadata: null,
|
||||
runtimeServices: (workspace.runtimeServices ?? [])
|
||||
.filter((service) =>
|
||||
service.status === "provisioning" || service.status === "starting" || service.status === "running"
|
||||
)
|
||||
.map(compactIssueRuntimeService),
|
||||
runtimeServices: (workspace.runtimeServices ?? []).map(compactIssueRuntimeService),
|
||||
createdAt: workspace.createdAt,
|
||||
updatedAt: workspace.updatedAt,
|
||||
};
|
||||
|
||||
@@ -13,9 +13,11 @@ import {
|
||||
issues,
|
||||
projects,
|
||||
projectWorkspaces,
|
||||
workspaceOperations,
|
||||
workspaceRuntimeServices,
|
||||
} from "@paperclipai/db";
|
||||
import type {
|
||||
EffectiveWorkspaceRuntimeConfig,
|
||||
ExecutionWorkspace,
|
||||
ExecutionWorkspaceDeliveryState,
|
||||
ExecutionWorkspaceSummary,
|
||||
@@ -27,6 +29,7 @@ import type {
|
||||
WorkspaceOverviewItem,
|
||||
WorkspaceOverviewLinkedIssue,
|
||||
WorkspaceRuntimeDesiredState,
|
||||
WorkspaceRuntimeFailureEvidence,
|
||||
WorkspaceRuntimeService,
|
||||
WorkspaceOverviewPrimaryService,
|
||||
WorkspaceOverviewQuery,
|
||||
@@ -61,10 +64,22 @@ import {
|
||||
|
||||
type ExecutionWorkspaceRow = typeof executionWorkspaces.$inferSelect;
|
||||
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
|
||||
type RuntimeServiceReadDb = Pick<Db, "select">;
|
||||
type RuntimeServiceReadDb = Pick<Db, "select" | "selectDistinctOn">;
|
||||
type DbTransaction = Parameters<Parameters<Db["transaction"]>[0]>[0];
|
||||
type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & {
|
||||
configIndex?: number | null;
|
||||
workspaceCommandId?: string | null;
|
||||
desiredState?: WorkspaceRuntimeDesiredState | null;
|
||||
latestFailure?: WorkspaceRuntimeFailureEvidence | null;
|
||||
};
|
||||
type ProjectedWorkspaceRuntimeFailure = {
|
||||
serviceIndex: number | null;
|
||||
workspaceCommandId: string | null;
|
||||
evidence: WorkspaceRuntimeFailureEvidence;
|
||||
};
|
||||
const execFileAsync = promisify(execFile);
|
||||
const TERMINAL_ISSUE_STATUSES = new Set(["done", "cancelled"]);
|
||||
const ACTIVE_RUNTIME_SERVICE_STATUSES = new Set(["provisioning", "starting", "running"]);
|
||||
const WORKSPACE_BRANCH_INCOHERENCE_REASON = "git_worktree_branch_incoherence";
|
||||
const WORKSPACE_VALIDATION_RECOVERY_CAUSE = "workspace_validation_failed";
|
||||
export const ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON = "issue_terminal";
|
||||
@@ -627,7 +642,9 @@ function assertBranchReconcileRuntimeServicesStopped(input: {
|
||||
inspection: ExecutionWorkspaceBranchReconcileInspection;
|
||||
runtimeServices: WorkspaceRuntimeService[];
|
||||
}) {
|
||||
const activeRuntimeServices = input.runtimeServices.filter((service) => service.status !== "stopped");
|
||||
const activeRuntimeServices = input.runtimeServices.filter((service) =>
|
||||
ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState)
|
||||
);
|
||||
if (activeRuntimeServices.length > 0) {
|
||||
throw unprocessable("Execution workspace branch reconciliation requires all runtime services to be stopped", {
|
||||
inspection: input.inspection,
|
||||
@@ -952,8 +969,9 @@ export function mergeExecutionWorkspaceConfig(
|
||||
}
|
||||
|
||||
function toRuntimeService(
|
||||
row: WorkspaceRuntimeServiceRow & { configIndex?: number | null },
|
||||
row: ProjectedWorkspaceRuntimeServiceRow,
|
||||
): WorkspaceRuntimeService {
|
||||
const actualState = row.status as WorkspaceRuntimeService["actualState"];
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -964,7 +982,9 @@ function toRuntimeService(
|
||||
scopeType: row.scopeType as WorkspaceRuntimeService["scopeType"],
|
||||
scopeId: row.scopeId ?? null,
|
||||
serviceName: row.serviceName,
|
||||
status: row.status as WorkspaceRuntimeService["status"],
|
||||
status: actualState,
|
||||
actualState,
|
||||
desiredState: row.desiredState ?? null,
|
||||
lifecycle: row.lifecycle as WorkspaceRuntimeService["lifecycle"],
|
||||
reuseKey: row.reuseKey ?? null,
|
||||
command: row.command ?? null,
|
||||
@@ -981,6 +1001,8 @@ function toRuntimeService(
|
||||
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
|
||||
healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"],
|
||||
configIndex: row.configIndex ?? null,
|
||||
workspaceCommandId: row.workspaceCommandId ?? null,
|
||||
latestFailure: row.latestFailure ?? null,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
@@ -990,7 +1012,9 @@ function toExecutionWorkspace(
|
||||
row: ExecutionWorkspaceRow,
|
||||
runtimeServices: WorkspaceRuntimeService[] = [],
|
||||
deliveryState: ExecutionWorkspaceDeliveryState = "unknown",
|
||||
effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null,
|
||||
): ExecutionWorkspace {
|
||||
const config = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -1014,7 +1038,17 @@ function toExecutionWorkspace(
|
||||
closedAt: row.closedAt ?? null,
|
||||
cleanupEligibleAt: row.cleanupEligibleAt ?? null,
|
||||
cleanupReason: row.cleanupReason ?? null,
|
||||
config: readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null),
|
||||
config,
|
||||
effectiveRuntimeConfig: effectiveRuntimeConfig === undefined
|
||||
? config?.workspaceRuntime
|
||||
? {
|
||||
workspaceRuntime: config.workspaceRuntime,
|
||||
source: { type: "execution_workspace", id: row.id },
|
||||
desiredState: config.desiredState,
|
||||
serviceStates: config.serviceStates ?? null,
|
||||
}
|
||||
: null
|
||||
: effectiveRuntimeConfig,
|
||||
metadata: (row.metadata as Record<string, unknown> | null) ?? null,
|
||||
runtimeServices,
|
||||
createdAt: row.createdAt,
|
||||
@@ -1075,6 +1109,11 @@ function usesInheritedProjectRuntimeServices(row: ExecutionWorkspaceRow) {
|
||||
return !readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime;
|
||||
}
|
||||
|
||||
function inheritsProjectRuntimeConfig(row: ExecutionWorkspaceRow) {
|
||||
if (!row.projectWorkspaceId) return false;
|
||||
return !readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime;
|
||||
}
|
||||
|
||||
function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow) {
|
||||
const inheritedProjectWorkspaceId = usesInheritedProjectRuntimeServices(row) ? row.projectWorkspaceId : null;
|
||||
const activeServiceConditions = inheritedProjectWorkspaceId
|
||||
@@ -1087,22 +1126,124 @@ function noActiveRuntimeServicesForWorkspaceCondition(row: ExecutionWorkspaceRow
|
||||
),
|
||||
eq(workspaceRuntimeServices.executionWorkspaceId, row.id),
|
||||
),
|
||||
ne(workspaceRuntimeServices.status, "stopped"),
|
||||
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]),
|
||||
)
|
||||
: and(
|
||||
eq(workspaceRuntimeServices.companyId, row.companyId),
|
||||
eq(workspaceRuntimeServices.executionWorkspaceId, row.id),
|
||||
ne(workspaceRuntimeServices.status, "stopped"),
|
||||
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_SERVICE_STATUSES]),
|
||||
);
|
||||
return sql`not exists (select 1 from ${workspaceRuntimeServices} where ${activeServiceConditions})`;
|
||||
}
|
||||
|
||||
function readWorkspaceRuntimeFailureEvidence(
|
||||
operation: {
|
||||
id: string;
|
||||
metadata: Record<string, unknown> | null;
|
||||
startedAt: Date;
|
||||
finishedAt: Date | null;
|
||||
},
|
||||
): WorkspaceRuntimeFailureEvidence {
|
||||
const raw = isRecord(operation.metadata?.failureEvidence)
|
||||
? operation.metadata.failureEvidence
|
||||
: null;
|
||||
const rawDetails = isRecord(raw?.details) ? raw.details : null;
|
||||
const details: NonNullable<WorkspaceRuntimeFailureEvidence["details"]> = {};
|
||||
if (typeof rawDetails?.port === "number" && Number.isInteger(rawDetails.port)) details.port = rawDetails.port;
|
||||
if (typeof rawDetails?.attemptedPortCount === "number" && Number.isInteger(rawDetails.attemptedPortCount)) {
|
||||
details.attemptedPortCount = rawDetails.attemptedPortCount;
|
||||
}
|
||||
if (typeof rawDetails?.conflictingExecutionWorkspaceId === "string") {
|
||||
details.conflictingExecutionWorkspaceId = rawDetails.conflictingExecutionWorkspaceId;
|
||||
}
|
||||
if (typeof rawDetails?.conflictingProjectWorkspaceId === "string") {
|
||||
details.conflictingProjectWorkspaceId = rawDetails.conflictingProjectWorkspaceId;
|
||||
}
|
||||
return {
|
||||
operationId: operation.id,
|
||||
operationLogPath: `/api/workspace-operations/${operation.id}/log`,
|
||||
code: typeof raw?.code === "string" ? raw.code : "workspace_runtime_start_failed",
|
||||
message: typeof raw?.message === "string" ? raw.message : "Workspace runtime service failed to start.",
|
||||
remediation: typeof raw?.remediation === "string"
|
||||
? raw.remediation
|
||||
: "Review the workspace operation log, correct the runtime configuration or dependency, and retry.",
|
||||
details: Object.keys(details).length > 0 ? details : null,
|
||||
failedAt: operation.finishedAt ?? operation.startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
async function loadLatestRuntimeFailuresByExecutionWorkspace(
|
||||
db: RuntimeServiceReadDb,
|
||||
companyId: string,
|
||||
runtimeRowsByWorkspaceId: Map<string, ProjectedWorkspaceRuntimeServiceRow[]>,
|
||||
) {
|
||||
const configuredRows = [...runtimeRowsByWorkspaceId.entries()].flatMap(([executionWorkspaceId, rows]) =>
|
||||
rows.flatMap((row) => row.configIndex === undefined || row.configIndex === null
|
||||
? []
|
||||
: [{ executionWorkspaceId, configIndex: row.configIndex }]),
|
||||
);
|
||||
if (configuredRows.length === 0) {
|
||||
return new Map<string, ProjectedWorkspaceRuntimeFailure[]>();
|
||||
}
|
||||
|
||||
const executionWorkspaceIds = [...new Set(configuredRows.map((row) => row.executionWorkspaceId))];
|
||||
const serviceIndexes = [...new Set(configuredRows.map((row) => String(row.configIndex)))];
|
||||
const serviceIndexExpression = sql<string | null>`${workspaceOperations.metadata}->>'serviceIndex'`;
|
||||
const actionExpression = sql<string | null>`${workspaceOperations.metadata}->>'action'`;
|
||||
const operations = await db
|
||||
.selectDistinctOn(
|
||||
[workspaceOperations.executionWorkspaceId, serviceIndexExpression],
|
||||
{
|
||||
id: workspaceOperations.id,
|
||||
executionWorkspaceId: workspaceOperations.executionWorkspaceId,
|
||||
serviceIndex: serviceIndexExpression,
|
||||
workspaceCommandId: sql<string | null>`${workspaceOperations.metadata}->>'workspaceCommandId'`,
|
||||
metadata: workspaceOperations.metadata,
|
||||
startedAt: workspaceOperations.startedAt,
|
||||
finishedAt: workspaceOperations.finishedAt,
|
||||
},
|
||||
)
|
||||
.from(workspaceOperations)
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceOperations.companyId, companyId),
|
||||
inArray(workspaceOperations.executionWorkspaceId, executionWorkspaceIds),
|
||||
eq(workspaceOperations.status, "failed"),
|
||||
inArray(actionExpression, ["start", "restart"]),
|
||||
or(inArray(serviceIndexExpression, serviceIndexes), isNull(serviceIndexExpression)),
|
||||
),
|
||||
)
|
||||
.orderBy(
|
||||
workspaceOperations.executionWorkspaceId,
|
||||
serviceIndexExpression,
|
||||
desc(workspaceOperations.startedAt),
|
||||
desc(workspaceOperations.createdAt),
|
||||
);
|
||||
|
||||
const result = new Map<string, ProjectedWorkspaceRuntimeFailure[]>();
|
||||
for (const operation of operations) {
|
||||
if (!operation.executionWorkspaceId) continue;
|
||||
const serviceIndex = operation.serviceIndex === null
|
||||
? null
|
||||
: Number.parseInt(operation.serviceIndex, 10);
|
||||
if (serviceIndex !== null && !Number.isInteger(serviceIndex)) continue;
|
||||
const existing = result.get(operation.executionWorkspaceId) ?? [];
|
||||
existing.push({
|
||||
serviceIndex,
|
||||
workspaceCommandId: operation.workspaceCommandId,
|
||||
evidence: readWorkspaceRuntimeFailureEvidence(operation),
|
||||
});
|
||||
result.set(operation.executionWorkspaceId, existing);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function loadEffectiveRuntimeServicesByExecutionWorkspace(
|
||||
db: RuntimeServiceReadDb,
|
||||
companyId: string,
|
||||
rows: ExecutionWorkspaceRow[],
|
||||
) {
|
||||
const inheritedRows = rows.filter((row) => usesInheritedProjectRuntimeServices(row));
|
||||
const inheritedRows = rows.filter((row) => inheritsProjectRuntimeConfig(row));
|
||||
const projectWorkspaceIds = inheritedRows
|
||||
.map((row) => row.projectWorkspaceId)
|
||||
.filter((value): value is string => Boolean(value));
|
||||
@@ -1136,8 +1277,7 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace(
|
||||
const projectRuntimeConfigByWorkspaceId = new Map(
|
||||
projectWorkspaceRows.map((row) => [
|
||||
row.id,
|
||||
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null)?.workspaceRuntime
|
||||
?? null,
|
||||
readProjectWorkspaceRuntimeConfig((row.metadata as Record<string, unknown> | null) ?? null),
|
||||
]),
|
||||
);
|
||||
const effectiveProjectRuntimeServices = new Map(
|
||||
@@ -1145,35 +1285,113 @@ async function loadEffectiveRuntimeServicesByExecutionWorkspace(
|
||||
projectWorkspaceId,
|
||||
selectConfiguredRuntimeServiceRows(
|
||||
projectRuntimeServices.get(projectWorkspaceId) ?? [],
|
||||
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId) ?? null,
|
||||
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.workspaceRuntime ?? null,
|
||||
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.desiredState ?? null,
|
||||
projectRuntimeConfigByWorkspaceId.get(projectWorkspaceId)?.serviceStates ?? null,
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
return new Map(
|
||||
rows.map((row) => {
|
||||
if (!usesInheritedProjectRuntimeServices(row)) {
|
||||
return [row.id, executionRuntimeServices.get(row.id) ?? []] as const;
|
||||
}
|
||||
const runtimeRowsByWorkspaceId = new Map<string, ProjectedWorkspaceRuntimeServiceRow[]>();
|
||||
const effectiveConfigByWorkspaceId = new Map<string, EffectiveWorkspaceRuntimeConfig | null>();
|
||||
for (const row of rows) {
|
||||
const localConfig = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
|
||||
const projectConfig = row.projectWorkspaceId
|
||||
? projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId) ?? null
|
||||
: null;
|
||||
const workspaceRuntime = localConfig?.workspaceRuntime ?? projectConfig?.workspaceRuntime ?? null;
|
||||
const desiredState = localConfig?.desiredState ?? projectConfig?.desiredState ?? null;
|
||||
const serviceStates = localConfig?.serviceStates ?? projectConfig?.serviceStates ?? null;
|
||||
effectiveConfigByWorkspaceId.set(
|
||||
row.id,
|
||||
workspaceRuntime
|
||||
? {
|
||||
workspaceRuntime,
|
||||
source: localConfig?.workspaceRuntime
|
||||
? { type: "execution_workspace", id: row.id }
|
||||
: { type: "project_workspace", id: row.projectWorkspaceId! },
|
||||
desiredState,
|
||||
serviceStates,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
|
||||
const workspaceRuntime = projectRuntimeConfigByWorkspaceId.get(row.projectWorkspaceId!) ?? null;
|
||||
const executionScopedRows = selectConfiguredRuntimeServiceRows(
|
||||
(executionRuntimeServices.get(row.id) ?? []).filter(
|
||||
(runtimeService) => runtimeService.scopeType !== "project_workspace",
|
||||
if (!workspaceRuntime) {
|
||||
runtimeRowsByWorkspaceId.set(
|
||||
row.id,
|
||||
usesInheritedProjectRuntimeServices(row)
|
||||
? []
|
||||
: (executionRuntimeServices.get(row.id) ?? []).map((runtimeService) => ({
|
||||
...runtimeService,
|
||||
desiredState,
|
||||
})),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!usesInheritedProjectRuntimeServices(row)) {
|
||||
runtimeRowsByWorkspaceId.set(
|
||||
row.id,
|
||||
selectConfiguredRuntimeServiceRows(
|
||||
executionRuntimeServices.get(row.id) ?? [],
|
||||
workspaceRuntime,
|
||||
desiredState,
|
||||
serviceStates,
|
||||
),
|
||||
workspaceRuntime,
|
||||
);
|
||||
const effectiveRows = [
|
||||
...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []),
|
||||
...executionScopedRows,
|
||||
].sort(
|
||||
(left, right) =>
|
||||
(left.configIndex ?? Number.MAX_SAFE_INTEGER) -
|
||||
(right.configIndex ?? Number.MAX_SAFE_INTEGER),
|
||||
);
|
||||
return [row.id, effectiveRows] as const;
|
||||
}),
|
||||
continue;
|
||||
}
|
||||
|
||||
const executionScopedRows = selectConfiguredRuntimeServiceRows(
|
||||
(executionRuntimeServices.get(row.id) ?? []).filter(
|
||||
(runtimeService) => runtimeService.scopeType !== "project_workspace",
|
||||
),
|
||||
workspaceRuntime,
|
||||
desiredState,
|
||||
serviceStates,
|
||||
);
|
||||
const effectiveRows = [
|
||||
...(effectiveProjectRuntimeServices.get(row.projectWorkspaceId!) ?? []).map((runtimeService) => ({
|
||||
...runtimeService,
|
||||
desiredState:
|
||||
(runtimeService.configIndex === null || runtimeService.configIndex === undefined
|
||||
? null
|
||||
: serviceStates?.[String(runtimeService.configIndex)])
|
||||
?? desiredState,
|
||||
})),
|
||||
...executionScopedRows,
|
||||
].sort(
|
||||
(left, right) =>
|
||||
(left.configIndex ?? Number.MAX_SAFE_INTEGER) -
|
||||
(right.configIndex ?? Number.MAX_SAFE_INTEGER),
|
||||
);
|
||||
runtimeRowsByWorkspaceId.set(row.id, effectiveRows);
|
||||
}
|
||||
|
||||
const failuresByWorkspaceId = await loadLatestRuntimeFailuresByExecutionWorkspace(
|
||||
db,
|
||||
companyId,
|
||||
runtimeRowsByWorkspaceId,
|
||||
);
|
||||
|
||||
return new Map(rows.map((row) => {
|
||||
const failures = failuresByWorkspaceId.get(row.id) ?? [];
|
||||
const runtimeServices = (runtimeRowsByWorkspaceId.get(row.id) ?? []).map((runtimeService) => {
|
||||
if (runtimeService.configIndex === undefined || runtimeService.configIndex === null) return runtimeService;
|
||||
const candidates = failures.filter((failure) =>
|
||||
(failure.serviceIndex === runtimeService.configIndex || failure.serviceIndex === null)
|
||||
&& (!failure.workspaceCommandId || failure.workspaceCommandId === runtimeService.workspaceCommandId),
|
||||
);
|
||||
const latestFailure = candidates
|
||||
.sort((left, right) => right.evidence.failedAt.getTime() - left.evidence.failedAt.getTime())[0]
|
||||
?.evidence ?? null;
|
||||
return { ...runtimeService, latestFailure };
|
||||
});
|
||||
return [row.id, {
|
||||
runtimeServices,
|
||||
effectiveRuntimeConfig: effectiveConfigByWorkspaceId.get(row.id) ?? null,
|
||||
}] as const;
|
||||
}));
|
||||
}
|
||||
|
||||
type WorkspaceOverviewPageRow = ExecutionWorkspaceRow & {
|
||||
@@ -1376,11 +1594,15 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
}
|
||||
|
||||
async function hydrateWorkspace(row: ExecutionWorkspaceRow, runtimeServices: WorkspaceRuntimeService[] = []) {
|
||||
const workspace = toExecutionWorkspace(row, runtimeServices);
|
||||
async function hydrateWorkspace(
|
||||
row: ExecutionWorkspaceRow,
|
||||
runtimeServices: WorkspaceRuntimeService[] = [],
|
||||
effectiveRuntimeConfig?: EffectiveWorkspaceRuntimeConfig | null,
|
||||
) {
|
||||
const workspace = toExecutionWorkspace(row, runtimeServices, "unknown", effectiveRuntimeConfig);
|
||||
const { git } = await inspectGitCloseReadiness(workspace);
|
||||
const assessment = await assessDelivery(row, git);
|
||||
return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState);
|
||||
return toExecutionWorkspace(row, runtimeServices, assessment.deliveryState, effectiveRuntimeConfig);
|
||||
}
|
||||
|
||||
async function workspaceHasActiveRun(workspace: Pick<ExecutionWorkspaceRow, "id" | "companyId" | "sourceIssueId">) {
|
||||
@@ -1903,7 +2125,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
}
|
||||
|
||||
const items: WorkspaceOverviewItem[] = pageRows.map((row) => {
|
||||
const runtimeServices = (runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService);
|
||||
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
|
||||
const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService);
|
||||
const runningServiceCount = runtimeServices.filter((service) => service.status === "running").length;
|
||||
const primaryService = selectPrimaryOverviewService(runtimeServices);
|
||||
const config = readExecutionWorkspaceConfig((row.metadata as Record<string, unknown> | null) ?? null);
|
||||
@@ -1971,12 +2194,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
.where(and(...conditions))
|
||||
.orderBy(desc(executionWorkspaces.lastUsedAt), desc(executionWorkspaces.createdAt));
|
||||
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, companyId, rows);
|
||||
return Promise.all(rows.map((row) =>
|
||||
hydrateWorkspace(
|
||||
return Promise.all(rows.map((row) => {
|
||||
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
|
||||
return hydrateWorkspace(
|
||||
row,
|
||||
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
|
||||
),
|
||||
));
|
||||
(runtimeProjection?.runtimeServices ?? []).map(toRuntimeService),
|
||||
runtimeProjection?.effectiveRuntimeConfig ?? null,
|
||||
);
|
||||
}));
|
||||
},
|
||||
|
||||
listSummaries: async (companyId: string, filters?: {
|
||||
@@ -2130,9 +2355,11 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
.then((rows) => rows[0] ?? null);
|
||||
if (!row) return null;
|
||||
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, row.companyId, [row]);
|
||||
const runtimeProjection = runtimeServicesByWorkspaceId.get(row.id);
|
||||
return hydrateWorkspace(
|
||||
row,
|
||||
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
|
||||
(runtimeProjection?.runtimeServices ?? []).map(toRuntimeService),
|
||||
runtimeProjection?.effectiveRuntimeConfig ?? null,
|
||||
);
|
||||
},
|
||||
|
||||
@@ -2145,7 +2372,8 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
if (!workspace) return null;
|
||||
|
||||
const runtimeServicesByWorkspaceId = await loadEffectiveRuntimeServicesByExecutionWorkspace(db, workspace.companyId, [workspace]);
|
||||
const runtimeServices = (runtimeServicesByWorkspaceId.get(workspace.id) ?? []).map(toRuntimeService);
|
||||
const runtimeProjection = runtimeServicesByWorkspaceId.get(workspace.id);
|
||||
const runtimeServices = (runtimeProjection?.runtimeServices ?? []).map(toRuntimeService);
|
||||
|
||||
const linkedIssues = await db
|
||||
.select({
|
||||
@@ -2240,7 +2468,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
warnings.push("This shared workspace session points at project workspace infrastructure. Archiving it only removes the session record.");
|
||||
}
|
||||
|
||||
if (runtimeServices.some((service) => service.status !== "stopped")) {
|
||||
if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) {
|
||||
warnings.push(
|
||||
runtimeServices.length === 1
|
||||
? "Closing this workspace will stop 1 attached runtime service."
|
||||
@@ -2291,7 +2519,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
},
|
||||
];
|
||||
|
||||
if (runtimeServices.some((service) => service.status !== "stopped")) {
|
||||
if (runtimeServices.some((service) => ACTIVE_RUNTIME_SERVICE_STATUSES.has(service.actualState))) {
|
||||
plannedActions.push({
|
||||
kind: "stop_runtime_services",
|
||||
label: runtimeServices.length === 1 ? "Stop attached runtime service" : "Stop attached runtime services",
|
||||
@@ -3225,7 +3453,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
);
|
||||
assertBranchReconcileRuntimeServicesStopped({
|
||||
inspection,
|
||||
runtimeServices: (runtimeServicesByWorkspaceId.get(existing.id) ?? []).map(toRuntimeService),
|
||||
runtimeServices: (
|
||||
runtimeServicesByWorkspaceId.get(existing.id)?.runtimeServices ?? []
|
||||
).map(toRuntimeService),
|
||||
});
|
||||
// The git rescue has to happen before the DB transaction because the
|
||||
// transaction may be retried/rolled back, while git side effects cannot.
|
||||
@@ -3303,8 +3533,14 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
lockedRow.companyId,
|
||||
[lockedRow],
|
||||
);
|
||||
const lockedRuntimeServices = (lockedRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService);
|
||||
const lockedWorkspace = toExecutionWorkspace(lockedRow, lockedRuntimeServices);
|
||||
const lockedRuntimeProjection = lockedRuntimeServicesByWorkspaceId.get(lockedRow.id);
|
||||
const lockedRuntimeServices = (lockedRuntimeProjection?.runtimeServices ?? []).map(toRuntimeService);
|
||||
const lockedWorkspace = toExecutionWorkspace(
|
||||
lockedRow,
|
||||
lockedRuntimeServices,
|
||||
"unknown",
|
||||
lockedRuntimeProjection?.effectiveRuntimeConfig ?? null,
|
||||
);
|
||||
if (!lockedWorkspace.sourceIssueId) {
|
||||
throw unprocessable("Execution workspace needs a source issue before Paperclip can audit branch reconciliation");
|
||||
}
|
||||
@@ -3352,7 +3588,9 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic
|
||||
lockedRow.companyId,
|
||||
[lockedRow],
|
||||
);
|
||||
const latestRuntimeServices = (latestRuntimeServicesByWorkspaceId.get(lockedRow.id) ?? []).map(toRuntimeService);
|
||||
const latestRuntimeServices = (
|
||||
latestRuntimeServicesByWorkspaceId.get(lockedRow.id)?.runtimeServices ?? []
|
||||
).map(toRuntimeService);
|
||||
assertBranchReconcileWorkspaceIsSafe({
|
||||
workspaceStatus: lockedWorkspace.status,
|
||||
inspection,
|
||||
|
||||
@@ -24,11 +24,15 @@ import {
|
||||
type ProjectManagedByPlugin,
|
||||
type ProjectWorkspaceRuntimeConfig,
|
||||
type ProjectWorkspace,
|
||||
type WorkspaceRuntimeDesiredState,
|
||||
type WorkspaceRuntimeService,
|
||||
type PluginManagedProjectDeclaration,
|
||||
type PluginManagedProjectResolution,
|
||||
} from "@paperclipai/shared";
|
||||
import { listCurrentRuntimeServicesForProjectWorkspaces } from "./workspace-runtime-read-model.js";
|
||||
import {
|
||||
listCurrentRuntimeServicesForProjectWorkspaces,
|
||||
selectConfiguredRuntimeServiceRows,
|
||||
} from "./workspace-runtime-read-model.js";
|
||||
import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-policy.js";
|
||||
import { mergeProjectWorkspaceRuntimeConfig, readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js";
|
||||
import { resolveManagedProjectWorkspaceDir } from "../home-paths.js";
|
||||
@@ -36,6 +40,11 @@ import { resolveManagedProjectWorkspaceDir } from "../home-paths.js";
|
||||
type ProjectRow = typeof projects.$inferSelect;
|
||||
type ProjectWorkspaceRow = typeof projectWorkspaces.$inferSelect;
|
||||
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
|
||||
type ProjectedWorkspaceRuntimeServiceRow = WorkspaceRuntimeServiceRow & {
|
||||
configIndex?: number | null;
|
||||
workspaceCommandId?: string | null;
|
||||
desiredState?: WorkspaceRuntimeDesiredState | null;
|
||||
};
|
||||
const REPO_ONLY_CWD_SENTINEL = "/__paperclip_repo_only__";
|
||||
type CreateWorkspaceInput = {
|
||||
name?: string | null;
|
||||
@@ -117,7 +126,7 @@ async function attachGoals(db: Db, rows: ProjectRow[]): Promise<ProjectWithGoals
|
||||
});
|
||||
}
|
||||
|
||||
function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeService {
|
||||
function toRuntimeService(row: ProjectedWorkspaceRuntimeServiceRow): WorkspaceRuntimeService {
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -129,6 +138,8 @@ function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeServ
|
||||
scopeId: row.scopeId ?? null,
|
||||
serviceName: row.serviceName,
|
||||
status: row.status as WorkspaceRuntimeService["status"],
|
||||
actualState: row.status as WorkspaceRuntimeService["actualState"],
|
||||
desiredState: row.desiredState ?? null,
|
||||
lifecycle: row.lifecycle as WorkspaceRuntimeService["lifecycle"],
|
||||
reuseKey: row.reuseKey ?? null,
|
||||
command: row.command ?? null,
|
||||
@@ -144,11 +155,29 @@ function toRuntimeService(row: WorkspaceRuntimeServiceRow): WorkspaceRuntimeServ
|
||||
stoppedAt: row.stoppedAt ?? null,
|
||||
stopPolicy: (row.stopPolicy as Record<string, unknown> | null) ?? null,
|
||||
healthStatus: row.healthStatus as WorkspaceRuntimeService["healthStatus"],
|
||||
configIndex: row.configIndex ?? null,
|
||||
workspaceCommandId: row.workspaceCommandId ?? null,
|
||||
latestFailure: null,
|
||||
createdAt: row.createdAt,
|
||||
updatedAt: row.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function projectWorkspaceRuntimeServices(
|
||||
workspace: ProjectWorkspaceRow,
|
||||
rows: WorkspaceRuntimeServiceRow[],
|
||||
) {
|
||||
const runtimeConfig = readProjectWorkspaceRuntimeConfig(
|
||||
(workspace.metadata as Record<string, unknown> | null) ?? null,
|
||||
);
|
||||
return selectConfiguredRuntimeServiceRows(
|
||||
rows,
|
||||
runtimeConfig?.workspaceRuntime ?? null,
|
||||
runtimeConfig?.desiredState ?? null,
|
||||
runtimeConfig?.serviceStates ?? null,
|
||||
).map(toRuntimeService);
|
||||
}
|
||||
|
||||
function toWorkspace(
|
||||
row: ProjectWorkspaceRow,
|
||||
runtimeServices: WorkspaceRuntimeService[] = [],
|
||||
@@ -246,9 +275,12 @@ async function attachWorkspaces(db: Db, rows: ProjectWithGoals[]): Promise<Proje
|
||||
workspaceRows.map((workspace) => workspace.id),
|
||||
);
|
||||
const sharedRuntimeServicesByWorkspaceId = new Map(
|
||||
Array.from(runtimeServicesByWorkspaceId.entries()).map(([workspaceId, services]) => [
|
||||
workspaceId,
|
||||
services.map(toRuntimeService),
|
||||
workspaceRows.map((workspace) => [
|
||||
workspace.id,
|
||||
projectWorkspaceRuntimeServices(
|
||||
workspace,
|
||||
runtimeServicesByWorkspaceId.get(workspace.id) ?? [],
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
@@ -891,7 +923,7 @@ export function projectService(db: Db) {
|
||||
return rows.map((row) =>
|
||||
toWorkspace(
|
||||
row,
|
||||
(runtimeServicesByWorkspaceId.get(row.id) ?? []).map(toRuntimeService),
|
||||
projectWorkspaceRuntimeServices(row, runtimeServicesByWorkspaceId.get(row.id) ?? []),
|
||||
),
|
||||
);
|
||||
},
|
||||
|
||||
@@ -3,13 +3,109 @@ import type { Db } from "@paperclipai/db";
|
||||
import { workspaceOperations } from "@paperclipai/db";
|
||||
import type { WorkspaceOperation, WorkspaceOperationPhase, WorkspaceOperationStatus } from "@paperclipai/shared";
|
||||
import { asc, desc, eq, inArray, isNull, or, and } from "drizzle-orm";
|
||||
import { notFound } from "../errors.js";
|
||||
import { HttpError, notFound } from "../errors.js";
|
||||
import { redactCurrentUserText, redactCurrentUserValue } from "../log-redaction.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { getWorkspaceOperationLogStore } from "./workspace-operation-log-store.js";
|
||||
|
||||
type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
|
||||
|
||||
export interface WorkspaceOperationFailureEvidence {
|
||||
operationId: string;
|
||||
operationLogPath: string;
|
||||
code: string;
|
||||
message: string;
|
||||
remediation: string;
|
||||
details: {
|
||||
port?: number;
|
||||
attemptedPortCount?: number;
|
||||
conflictingExecutionWorkspaceId?: string;
|
||||
conflictingProjectWorkspaceId?: string;
|
||||
} | null;
|
||||
failedAt: string;
|
||||
}
|
||||
|
||||
const WORKSPACE_OPERATION_FAILURE_PROPERTY = "paperclipWorkspaceOperationFailure";
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function readSafeFailureDetails(value: unknown): WorkspaceOperationFailureEvidence["details"] {
|
||||
if (!isRecord(value)) return null;
|
||||
const details: NonNullable<WorkspaceOperationFailureEvidence["details"]> = {};
|
||||
if (typeof value.port === "number" && Number.isInteger(value.port)) details.port = value.port;
|
||||
if (typeof value.attemptedPortCount === "number" && Number.isInteger(value.attemptedPortCount)) {
|
||||
details.attemptedPortCount = value.attemptedPortCount;
|
||||
}
|
||||
if (typeof value.conflictingExecutionWorkspaceId === "string") {
|
||||
details.conflictingExecutionWorkspaceId = value.conflictingExecutionWorkspaceId;
|
||||
}
|
||||
if (typeof value.conflictingProjectWorkspaceId === "string") {
|
||||
details.conflictingProjectWorkspaceId = value.conflictingProjectWorkspaceId;
|
||||
}
|
||||
return Object.keys(details).length > 0 ? details : null;
|
||||
}
|
||||
|
||||
function buildWorkspaceOperationFailureEvidence(input: {
|
||||
operationId: string;
|
||||
failedAt: Date;
|
||||
error: unknown;
|
||||
metadata: Record<string, unknown> | null | undefined;
|
||||
}): WorkspaceOperationFailureEvidence {
|
||||
const httpDetails = input.error instanceof HttpError && isRecord(input.error.details)
|
||||
? input.error.details
|
||||
: null;
|
||||
const action = typeof input.metadata?.action === "string" ? input.metadata.action : null;
|
||||
const isRuntimeStart = action === "start" || action === "restart";
|
||||
const code = typeof httpDetails?.code === "string"
|
||||
? httpDetails.code
|
||||
: isRuntimeStart
|
||||
? "workspace_runtime_start_failed"
|
||||
: "workspace_operation_failed";
|
||||
const message = input.error instanceof HttpError
|
||||
? input.error.message
|
||||
: isRuntimeStart
|
||||
? "Workspace runtime service failed to start."
|
||||
: "Workspace operation failed.";
|
||||
const remediation = typeof httpDetails?.remediation === "string"
|
||||
? httpDetails.remediation
|
||||
: "Review the workspace operation log, correct the runtime configuration or dependency, and retry.";
|
||||
|
||||
return {
|
||||
operationId: input.operationId,
|
||||
operationLogPath: `/api/workspace-operations/${input.operationId}/log`,
|
||||
code,
|
||||
message,
|
||||
remediation,
|
||||
details: readSafeFailureDetails(httpDetails),
|
||||
failedAt: input.failedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function attachWorkspaceOperationFailureEvidence(
|
||||
error: unknown,
|
||||
evidence: WorkspaceOperationFailureEvidence,
|
||||
) {
|
||||
const target = error instanceof Error ? error : new Error(String(error));
|
||||
Object.defineProperty(target, WORKSPACE_OPERATION_FAILURE_PROPERTY, {
|
||||
configurable: true,
|
||||
enumerable: false,
|
||||
value: evidence,
|
||||
writable: true,
|
||||
});
|
||||
return target;
|
||||
}
|
||||
|
||||
export function getWorkspaceOperationFailureEvidence(
|
||||
error: unknown,
|
||||
): WorkspaceOperationFailureEvidence | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
const value = (error as unknown as Record<string, unknown>)[WORKSPACE_OPERATION_FAILURE_PROPERTY];
|
||||
if (!isRecord(value)) return null;
|
||||
return value as unknown as WorkspaceOperationFailureEvidence;
|
||||
}
|
||||
|
||||
function toWorkspaceOperation(row: WorkspaceOperationRow): WorkspaceOperation {
|
||||
return {
|
||||
id: row.id,
|
||||
@@ -187,6 +283,12 @@ export function workspaceOperationService(db: Db) {
|
||||
await append("stderr", error instanceof Error ? error.message : String(error));
|
||||
const finalized = await logStore.finalize(handle).catch(() => null);
|
||||
const finishedAt = new Date();
|
||||
const failureEvidence = buildWorkspaceOperationFailureEvidence({
|
||||
operationId: id,
|
||||
failedAt: finishedAt,
|
||||
error,
|
||||
metadata: recordInput.metadata,
|
||||
});
|
||||
await db
|
||||
.update(workspaceOperations)
|
||||
.set({
|
||||
@@ -197,11 +299,15 @@ export function workspaceOperationService(db: Db) {
|
||||
logBytes: finalized?.bytes ?? null,
|
||||
logSha256: finalized?.sha256 ?? null,
|
||||
logCompressed: finalized?.compressed ?? false,
|
||||
metadata: redactCurrentUserValue(
|
||||
combineMetadata(recordInput.metadata, { failureEvidence }),
|
||||
currentUserRedactionOptions,
|
||||
) as Record<string, unknown> | null,
|
||||
finishedAt,
|
||||
updatedAt: finishedAt,
|
||||
})
|
||||
.where(eq(workspaceOperations.id, id));
|
||||
throw error;
|
||||
throw attachWorkspaceOperationFailureEvidence(error, failureEvidence);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
@@ -53,7 +53,7 @@ describe("selectConfiguredRuntimeServiceRows", () => {
|
||||
expect(selectConfiguredRuntimeServiceRows(historicalRows, null)).toEqual([]);
|
||||
});
|
||||
|
||||
it("selects the newest current row for each configured service and annotates its config index", () => {
|
||||
it("selects the newest configured row including failed state and annotates effective intent", () => {
|
||||
const oldWeb = runtimeServiceRow({
|
||||
serviceName: "web",
|
||||
command: "pnpm dev",
|
||||
@@ -62,6 +62,8 @@ describe("selectConfiguredRuntimeServiceRows", () => {
|
||||
const currentWeb = runtimeServiceRow({
|
||||
serviceName: "web",
|
||||
command: "pnpm dev",
|
||||
status: "failed",
|
||||
healthStatus: "unhealthy",
|
||||
updatedAt: new Date("2026-07-30T10:00:00.000Z"),
|
||||
});
|
||||
const removedWorker = runtimeServiceRow({
|
||||
@@ -73,13 +75,18 @@ describe("selectConfiguredRuntimeServiceRows", () => {
|
||||
const selected = selectConfiguredRuntimeServiceRows(
|
||||
[removedWorker, currentWeb, oldWeb],
|
||||
{ services: [{ name: "web", command: "pnpm dev" }] },
|
||||
"stopped",
|
||||
{ "0": "running" },
|
||||
);
|
||||
|
||||
expect(selected).toEqual([
|
||||
expect.objectContaining({
|
||||
id: currentWeb.id,
|
||||
serviceName: "web",
|
||||
status: "failed",
|
||||
configIndex: 0,
|
||||
workspaceCommandId: "service:web",
|
||||
desiredState: "running",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
listWorkspaceServiceCommandDefinitions,
|
||||
matchWorkspaceRuntimeServiceToCommand,
|
||||
} from "@paperclipai/shared";
|
||||
import type { WorkspaceRuntimeDesiredState, WorkspaceRuntimeServiceStateMap } from "@paperclipai/shared";
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
|
||||
type WorkspaceRuntimeServiceRow = typeof workspaceRuntimeServices.$inferSelect;
|
||||
@@ -34,12 +35,20 @@ export function selectCurrentRuntimeServiceRows(rows: WorkspaceRuntimeServiceRow
|
||||
export function selectConfiguredRuntimeServiceRows(
|
||||
rows: WorkspaceRuntimeServiceRow[],
|
||||
workspaceRuntime: Record<string, unknown> | null | undefined,
|
||||
desiredState: WorkspaceRuntimeDesiredState | null = null,
|
||||
serviceStates: WorkspaceRuntimeServiceStateMap | null = null,
|
||||
) {
|
||||
const availableRows = selectCurrentRuntimeServiceRows(rows).map((row) => ({
|
||||
...row,
|
||||
configIndex: null as number | null,
|
||||
workspaceCommandId: null as string | null,
|
||||
desiredState: null as WorkspaceRuntimeDesiredState | null,
|
||||
}));
|
||||
const selectedRows: Array<WorkspaceRuntimeServiceRow & { configIndex: number | null }> = [];
|
||||
const selectedRows: Array<WorkspaceRuntimeServiceRow & {
|
||||
configIndex: number | null;
|
||||
workspaceCommandId: string | null;
|
||||
desiredState: WorkspaceRuntimeDesiredState | null;
|
||||
}> = [];
|
||||
|
||||
for (const command of listWorkspaceServiceCommandDefinitions(workspaceRuntime)) {
|
||||
const reuseScope = command.rawConfig.reuseScope;
|
||||
@@ -60,6 +69,10 @@ export function selectConfiguredRuntimeServiceRows(
|
||||
selectedRows.push({
|
||||
...matchedRow,
|
||||
configIndex: command.serviceIndex,
|
||||
workspaceCommandId: command.id,
|
||||
desiredState:
|
||||
(command.serviceIndex === null ? null : serviceStates?.[String(command.serviceIndex)])
|
||||
?? desiredState,
|
||||
});
|
||||
availableRows.splice(availableRows.indexOf(matchedRow), 1);
|
||||
}
|
||||
|
||||
@@ -186,6 +186,24 @@ const runtimeProvisionByWorkspace = new Map<string, Promise<void>>();
|
||||
const DEFAULT_EXECUTE_PROCESS_OUTPUT_BYTES = 256 * 1024;
|
||||
export const WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS = 32;
|
||||
const ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES = ["provisioning", "starting", "running"] as const;
|
||||
const RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY = "paperclipRuntimeServiceFailureRecord";
|
||||
|
||||
function attachRuntimeServiceFailureRecord(error: unknown, record: RuntimeServiceRecord) {
|
||||
const target = error instanceof Error ? error : new Error(String(error));
|
||||
Object.defineProperty(target, RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY, {
|
||||
configurable: true,
|
||||
enumerable: false,
|
||||
value: record,
|
||||
writable: true,
|
||||
});
|
||||
return target;
|
||||
}
|
||||
|
||||
function getRuntimeServiceFailureRecord(error: unknown): RuntimeServiceRecord | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
const value = (error as unknown as Record<string, unknown>)[RUNTIME_SERVICE_FAILURE_RECORD_PROPERTY];
|
||||
return value && typeof value === "object" ? value as RuntimeServiceRecord : null;
|
||||
}
|
||||
|
||||
class RuntimeServicePortBindCollision extends Error {
|
||||
readonly port: number;
|
||||
@@ -905,7 +923,12 @@ async function findActiveRuntimeServiceBlockingDirtyQuarantine(input: {
|
||||
scopeType: workspaceRuntimeServices.scopeType,
|
||||
})
|
||||
.from(workspaceRuntimeServices)
|
||||
.where(and(serviceScopeCondition, ne(workspaceRuntimeServices.status, "stopped")))
|
||||
.where(
|
||||
and(
|
||||
serviceScopeCondition,
|
||||
inArray(workspaceRuntimeServices.status, [...ACTIVE_RUNTIME_PORT_RESERVATION_STATUSES]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(workspaceRuntimeServices.updatedAt), desc(workspaceRuntimeServices.createdAt))
|
||||
.limit(1);
|
||||
return service ?? null;
|
||||
@@ -4153,7 +4176,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: {
|
||||
eq(workspaceRuntimeServices.companyId, input.companyId),
|
||||
eq(workspaceRuntimeServices.reuseKey, input.reuseKey),
|
||||
eq(workspaceRuntimeServices.provider, "local_process"),
|
||||
eq(workspaceRuntimeServices.status, "stopped"),
|
||||
inArray(workspaceRuntimeServices.status, ["stopped", "failed"]),
|
||||
),
|
||||
)
|
||||
.orderBy(desc(workspaceRuntimeServices.updatedAt))
|
||||
@@ -4175,7 +4198,7 @@ async function findStoppedRuntimeServiceReuseCandidate(input: {
|
||||
and(
|
||||
eq(workspaceRuntimeServices.companyId, input.companyId),
|
||||
eq(workspaceRuntimeServices.provider, "local_process"),
|
||||
eq(workspaceRuntimeServices.status, "stopped"),
|
||||
inArray(workspaceRuntimeServices.status, ["stopped", "failed"]),
|
||||
eq(workspaceRuntimeServices.scopeType, input.scopeType),
|
||||
scopeIdCondition,
|
||||
eq(workspaceRuntimeServices.serviceName, input.serviceName),
|
||||
@@ -4770,7 +4793,7 @@ async function spawnLocalRuntimeService(input: StartLocalRuntimeServiceInput): P
|
||||
processGroupId: child.pid,
|
||||
});
|
||||
}
|
||||
record.status = "stopped";
|
||||
record.status = "failed";
|
||||
record.healthStatus = "unhealthy";
|
||||
record.lastUsedAt = new Date().toISOString();
|
||||
record.stoppedAt = new Date().toISOString();
|
||||
@@ -4856,28 +4879,54 @@ async function startLocalRuntimeService(
|
||||
);
|
||||
const retryBindCollisions = fixedPortFallbackEnabled || portType === "auto";
|
||||
const deferReadiness = Boolean(options?.deferReadiness && !fixedPortFallbackEnabled);
|
||||
const identity = resolveRuntimeServiceReuseIdentity({
|
||||
service: input.service,
|
||||
workspace: input.workspace,
|
||||
agent: input.agent,
|
||||
issue: input.issue,
|
||||
adapterEnv: input.adapterEnv,
|
||||
scopeType: input.scopeType,
|
||||
scopeId: input.scopeId,
|
||||
});
|
||||
if (!identity.command) throw new Error(`Runtime service "${identity.serviceName}" is missing command`);
|
||||
let currentAttemptRecord = provisioningRecord ?? createProvisioningRuntimeServiceRecord(input, identity);
|
||||
if (!provisioningRecord) {
|
||||
currentAttemptRecord.status = "starting";
|
||||
await persistRuntimeServiceRecord(input.db, currentAttemptRecord);
|
||||
}
|
||||
let retryRuntimeServiceId = input.runtimeServiceId;
|
||||
|
||||
try {
|
||||
for (let attempt = 0; attempt < WORKSPACE_RUNTIME_PORT_ALLOCATION_ATTEMPTS; attempt += 1) {
|
||||
try {
|
||||
const previousAttemptRecord = currentAttemptRecord;
|
||||
started = await spawnLocalRuntimeService({
|
||||
...input,
|
||||
excludedPorts,
|
||||
runtimeServiceId: provisioningRecord?.id ?? input.runtimeServiceId,
|
||||
runtimeServiceId: provisioningRecord?.id ?? retryRuntimeServiceId,
|
||||
});
|
||||
retryRuntimeServiceId = started.record.id;
|
||||
currentAttemptRecord = started.record;
|
||||
if (runtimeProvisionCommand) {
|
||||
await persistRuntimeServiceRecord(input.db, started.record);
|
||||
}
|
||||
if (provisioningRecord && started.record.id !== provisioningRecord.id && input.db) {
|
||||
if (started.record.id !== previousAttemptRecord.id && input.db) {
|
||||
await input.db
|
||||
.delete(workspaceRuntimeServices)
|
||||
.where(eq(workspaceRuntimeServices.id, provisioningRecord.id));
|
||||
.where(eq(workspaceRuntimeServices.id, previousAttemptRecord.id));
|
||||
}
|
||||
if (!deferReadiness) {
|
||||
await started.readiness;
|
||||
}
|
||||
return started;
|
||||
} catch (error) {
|
||||
const failedRecord = started?.record ?? currentAttemptRecord;
|
||||
const nowIso = new Date().toISOString();
|
||||
failedRecord.status = "failed";
|
||||
failedRecord.healthStatus = "unhealthy";
|
||||
failedRecord.lastUsedAt = nowIso;
|
||||
failedRecord.stoppedAt = nowIso;
|
||||
await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined);
|
||||
if (!(error instanceof RuntimeServicePortBindCollision) || !retryBindCollisions) throw error;
|
||||
excludedPorts.add(error.port);
|
||||
started = null;
|
||||
@@ -4899,15 +4948,15 @@ async function startLocalRuntimeService(
|
||||
remediation: "Retry the start or configure a different runtime service port.",
|
||||
});
|
||||
} catch (error) {
|
||||
if (!started && provisioningRecord && provisioningRecord.status === "starting") {
|
||||
if (!started && currentAttemptRecord.status !== "failed") {
|
||||
const nowIso = new Date().toISOString();
|
||||
provisioningRecord.status = "failed";
|
||||
provisioningRecord.healthStatus = "unhealthy";
|
||||
provisioningRecord.lastUsedAt = nowIso;
|
||||
provisioningRecord.stoppedAt = nowIso;
|
||||
await persistRuntimeServiceRecord(input.db, provisioningRecord).catch(() => undefined);
|
||||
currentAttemptRecord.status = "failed";
|
||||
currentAttemptRecord.healthStatus = "unhealthy";
|
||||
currentAttemptRecord.lastUsedAt = nowIso;
|
||||
currentAttemptRecord.stoppedAt = nowIso;
|
||||
await persistRuntimeServiceRecord(input.db, currentAttemptRecord).catch(() => undefined);
|
||||
}
|
||||
throw error;
|
||||
throw attachRuntimeServiceFailureRecord(error, currentAttemptRecord);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4921,12 +4970,13 @@ function scheduleIdleStop(record: RuntimeServiceRecord) {
|
||||
}, idleSeconds * 1000);
|
||||
}
|
||||
|
||||
async function stopRuntimeService(serviceId: string) {
|
||||
async function stopRuntimeService(serviceId: string, options?: { preserveFailure?: boolean }) {
|
||||
const record = runtimeServicesById.get(serviceId);
|
||||
if (!record) return;
|
||||
clearIdleTimer(record);
|
||||
record.status = "stopped";
|
||||
record.healthStatus = "unknown";
|
||||
const preserveFailure = options?.preserveFailure === true && record.status === "failed";
|
||||
record.status = preserveFailure ? "failed" : "stopped";
|
||||
record.healthStatus = preserveFailure ? "unhealthy" : "unknown";
|
||||
record.lastUsedAt = new Date().toISOString();
|
||||
record.stoppedAt = new Date().toISOString();
|
||||
runtimeServicesById.delete(serviceId);
|
||||
@@ -5488,7 +5538,16 @@ export async function startRuntimeServicesForWorkspaceControl(
|
||||
});
|
||||
} catch (error) {
|
||||
for (const serviceId of startBatch.startedServiceIds) {
|
||||
await stopRuntimeService(serviceId).catch(() => undefined);
|
||||
await stopRuntimeService(serviceId, { preserveFailure: true }).catch(() => undefined);
|
||||
}
|
||||
const failedRecord = getRuntimeServiceFailureRecord(error);
|
||||
if (failedRecord) {
|
||||
const nowIso = new Date().toISOString();
|
||||
failedRecord.status = "failed";
|
||||
failedRecord.healthStatus = "unhealthy";
|
||||
failedRecord.lastUsedAt = nowIso;
|
||||
failedRecord.stoppedAt = nowIso;
|
||||
await persistRuntimeServiceRecord(input.db, failedRecord).catch(() => undefined);
|
||||
}
|
||||
if (preparedProvisioning && startBatch.startedServiceIds.length === 0) {
|
||||
const nowIso = new Date().toISOString();
|
||||
|
||||
Reference in new issue
Block a user