From 6e14ed3031f576d42edc9d4c4fa074893b06c4f0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 11 Aug 2026 10:49:05 +0000 Subject: [PATCH] fix: reconcile desired workspace runtimes on startup Co-Authored-By: Paperclip --- .../src/__tests__/workspace-runtime.test.ts | 155 ++++++++++++++++++ server/src/index.ts | 10 +- server/src/services/workspace-runtime.ts | 17 +- 3 files changed, 177 insertions(+), 5 deletions(-) diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index 1d97db74918..4942658d9f6 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -6410,6 +6410,161 @@ describeEmbeddedPostgres("workspace runtime startup reconciliation", () => { await db.delete(companies); }); + it("restores desired services when one row is stopped and a live registered service has no row", async () => { + const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-desired-reconcile-")); + const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-")); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = `runtime-desired-reconcile-${randomUUID()}`; + + const reservePort = async () => { + const probe = net.createServer(); + await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); + const address = probe.address(); + const port = typeof address === "object" && address ? address.port : null; + await new Promise((resolve, reject) => { + probe.close((error) => error ? reject(error) : resolve()); + }); + if (!port) throw new Error("Failed to reserve runtime reconciliation test port"); + return port; + }; + const stoppedPort = await reservePort(); + const livePort = await reservePort(); + const companyId = randomUUID(); + const projectId = randomUUID(); + const projectWorkspaceId = randomUUID(); + const executionWorkspaceId = randomUUID(); + const command = + "node -e \"require('node:http').createServer((req,res)=>res.end(process.env.PORT)).listen(Number(process.env.PORT), '127.0.0.1')\""; + const workspaceRuntime = { + services: [ + { + name: "persisted-service", + command, + port: stoppedPort, + expose: { urlTemplate: "http://127.0.0.1:{{port}}" }, + readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 }, + lifecycle: "shared", + reuseScope: "execution_workspace", + stopPolicy: { type: "manual" }, + }, + { + name: "registry-only-service", + command, + port: livePort, + expose: { urlTemplate: "http://127.0.0.1:{{port}}" }, + readiness: { type: "http", urlTemplate: "http://127.0.0.1:{{port}}", timeoutSec: 10, intervalMs: 50 }, + lifecycle: "shared", + reuseScope: "execution_workspace", + stopPolicy: { type: "manual" }, + }, + ], + }; + + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Runtime desired reconciliation", + status: "in_progress", + }); + await db.insert(projectWorkspaces).values({ + id: projectWorkspaceId, + companyId, + projectId, + name: "Primary", + sourceType: "local_path", + cwd: workspaceRoot, + isPrimary: true, + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + projectWorkspaceId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "Runtime desired reconciliation workspace", + status: "active", + cwd: workspaceRoot, + providerType: "local_fs", + providerRef: workspaceRoot, + metadata: { + config: { + workspaceRuntime, + desiredState: "running", + serviceStates: { "0": "running", "1": "running" }, + }, + }, + }); + + const actor = { id: null, name: "Paperclip", companyId }; + const workspace = { + ...buildWorkspace(workspaceRoot), + projectId, + workspaceId: projectWorkspaceId, + }; + let stoppedServiceId: string | null = null; + let registryOnlyServiceId: string | null = null; + + try { + const persisted = await startRuntimeServicesForWorkspaceControl({ + db, + actor, + issue: null, + workspace, + executionWorkspaceId, + config: { workspaceRuntime }, + adapterEnv: {}, + serviceIndex: 0, + }); + stoppedServiceId = persisted[0]?.id ?? null; + await stopRuntimeServicesForExecutionWorkspace({ + db, + executionWorkspaceId, + workspaceCwd: workspaceRoot, + runtimeServiceId: stoppedServiceId, + }); + + const registryOnly = await startRuntimeServicesForWorkspaceControl({ + actor, + issue: null, + workspace, + executionWorkspaceId, + config: { workspaceRuntime }, + adapterEnv: {}, + serviceIndex: 1, + }); + registryOnlyServiceId = registryOnly[0]?.id ?? null; + expect(registryOnlyServiceId).toBeTruthy(); + expect(await db.select().from(workspaceRuntimeServices)).toHaveLength(1); + + await resetRuntimeServicesForTests(); + const result = await reconcilePersistedRuntimeServicesOnStartup(db); + + expect(result).toMatchObject({ reconciled: 0, adopted: 0, stopped: 0, restarted: 1, restartFailed: 0 }); + const rows = await db.select().from(workspaceRuntimeServices); + expect(rows).toHaveLength(2); + expect(rows).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: stoppedServiceId, port: stoppedPort, status: "running" }), + expect.objectContaining({ id: registryOnlyServiceId, port: livePort, status: "running" }), + ])); + await expect(fetch(`http://127.0.0.1:${stoppedPort}`)).resolves.toMatchObject({ ok: true }); + await expect(fetch(`http://127.0.0.1:${livePort}`)).resolves.toMatchObject({ ok: true }); + } finally { + await stopRuntimeServicesForExecutionWorkspace({ + db, + executionWorkspaceId, + workspaceCwd: workspaceRoot, + }); + await fs.rm(paperclipHome, { recursive: true, force: true }); + } + }, 20_000); + it("adopts a live auto-port shared service after runtime state is reset", async () => { const workspaceRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-reconcile-")); const paperclipHome = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-runtime-home-")); diff --git a/server/src/index.ts b/server/src/index.ts index c338564356d..22d914d7a1c 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -818,9 +818,15 @@ export async function startServer(): Promise { void reconcilePersistedRuntimeServicesOnStartup(db as any) .then((result) => { - if (result.reconciled > 0) { + if (result.reconciled > 0 || result.restarted > 0 || result.restartFailed > 0) { logger.warn( - { reconciled: result.reconciled }, + { + reconciled: result.reconciled, + adopted: result.adopted, + stopped: result.stopped, + restarted: result.restarted, + restartFailed: result.restartFailed, + }, "reconciled persisted runtime services from a previous server process", ); } diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index b3ce928873c..6707038abc7 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -5777,8 +5777,6 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { ), ); - if (rows.length === 0) return { reconciled: 0, adopted: 0, stopped: 0 }; - let reconciled = 0; let adopted = 0; let stopped = 0; @@ -5904,7 +5902,20 @@ export async function reconcilePersistedRuntimeServicesOnStartup(db: Db) { stopped += 1; } - return { reconciled, adopted, stopped }; + // Row reconciliation alone cannot repair the process-start crash window: the + // local service registry may contain a healthy managed process whose DB row + // was never committed. Re-applying persisted desired state adopts that + // registry entry (or restarts a missing service) and makes it visible to + // workspace cleanup through workspace_runtime_services again. + const desiredState = await restartDesiredRuntimeServicesOnStartup(db); + + return { + reconciled, + adopted, + stopped, + restarted: desiredState.restarted, + restartFailed: desiredState.failed, + }; } export async function restartDesiredRuntimeServicesOnStartup(db: Db) {