feat(workspaces): verify seeds and add managed repair

Replace binary readiness markers with an evidence-bearing seed manifest and enforce a shared validation contract for minimal and full clones. Add an exclusive audited database repair path with recoverable backup, bounded phase diagnostics, and healthy restart validation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-18 23:20:22 +00:00
1 parent 4b968d8c05
commit 53a0aa3020
21 files changed
+1568 -154

No files matched your search

@@ -36,6 +36,36 @@ describe("dev-runner worktree env bootstrap", () => {
expect(isWorktreeSeedPending(root)).toBe(false);
});
it("guards every manifest state except a complete verified manifest", () => {
const root = createTempRoot("paperclip-dev-runner-seed-manifest-");
const manifestPath = path.join(root, ".paperclip", "seed-manifest.json");
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "failed" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "verified" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "minimal",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}), "utf8");
expect(isWorktreeSeedPending(root)).toBe(false);
fs.writeFileSync(manifestPath, "not-json", "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
});
it("detects linked git worktrees from .git files", () => {
const root = createTempRoot("paperclip-dev-runner-worktree-");
fs.writeFileSync(path.join(root, ".git"), "gitdir: /tmp/paperclip/.git/worktrees/feature\n", "utf8");
@@ -1,3 +1,8 @@
import { EventEmitter } from "node:events";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -29,6 +34,12 @@ const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
// `execution-workspace-runtime-lease-route.test.ts` exercises the lease itself against a real db.
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
const mockSpawn = vi.hoisted(() => vi.fn());
vi.mock("node:child_process", async () => ({
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
spawn: mockSpawn,
}));
vi.mock("../telemetry.js", () => ({ getTelemetryClient: mockGetTelemetryClient }));
@@ -45,7 +56,7 @@ vi.mock("../services/index.js", () => ({
claim: mockClaimRuntimeLease,
release: mockReleaseRuntimeLease,
}),
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
@@ -135,6 +146,7 @@ async function createApp() {
describe.sequential("execution workspace runtime control conflict and failure reconciliation", () => {
beforeEach(() => {
vi.clearAllMocks();
mockSpawn.mockReset();
mockAccessService.decide.mockResolvedValue({
allowed: true,
action: "runtime:manage",
@@ -165,7 +177,7 @@ describe.sequential("execution workspace runtime control conflict and failure re
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
// the error still propagates to the caller.
try {
await input.run();
await input.run(async () => undefined);
} catch (error) {
(error as any).recordedOperationStatus = "failed";
throw error;
@@ -247,6 +259,156 @@ describe.sequential("execution workspace runtime control conflict and failure re
);
});
it("runs an audited full repair through terminal seed and service readiness validation", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-success-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
fs.mkdirSync(configDir, { recursive: true });
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
fs.mkdirSync(path.dirname(cliRunner), { recursive: true });
fs.mkdirSync(path.dirname(cliEntry), { recursive: true });
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(sourceConfigPath, "{}\n");
fs.writeFileSync(cliRunner, "// test runner\n");
fs.writeFileSync(cliEntry, "// test entry\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
attemptId: "previous",
}));
const { deriveWorktreeInstanceId } = await import("../services/workspace-instance-cleanup.js");
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
mockStartRuntimeServices.mockResolvedValue([{
id: "service-1",
status: "running",
healthStatus: "healthy",
}]);
mockSpawn.mockImplementation((_command: string, args: string[]) => {
expect(args).toEqual(expect.arrayContaining([
"worktree", "reseed", "--seed-mode", "full", "--yes", "--backup-target",
]));
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0142_test.sql",
targetInstanceId: deriveWorktreeInstanceId(workspaceCwd),
phase: "complete",
state: "verified",
attemptId: "repair-attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}));
child.emit("exit", 0);
});
return child;
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(200);
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledWith(
expect.objectContaining({ executionWorkspaceId, workspaceCwd }),
);
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(1);
expect(mockLogActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ action: "execution_workspace.runtime_repair" }),
);
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it("fails repair at the exact seed phase and leaves managed services stopped", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-failure-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
for (const dir of [configDir, path.dirname(sourceConfigPath), path.dirname(cliRunner), path.dirname(cliEntry)]) {
fs.mkdirSync(dir, { recursive: true });
}
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(sourceConfigPath, "{}\n");
fs.writeFileSync(cliRunner, "// test runner\n");
fs.writeFileSync(cliEntry, "// test entry\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
attemptId: "previous",
}));
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
mockSpawn.mockImplementation(() => {
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
phase: "restore",
attemptId: "repair-attempt",
}));
child.emit("exit", 4);
});
return child;
});
const progress: Array<Record<string, unknown>> = [];
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async (input: any) => {
await input.run(async (entry: Record<string, unknown>) => {
progress.push(entry);
});
},
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBeGreaterThanOrEqual(400);
expect(progress).toEqual(expect.arrayContaining([
expect.objectContaining({ metadata: { seedFailurePhase: "restore" } }),
]));
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(2);
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it("reconciles once when the operation's own time budget fails a start that never settles", async () => {
const { WorkspaceOperationTimeoutError } = await import("../services/workspace-operations.js");
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
@@ -19,6 +19,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
const mockWorkspaceOperationService = vi.hoisted(() => ({
listForExecutionWorkspace: vi.fn(),
createRecorder: vi.fn(),
assertRuntimeControlAvailable: vi.fn(async () => undefined),
}));
const mockWorkspaceRuntimeLeaseService = vi.hoisted(() => ({
@@ -47,7 +48,7 @@ vi.mock("../services/index.js", () => ({
logActivity: mockLogActivity,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/environment-runtime.js", () => ({
@@ -44,14 +44,14 @@ describe("workspace runtime control serialization", () => {
await expect(runExclusiveWorkspaceRuntimeControl({
executionWorkspaceId: "workspace-1",
action: "restart",
run: async () => "restarted",
action: "repair",
run: async () => "repaired",
})).rejects.toMatchObject({
status: 409,
details: {
code: "workspace_runtime_control_in_progress",
activeAction: "start",
requestedAction: "restart",
requestedAction: "repair",
},
});
@@ -204,4 +204,71 @@ describeEmbeddedPostgres("workspace operation reconciliation", () => {
.then((rows) => rows[0]);
expect(unrelated?.status).toBe("running");
});
it("persists bounded repair phase progress on the operation row and log", async () => {
const companyId = randomUUID();
const projectId = randomUUID();
const executionWorkspaceId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Workspace repair diagnostics",
issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(projects).values({
id: projectId,
companyId,
name: "Repair diagnostics",
status: "in_progress",
});
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Repair diagnostics workspace",
status: "active",
cwd: "/tmp/repair-diagnostics-workspace",
});
const operation = await workspaceOperationService(db)
.createRecorder({ companyId, executionWorkspaceId })
.recordOperation({
phase: "workspace_repair",
command: "workspace command repair",
cwd: "/tmp/repair-diagnostics-workspace",
metadata: { action: "repair" },
run: async (reportProgress) => {
await reportProgress({
metadata: {
repairPhase: "target_backup",
repairDiagnostics: [{
phase: "target_backup",
status: "succeeded",
at: "2026-08-18T00:00:00.000Z",
}],
databaseOnly: true,
worktreePreserved: true,
},
system: "Workspace repair target_backup: succeeded.\n",
});
return { status: "succeeded", metadata: { backupRetained: true } };
},
});
expect(operation).toMatchObject({
status: "succeeded",
metadata: {
action: "repair",
repairPhase: "target_backup",
databaseOnly: true,
worktreePreserved: true,
backupRetained: true,
},
});
expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain(
"Workspace repair target_backup: succeeded.",
);
});
});
@@ -54,7 +54,7 @@ vi.mock("../services/index.js", () => ({
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
@@ -84,7 +84,7 @@ function registerWorkspaceRouteMocks() {
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.doMock("../services/workspace-runtime.js", () => ({
@@ -456,6 +456,39 @@ describe("resolveRuntimeProvisionCommand", () => {
await fs.writeFile(path.join(cwd, ".paperclip", "seed-complete"), "{}\n");
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "failed" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "verified" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
} finally {
await fs.rm(baseCwd, { recursive: true, force: true });
}
+5
View File
@@ -1,6 +1,7 @@
import { existsSync, lstatSync, readFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { hasVerifiedWorktreeSeedManifest } from "./worktree-seed-manifest.js";
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
@@ -58,6 +59,10 @@ export function resolveWorktreeEnvFilePath(rootDir: string): string {
export function isWorktreeSeedPending(rootDir: string): boolean {
const markerDir = path.resolve(rootDir, ".paperclip");
const manifestPath = path.resolve(markerDir, "seed-manifest.json");
if (existsSync(manifestPath)) {
return !hasVerifiedWorktreeSeedManifest(manifestPath);
}
return existsSync(path.resolve(markerDir, "seed-pending"))
&& !existsSync(path.resolve(markerDir, "seed-complete"));
}
+248 -8
View File
@@ -1,3 +1,6 @@
import { spawn } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { Router, type Request, type Response } from "express";
import type { Db } from "@paperclipai/db";
@@ -45,6 +48,8 @@ import { appendWithCap } from "../adapters/utils.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
import { deriveWorktreeInstanceId } from "../services/workspace-instance-cleanup.js";
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
@@ -151,7 +156,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
async function handleExecutionWorkspaceRuntimeCommand(req: Request, res: Response) {
const id = req.params.id as string;
const action = String(req.params.action ?? "").trim().toLowerCase();
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "run") {
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "repair" && action !== "run") {
res.status(404).json({ error: "Workspace command action not found" });
return;
}
@@ -262,7 +267,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
return;
}
if ((action === "start" || action === "restart") && !effectiveRuntimeConfig) {
if ((action === "start" || action === "restart" || action === "repair") && !effectiveRuntimeConfig) {
res.status(422).json({ error: "Execution workspace has no workspace command configuration or inherited project workspace default" });
return;
}
@@ -327,7 +332,11 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
}
const recordRuntimeControlOperation = () => recorder.recordOperation({
phase: action === "stop" ? "workspace_teardown" : "workspace_provision",
phase: action === "stop"
? "workspace_teardown"
: action === "repair"
? "workspace_repair"
: "workspace_provision",
command: workspaceCommand?.command ?? `workspace command ${action}`,
cwd: existing.cwd,
metadata: {
@@ -339,7 +348,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
runtimeServiceId: selectedRuntimeServiceId,
serviceIndex: selectedServiceIndex,
},
run: async () => {
run: async (reportProgress) => {
const ensureWorkspaceAvailable = async () =>
await ensurePersistedExecutionWorkspaceAvailable({
base: {
@@ -429,7 +438,234 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
else stderr = appendWithCap(stderr, chunk, WORKSPACE_CONTROL_OUTPUT_MAX_CHARS);
};
if (action === "stop" || action === "restart") {
if (action === "repair") {
type RepairPhase =
| "managed_stop"
| "target_backup"
| "full_reseed"
| "managed_restart"
| "readiness_validation";
const repairDiagnostics: Array<{
phase: RepairPhase;
status: "started" | "succeeded" | "failed";
at: string;
}> = [];
let repairPhase: RepairPhase = "managed_stop";
const reportRepairPhase = async (
phase: RepairPhase,
status: "started" | "succeeded" | "failed",
) => {
repairPhase = phase;
repairDiagnostics.push({ phase, status, at: new Date().toISOString() });
if (repairDiagnostics.length > 32) repairDiagnostics.splice(0, repairDiagnostics.length - 32);
await reportProgress({
metadata: {
repairPhase,
repairDiagnostics: [...repairDiagnostics],
databaseOnly: true,
worktreePreserved: true,
},
system: `Workspace repair ${phase}: ${status}.\n`,
});
};
try {
await reportRepairPhase("managed_stop", "started");
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
workspaceCwd,
});
await reportRepairPhase("managed_stop", "succeeded");
const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json");
const targetConfigPath = path.join(workspaceCwd, ".paperclip", "config.json");
let sourceConfigPath: string | null = null;
let previousAttemptId: string | null = null;
if (existsSync(manifestPath)) {
try {
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
attemptId?: unknown;
source?: { configPath?: unknown };
};
previousAttemptId = typeof manifest.attemptId === "string" ? manifest.attemptId : null;
sourceConfigPath = typeof manifest.source?.configPath === "string"
? path.resolve(manifest.source.configPath)
: null;
} catch {
throw new Error("Workspace seed manifest is malformed; repair source identity cannot be trusted.");
}
}
const baseWorkspaceCwd = projectWorkspace?.cwd ?? workspaceCwd;
const baseConfigPath = path.join(baseWorkspaceCwd, ".paperclip", "config.json");
if (!sourceConfigPath && existsSync(baseConfigPath) && path.resolve(baseConfigPath) !== path.resolve(targetConfigPath)) {
sourceConfigPath = baseConfigPath;
}
if (!sourceConfigPath || !existsSync(sourceConfigPath)) {
throw new Error("Workspace repair has no readable configured source instance.");
}
const cliRunner = path.join(baseWorkspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(baseWorkspaceCwd, "cli", "src", "index.ts");
const cliDist = path.join(baseWorkspaceCwd, "cli", "dist", "index.js");
const cliArgs = existsSync(cliRunner) && existsSync(cliEntry)
? [cliRunner, cliEntry]
: existsSync(cliDist)
? [cliDist]
: null;
if (!cliArgs) {
throw new Error("Workspace repair cannot find a runnable Paperclip CLI in the base workspace.");
}
await reportRepairPhase("target_backup", "started");
const child = spawn(process.execPath, [
...cliArgs,
"worktree",
"reseed",
"--from-config",
sourceConfigPath,
"--to",
workspaceCwd,
"--seed-mode",
"full",
"--yes",
"--backup-target",
], {
cwd: baseWorkspaceCwd,
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout?.on("data", (chunk: Buffer) => {
void onLog("stdout", chunk.toString("utf8"));
});
child.stderr?.on("data", (chunk: Buffer) => {
void onLog("stderr", chunk.toString("utf8"));
});
let repairCommandTimedOut = false;
let repairCommandForceTimeout: NodeJS.Timeout | null = null;
const repairCommandTimeout = setTimeout(() => {
repairCommandTimedOut = true;
child.kill("SIGTERM");
repairCommandForceTimeout = setTimeout(() => child.kill("SIGKILL"), 5_000);
repairCommandForceTimeout.unref?.();
}, 20 * 60_000);
repairCommandTimeout.unref?.();
let reseedObserved = false;
const manifestPoll = setInterval(() => {
if (reseedObserved || !existsSync(manifestPath)) return;
try {
const current = JSON.parse(readFileSync(manifestPath, "utf8")) as {
attemptId?: unknown;
state?: unknown;
};
if (
typeof current.attemptId === "string"
&& current.attemptId !== previousAttemptId
&& (current.state === "pending" || current.state === "running" || current.state === "failed")
) {
reseedObserved = true;
}
} catch {
// The atomic manifest writer makes this unlikely; the terminal
// read below remains authoritative.
}
}, 100);
manifestPoll.unref?.();
const exitCode = await new Promise<number>((resolve, reject) => {
child.once("error", reject);
child.once("exit", (code) => resolve(code ?? 1));
}).finally(() => {
clearInterval(manifestPoll);
clearTimeout(repairCommandTimeout);
if (repairCommandForceTimeout) clearTimeout(repairCommandForceTimeout);
});
if (reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
if (exitCode !== 0) {
let seedFailurePhase: string | null = null;
try {
const failedManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
phase?: unknown;
state?: unknown;
};
seedFailurePhase = failedManifest.state === "failed" && typeof failedManifest.phase === "string"
? failedManifest.phase
: null;
} catch {
// The outer repair phase remains exact when no seed phase was persisted.
}
await reportProgress({
metadata: { seedFailurePhase },
system: seedFailurePhase ? `Workspace seed failed during ${seedFailurePhase}.\n` : null,
});
throw new Error(
repairCommandTimedOut
? `Workspace database repair command timed out during ${repairPhase}.`
: `Workspace database repair command failed during ${repairPhase}.`,
);
}
if (!reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
if (!isVerifiedWorktreeSeedManifest(manifest)) {
throw new Error("Workspace reseed returned without a verified terminal manifest.");
}
const expectedInstanceId = deriveWorktreeInstanceId(workspaceCwd);
if (manifest.targetInstanceId !== expectedInstanceId) {
throw new Error("Verified seed manifest belongs to a different workspace instance.");
}
await reportRepairPhase("full_reseed", "succeeded");
await reportRepairPhase("managed_restart", "started");
const availableWorkspace = await ensureWorkspaceAvailable();
if (!availableWorkspace) {
throw new Error("Execution workspace needs a local path before Paperclip can restart it.");
}
const startedServices = await startRuntimeServicesForWorkspaceControl({
db,
actor: {
id: actor.agentId ?? null,
name: actor.actorType === "user" ? "Board" : "Agent",
companyId: existing.companyId,
},
issue: existing.sourceIssueId
? { id: existing.sourceIssueId, identifier: null, title: existing.name }
: null,
workspace: availableWorkspace,
executionWorkspaceId: existing.id,
config: {
workspaceRuntime: effectiveRuntimeConfig,
runtimeProvisionCommand:
existing.config?.runtimeProvisionCommand
?? projectPolicy?.workspaceStrategy?.runtimeProvisionCommand
?? null,
},
adapterEnv: {},
onLog,
recorder,
});
runtimeServiceCount = startedServices.length;
await reportRepairPhase("managed_restart", "succeeded");
await reportRepairPhase("readiness_validation", "started");
if (
startedServices.length === 0
|| startedServices.some((service) => service.status !== "running" || service.healthStatus !== "healthy")
) {
throw new Error("Managed restart did not produce healthy runtime services.");
}
await reportRepairPhase("readiness_validation", "succeeded");
} catch (error) {
await reportRepairPhase(repairPhase, "failed");
throw error;
}
} else if (action === "stop" || action === "restart") {
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
@@ -481,7 +717,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
throw error;
}
runtimeServiceCount = startedServices.length;
} else {
} else if (action !== "repair") {
runtimeServiceCount = selectedRuntimeServiceId ? Math.max(0, (existing.runtimeServices?.length ?? 1) - 1) : 0;
}
@@ -504,7 +740,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
config: { workspaceRuntime: effectiveRuntimeConfig },
currentDesiredState,
currentServiceStates: existing.config?.serviceStates ?? null,
action,
action: action === "repair" ? "start" : action,
serviceIndex: selectedServiceIndex,
});
const metadata = mergeExecutionWorkspaceConfig(existing.metadata as Record<string, unknown> | null, {
@@ -522,6 +758,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
? "Stopped execution workspace runtime services.\n"
: action === "restart"
? "Restarted execution workspace runtime services.\n"
: action === "repair"
? "Repaired the isolated workspace database and restarted healthy runtime services.\n"
: "Started execution workspace runtime services.\n",
metadata: {
runtimeServiceCount,
@@ -565,7 +803,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
// The operation is already terminal here. This also catches the recorder's own time
// budget expiring on a start that never settles, which is the one failure the inner
// handler cannot see — reconcile there too so no listener or desired state is left over.
if (action === "start" || action === "restart") await reconcileFailedRuntimeStart(error);
if (action === "start" || action === "restart" || action === "repair") {
await reconcileFailedRuntimeStart(error);
}
throw error;
}
},
+43 -16
View File
@@ -14,7 +14,7 @@ type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
* Managed runtime control actions. Every one of these mutates the runtime rows and
* local listeners of a single execution workspace, so only one may be live at a time.
*/
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "run"]);
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]);
/**
* Identity of this server process. A `running` runtime-control operation stamped with a
@@ -234,7 +234,10 @@ export interface WorkspaceOperationRecorder {
* timeout error is rethrown, so a hung provider can never leave an active operation.
*/
timeoutMs?: number | null;
run: () => Promise<{
run: (reportProgress: (input: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => Promise<void>) => Promise<{
status?: WorkspaceOperationStatus;
exitCode?: number | null;
stdout?: string | null;
@@ -484,7 +487,7 @@ export function workspaceOperationService(db: Db) {
// Managed runtime controls get an ownership stamp so bounded recovery can tell a
// slow-but-live operation from one abandoned by a dead request or server process.
const runtimeControlAction = readRuntimeControlAction(recordInput.metadata);
const insertedMetadata = runtimeControlAction
let currentMetadata = runtimeControlAction
? {
...(recordInput.metadata ?? {}),
runtimeControlOwner: {
@@ -515,7 +518,7 @@ export function workspaceOperationService(db: Db) {
logStore: handle.store,
logRef: handle.logRef,
metadata: redactCurrentUserValue(
insertedMetadata,
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
startedAt,
@@ -532,17 +535,21 @@ export function workspaceOperationService(db: Db) {
if (runtimeControlAction) {
heartbeatTimer = setInterval(() => {
const heartbeatAt = new Date();
currentMetadata = combineMetadata(currentMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
});
void db
.update(workspaceOperations)
.set({
metadata: combineMetadata(insertedMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
}),
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
updatedAt: heartbeatAt,
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")))
@@ -551,9 +558,29 @@ export function workspaceOperationService(db: Db) {
heartbeatTimer.unref?.();
}
const reportProgress = async (progress: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => {
await append("system", progress.system ?? null);
currentMetadata = combineMetadata(currentMetadata, progress.metadata);
await db
.update(workspaceOperations)
.set({
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
stdoutExcerpt: stdoutExcerpt || null,
stderrExcerpt: stderrExcerpt || null,
updatedAt: new Date(),
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")));
};
const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction);
const settle = async () => {
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run();
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress);
const timeout = new Promise<never>((_resolve, reject) => {
timeoutTimer = setTimeout(
() => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)),
@@ -561,7 +588,7 @@ export function workspaceOperationService(db: Db) {
);
timeoutTimer.unref?.();
});
return await Promise.race([recordInput.run(), timeout]);
return await Promise.race([recordInput.run(reportProgress), timeout]);
};
try {
@@ -583,7 +610,7 @@ export function workspaceOperationService(db: Db) {
logSha256: finalized.sha256,
logCompressed: finalized.compressed,
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, result.metadata),
combineMetadata(currentMetadata, result.metadata),
currentUserRedactionOptions,
) as Record<string, unknown> | null,
finishedAt,
@@ -613,7 +640,7 @@ export function workspaceOperationService(db: Db) {
...(runtimeControlAction
? {
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, {
combineMetadata(currentMetadata, {
failureReason: error instanceof WorkspaceOperationTimeoutError
? "runtime_control_timeout"
: "runtime_control_error",
@@ -28,7 +28,7 @@ const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([
]);
/** Runtime control actions that mutate the workspace and therefore need the lease. */
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart"];
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart", "repair"];
/**
* Upper bound on how long a lease survives without the owner touching it. Recovery
+7 -5
View File
@@ -36,6 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm";
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
import { conflict } from "../errors.js";
import { resolveHomeAwarePath } from "../home-paths.js";
import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
import {
createLocalServiceKey,
findLocalServiceRegistryRecordByRuntimeServiceId,
@@ -5015,6 +5016,7 @@ export function resolveRuntimeProvisionCommand(input: {
if (input.workspace.strategy !== "git_worktree") return "";
const stateDir = path.join(input.workspace.cwd, ".paperclip");
const manifestPath = path.join(stateDir, "seed-manifest.json");
const pendingMarker = path.join(stateDir, "seed-pending");
const completeMarker = path.join(stateDir, "seed-complete");
const provisionScript = path.join(
@@ -5022,11 +5024,11 @@ export function resolveRuntimeProvisionCommand(input: {
"scripts",
"provision-worktree-runtime.sh",
);
if (
!existsSync(pendingMarker)
|| existsSync(completeMarker)
|| !existsSync(provisionScript)
) {
let needsSeed = existsSync(pendingMarker) && !existsSync(completeMarker);
if (existsSync(manifestPath)) {
needsSeed = !hasVerifiedWorktreeSeedManifest(manifestPath);
}
if (!needsSeed || !existsSync(provisionScript)) {
return "";
}
+54
View File
@@ -0,0 +1,54 @@
import { existsSync, readFileSync } from "node:fs";
const WORKTREE_SEED_MODES = new Set(["minimal", "full"]);
type SeedDiagnostic = {
phase?: unknown;
status?: unknown;
at?: unknown;
};
/**
* A manifest is readiness evidence only when the complete validation phase was
* durably recorded. A bare `state: verified` is deliberately insufficient so
* truncated or hand-edited files cannot start a partially restored workspace.
*/
export function isVerifiedWorktreeSeedManifest(value: unknown): boolean {
if (!value || typeof value !== "object") return false;
const manifest = value as Record<string, unknown>;
const source = manifest.source as Record<string, unknown> | null;
const diagnostics = manifest.diagnostics as SeedDiagnostic[] | null;
return manifest.version === 2
&& manifest.state === "verified"
&& manifest.phase === "complete"
&& typeof source?.instanceId === "string"
&& source.instanceId.length > 0
&& typeof source.configPath === "string"
&& source.configPath.length > 0
&& WORKTREE_SEED_MODES.has(String(manifest.seedMode ?? ""))
&& typeof manifest.snapshotAt === "string"
&& manifest.snapshotAt.length > 0
&& typeof manifest.migrationRevision === "string"
&& manifest.migrationRevision.length > 0
&& typeof manifest.targetInstanceId === "string"
&& manifest.targetInstanceId.length > 0
&& typeof manifest.attemptId === "string"
&& manifest.attemptId.length > 0
&& typeof manifest.startedAt === "string"
&& typeof manifest.finishedAt === "string"
&& Array.isArray(diagnostics)
&& diagnostics.some((diagnostic) => (
diagnostic?.phase === "complete"
&& diagnostic.status === "succeeded"
&& typeof diagnostic.at === "string"
));
}
export function hasVerifiedWorktreeSeedManifest(manifestPath: string): boolean {
if (!existsSync(manifestPath)) return false;
try {
return isVerifiedWorktreeSeedManifest(JSON.parse(readFileSync(manifestPath, "utf8")));
} catch {
return false;
}
}