feat(workspaces): verify seeds and add managed repair

Replace binary readiness markers with an evidence-bearing seed manifest and enforce a shared validation contract for minimal and full clones. Add an exclusive audited database repair path with recoverable backup, bounded phase diagnostics, and healthy restart validation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-18 23:20:22 +00:00
1 parent 4b968d8c05
commit 53a0aa3020
21 files changed
+1568 -154

No files matched your search

+195 -41
View File
@@ -8,13 +8,16 @@ import { eq } from "drizzle-orm";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
agents,
authAccounts,
authUsers,
companies,
companyMemberships,
createDb,
executionWorkspaces,
issueComments,
issues,
projectWorkspaces,
instanceUserRoles,
projects,
routines,
routineTriggers,
@@ -27,6 +30,7 @@ import {
markWorktreeSeedPending,
pauseSeededScheduledRoutines,
quarantineSeededWorktreeExecutionState,
readWorktreeSeedManifest,
readSourceAttachmentBody,
rebindWorkspaceCwd,
resolveSourceConfigPath,
@@ -64,6 +68,88 @@ const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
const itEmbeddedPostgres = embeddedPostgresSupport.supported ? it : it.skip;
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
function mockVerifiedSeedResult() {
return {
backupSummary: "snapshot.sql",
snapshotAt: "2026-08-18T00:00:00.000Z",
migrationRevision: "0142_test.sql",
pausedScheduledRoutines: 0,
executionQuarantine: {
disabledTimerHeartbeats: 0,
resetRunningAgents: 0,
quarantinedInProgressIssues: 0,
unassignedTodoIssues: 0,
unassignedReviewIssues: 0,
stoppedProjectWorkspaceRuntimes: 0,
stoppedExecutionWorkspaceRuntimes: 0,
stoppedRuntimeServices: 0,
},
reboundWorkspaces: [],
validation: {
authUserCount: 1,
credentialAccountCount: 1,
instanceAdminCount: 1,
activeMembershipCount: 1,
companyCount: 1,
issueCount: 1,
representativeCompanyId: "00000000-0000-4000-8000-000000000001",
representativeIssueId: "00000000-0000-4000-8000-000000000002",
migrationRevision: "0142_test.sql",
},
};
}
async function seedValidWorktreeSource(connectionString: string) {
const db = createDb(connectionString);
const companyId = randomUUID();
const issueId = randomUUID();
const now = new Date();
await db.insert(authUsers).values({
id: "user-existing",
email: "existing@paperclip.ing",
name: "Existing User",
emailVerified: true,
createdAt: now,
updatedAt: now,
});
await db.insert(authAccounts).values({
id: "credential-existing",
accountId: "existing@paperclip.ing",
providerId: "credential",
userId: "user-existing",
password: "fixture-password-hash",
createdAt: now,
updatedAt: now,
});
await db.insert(instanceUserRoles).values({
userId: "user-existing",
role: "instance_admin",
});
await db.insert(companies).values({
id: companyId,
name: "Seed Source",
issuePrefix: "SEED",
requireBoardApprovalForNewAgents: false,
});
await db.insert(companyMemberships).values({
companyId,
principalType: "user",
principalId: "user-existing",
status: "active",
});
await db.insert(issues).values({
id: issueId,
companyId,
title: "Representative seed issue",
status: "backlog",
priority: "medium",
issueNumber: 1,
identifier: "SEED-1",
});
await db.$client.end({ timeout: 5 });
return { companyId, issueId };
}
if (!embeddedPostgresSupport.supported) {
console.warn(
`Skipping embedded Postgres worktree CLI tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
@@ -400,7 +486,7 @@ describe("worktree helpers", () => {
expect(full.nullifyColumns).toEqual({});
});
it("ensure-seeded seeds once and fast-exits on the seed-complete marker", async () => {
it("ensure-seeded seeds once and fast-exits on the verified manifest", async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-"));
try {
const sourceConfigPath = path.join(tempRoot, "source", "config.json");
@@ -429,7 +515,7 @@ describe("worktree helpers", () => {
markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath });
const seedDatabase = vi.fn().mockResolvedValue({
backupSummary: "snapshot.sql",
...mockVerifiedSeedResult(),
pausedScheduledRoutines: 2,
executionQuarantine: {
disabledTimerHeartbeats: 1,
@@ -441,7 +527,6 @@ describe("worktree helpers", () => {
stoppedExecutionWorkspaceRuntimes: 0,
stoppedRuntimeServices: 0,
},
reboundWorkspaces: [],
});
await expect(
@@ -449,7 +534,7 @@ describe("worktree helpers", () => {
).resolves.toMatchObject({ seeded: true, reason: "seeded" });
await expect(
ensureWorktreeSeeded({ config: targetConfigPath }, { seedDatabase }),
).resolves.toEqual({ seeded: false, reason: "complete_marker" });
).resolves.toEqual({ seeded: false, reason: "verified_manifest" });
expect(seedDatabase).toHaveBeenCalledTimes(1);
expect(seedDatabase).toHaveBeenCalledWith(expect.objectContaining({
@@ -458,7 +543,14 @@ describe("worktree helpers", () => {
instanceId: "ensure-seeded-test",
}));
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false);
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(true);
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false);
expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({
version: 2,
state: "verified",
phase: "complete",
migrationRevision: "0142_test.sql",
targetInstanceId: "ensure-seeded-test",
});
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true });
}
@@ -499,7 +591,11 @@ describe("worktree helpers", () => {
),
).rejects.toThrow("seed failed");
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(true);
expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({
state: "failed",
phase: "pending",
});
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false);
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false);
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false);
} finally {
@@ -537,21 +633,7 @@ describe("worktree helpers", () => {
const seedDatabase = vi.fn(async () => {
await new Promise((resolve) => setTimeout(resolve, 100));
return {
backupSummary: "snapshot.sql",
pausedScheduledRoutines: 0,
executionQuarantine: {
disabledTimerHeartbeats: 0,
resetRunningAgents: 0,
quarantinedInProgressIssues: 0,
unassignedTodoIssues: 0,
unassignedReviewIssues: 0,
stoppedProjectWorkspaceRuntimes: 0,
stoppedExecutionWorkspaceRuntimes: 0,
stoppedRuntimeServices: 0,
},
reboundWorkspaces: [],
};
return mockVerifiedSeedResult();
});
const results = await Promise.all([
@@ -561,7 +643,7 @@ describe("worktree helpers", () => {
expect(results).toEqual(expect.arrayContaining([
expect.objectContaining({ seeded: true, reason: "seeded" }),
{ seeded: false, reason: "complete_marker" },
{ seeded: false, reason: "verified_manifest" },
]));
expect(seedDatabase).toHaveBeenCalledTimes(1);
expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false);
@@ -570,6 +652,58 @@ describe("worktree helpers", () => {
}
});
it("records an interrupted phase before retrying to a verified terminal state", async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-interrupted-seed-"));
try {
const sourceConfigPath = path.join(tempRoot, "source", "config.json");
const targetRoot = path.join(tempRoot, "worktree");
const targetConfigPath = path.join(targetRoot, ".paperclip", "config.json");
const targetPaths = resolveWorktreeLocalPaths({
cwd: targetRoot,
homeDir: path.join(tempRoot, "worktree-home"),
instanceId: "interrupted-seed",
});
const sourceConfig = buildSourceConfig();
const targetConfig = buildWorktreeConfig({
sourceConfig,
paths: targetPaths,
serverPort: 3196,
databasePort: 54996,
});
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
fs.mkdirSync(path.dirname(targetConfigPath), { recursive: true });
fs.writeFileSync(sourceConfigPath, `${JSON.stringify(sourceConfig)}\n`);
fs.writeFileSync(targetConfigPath, `${JSON.stringify(targetConfig)}\n`);
fs.writeFileSync(
path.join(targetRoot, ".paperclip", ".env"),
`PAPERCLIP_HOME=${targetPaths.homeDir}\nPAPERCLIP_INSTANCE_ID=${targetPaths.instanceId}\n`,
);
markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath });
const interrupted = readWorktreeSeedManifest(targetConfigPath)!;
fs.writeFileSync(
path.join(targetRoot, ".paperclip", "seed-manifest.json"),
`${JSON.stringify({ ...interrupted, state: "running", phase: "restore" }, null, 2)}\n`,
);
await expect(ensureWorktreeSeeded(
{ config: targetConfigPath },
{ seedDatabase: vi.fn().mockResolvedValue(mockVerifiedSeedResult()) },
)).resolves.toMatchObject({ seeded: true, reason: "seeded" });
const verified = readWorktreeSeedManifest(targetConfigPath)!;
expect(verified.state).toBe("verified");
expect(verified.diagnostics).toEqual(expect.arrayContaining([
expect.objectContaining({
phase: "restore",
status: "failed",
message: "The previous seed attempt ended without a terminal result.",
}),
]));
} finally {
fs.rmSync(tempRoot, { recursive: true, force: true });
}
});
it("fails closed instead of racing to reclaim a stale seed lock", async () => {
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-stale-lock-"));
try {
@@ -983,15 +1117,7 @@ describe("worktree helpers", () => {
const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-auth-source-");
try {
const sourceDbClient = createDb(sourceDb.connectionString);
await sourceDbClient.insert(authUsers).values({
id: "user-existing",
email: "existing@paperclip.ing",
name: "Existing User",
emailVerified: true,
createdAt: new Date(),
updatedAt: new Date(),
});
await seedValidWorktreeSource(sourceDb.connectionString);
fs.mkdirSync(path.dirname(sourceKeyPath), { recursive: true });
fs.mkdirSync(worktreeRoot, { recursive: true });
@@ -1028,6 +1154,18 @@ describe("worktree helpers", () => {
const targetConfig = JSON.parse(
fs.readFileSync(path.join(worktreeRoot, ".paperclip", "config.json"), "utf8"),
) as PaperclipConfig;
const manifestText = fs.readFileSync(
path.join(worktreeRoot, ".paperclip", "seed-manifest.json"),
"utf8",
);
expect(JSON.parse(manifestText)).toMatchObject({
version: 2,
seedMode: "minimal",
state: "verified",
phase: "complete",
});
expect(manifestText).not.toContain("fixture-password-hash");
expect(manifestText).not.toContain("source-master-key");
const { default: EmbeddedPostgres } = await import("embedded-postgres");
const targetPg = new EmbeddedPostgres({
databaseDir: targetConfig.database.embeddedPostgresDataDir,
@@ -1284,9 +1422,8 @@ describe("worktree helpers", () => {
const originalCwd = process.cwd();
const originalPaperclipConfig = process.env.PAPERCLIP_CONFIG;
const currentDatabaseReservation = await reserveTestPort();
const sourceDatabaseReservation = await reserveTestPort();
const currentDatabasePort = currentDatabaseReservation.port;
const sourceDatabasePort = sourceDatabaseReservation.port;
const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-reseed-source-");
try {
fs.mkdirSync(path.dirname(currentPaths.configPath), { recursive: true });
@@ -1301,15 +1438,28 @@ describe("worktree helpers", () => {
serverPort: 3114,
databasePort: currentDatabasePort,
});
const sourceConfig = buildWorktreeConfig({
sourceConfig: buildSourceConfig(),
paths: sourcePaths,
serverPort: 3200,
databasePort: sourceDatabasePort,
});
const sourceConfig = buildSourceConfig();
sourceConfig.database = {
mode: "postgres",
embeddedPostgresDataDir: sourcePaths.embeddedPostgresDataDir,
embeddedPostgresPort: 54329,
backup: {
enabled: true,
intervalMinutes: 60,
retentionDays: 30,
dir: sourcePaths.backupDir,
},
connectionString: sourceDb.connectionString,
};
sourceConfig.logging.logDir = sourcePaths.logDir;
sourceConfig.storage.localDisk.baseDir = sourcePaths.storageDir;
sourceConfig.secrets.localEncrypted.keyFilePath = sourcePaths.secretsKeyFilePath;
await seedValidWorktreeSource(sourceDb.connectionString);
fs.writeFileSync(currentPaths.configPath, JSON.stringify(currentConfig, null, 2), "utf8");
fs.writeFileSync(sourcePaths.configPath, JSON.stringify(sourceConfig, null, 2), "utf8");
fs.writeFileSync(sourcePaths.secretsKeyFilePath, "source-secret", "utf8");
const worktreeSentinelPath = path.join(repoRoot, "user-worktree-file.txt");
fs.writeFileSync(worktreeSentinelPath, "preserve me", "utf8");
fs.writeFileSync(
currentPaths.envPath,
[
@@ -1325,11 +1475,11 @@ describe("worktree helpers", () => {
process.chdir(repoRoot);
await currentDatabaseReservation.release();
await sourceDatabaseReservation.release();
await worktreeReseedCommand({
fromConfig: sourcePaths.configPath,
yes: true,
backupTarget: true,
});
const rewrittenConfig = JSON.parse(fs.readFileSync(currentPaths.configPath, "utf8"));
@@ -1341,9 +1491,13 @@ describe("worktree helpers", () => {
expect(rewrittenEnv).toContain(`PAPERCLIP_INSTANCE_ID=${currentInstanceId}`);
expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_NAME=existing-name");
expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_COLOR=\"#112233\"");
expect(fs.readFileSync(worktreeSentinelPath, "utf8")).toBe("preserve me");
expect(
fs.readdirSync(path.join(currentPaths.backupDir, "repair")).some((name) => name.endsWith(".sql.gz")),
).toBe(true);
} finally {
await currentDatabaseReservation.release();
await sourceDatabaseReservation.release();
await sourceDb.cleanup();
process.chdir(originalCwd);
if (originalPaperclipConfig === undefined) {
delete process.env.PAPERCLIP_CONFIG;
+42
View File
@@ -5,12 +5,52 @@ import { expandHomePrefix } from "../config/home.js";
export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees";
export const WORKTREE_SEED_MODES = ["minimal", "full"] as const;
export const WORKTREE_SEED_MANIFEST = "seed-manifest.json";
export const WORKTREE_SEED_PENDING_MARKER = "seed-pending";
export const WORKTREE_SEED_COMPLETE_MARKER = "seed-complete";
export const WORKTREE_SEED_LOCK_MARKER = "seed.lock";
export type WorktreeSeedMode = (typeof WORKTREE_SEED_MODES)[number];
export const WORKTREE_SEED_PHASES = [
"pending",
"source_validation",
"snapshot",
"restore",
"migrations",
"execution_quarantine",
"routine_pause",
"workspace_rebind",
"post_restore_validation",
"complete",
] as const;
export type WorktreeSeedPhase = (typeof WORKTREE_SEED_PHASES)[number];
export type WorktreeSeedState = "pending" | "running" | "verified" | "failed";
export type WorktreeSeedManifest = {
version: 2;
source: {
instanceId: string;
configPath: string;
};
snapshotAt: string | null;
seedMode: WorktreeSeedMode;
migrationRevision: string | null;
targetInstanceId: string;
phase: WorktreeSeedPhase;
state: WorktreeSeedState;
attemptId: string;
startedAt: string | null;
finishedAt: string | null;
diagnostics: Array<{
phase: WorktreeSeedPhase;
status: "started" | "succeeded" | "failed";
at: string;
message?: string;
}>;
};
export type WorktreeSeedPlan = {
mode: WorktreeSeedMode;
excludedTables: string[];
@@ -54,6 +94,7 @@ export type WorktreeUiBranding = {
};
export type WorktreeSeedMarkerPaths = {
manifest: string;
pending: string;
complete: string;
lock: string;
@@ -62,6 +103,7 @@ export type WorktreeSeedMarkerPaths = {
export function resolveWorktreeSeedMarkerPaths(configPath: string): WorktreeSeedMarkerPaths {
const configDir = path.dirname(path.resolve(configPath));
return {
manifest: path.resolve(configDir, WORKTREE_SEED_MANIFEST),
pending: path.resolve(configDir, WORKTREE_SEED_PENDING_MARKER),
complete: path.resolve(configDir, WORKTREE_SEED_COMPLETE_MARKER),
lock: path.resolve(configDir, WORKTREE_SEED_LOCK_MARKER),
+551 -52
View File
@@ -7,6 +7,7 @@ import {
readdirSync,
readFileSync,
readlinkSync,
renameSync,
rmSync,
statSync,
symlinkSync,
@@ -24,8 +25,11 @@ import { and, eq, inArray, sql } from "drizzle-orm";
import {
applyPendingMigrations,
agents,
authAccounts,
authUsers,
assets,
companies,
companyMemberships,
createDb,
documentRevisions,
documents,
@@ -39,6 +43,7 @@ import {
issueComments,
issueDocuments,
issues,
instanceUserRoles,
projectWorkspaces,
projects,
routines,
@@ -65,13 +70,16 @@ import {
formatShellExports,
generateWorktreeColor,
isWorktreeSeedMode,
WORKTREE_SEED_PHASES,
resolveSuggestedWorktreeName,
resolveWorktreeSeedPlan,
resolveWorktreeSeedMarkerPaths,
resolveWorktreeLocalPaths,
sanitizeWorktreeInstanceId,
type WorktreeSeedPlan,
type WorktreeSeedManifest,
type WorktreeSeedMode,
type WorktreeSeedPhase,
type WorktreeLocalPaths,
} from "./worktree-lib.js";
import {
@@ -137,6 +145,7 @@ type WorktreeReseedOptions = {
preserveLiveWork?: boolean;
yes?: boolean;
allowLiveTarget?: boolean;
backupTarget?: boolean;
};
type WorktreeRepairOptions = {
@@ -197,6 +206,8 @@ type CopiedGitHooksResult = {
type SeedWorktreeDatabaseResult = {
backupSummary: string;
snapshotAt: string;
migrationRevision: string;
pausedScheduledRoutines: number;
executionQuarantine: SeededWorktreeExecutionQuarantineSummary;
reboundWorkspaces: Array<{
@@ -204,28 +215,26 @@ type SeedWorktreeDatabaseResult = {
fromCwd: string;
toCwd: string;
}>;
validation: WorktreeSeedValidationSummary;
};
type WorktreeSeedPendingMarker = {
version: 1;
state: "pending";
sourceConfigPath: string;
seedMode: "minimal";
createdAt: string;
};
type WorktreeSeedCompleteMarker = {
version: 1;
state: "complete";
seedMode: WorktreeSeedMode;
completedAt: string;
export type WorktreeSeedValidationSummary = {
authUserCount: number;
credentialAccountCount: number;
instanceAdminCount: number;
activeMembershipCount: number;
companyCount: number;
issueCount: number;
representativeCompanyId: string;
representativeIssueId: string;
migrationRevision: string;
};
type SeedWorktreeDatabase = typeof seedWorktreeDatabase;
export type EnsureWorktreeSeededResult = {
seeded: boolean;
reason: "seeded" | "complete_marker" | "legacy_unmarked";
reason: "seeded" | "verified_manifest" | "complete_marker" | "legacy_unmarked";
details?: SeedWorktreeDatabaseResult;
};
@@ -1393,6 +1402,140 @@ export async function quarantineSeededWorktreeExecutionState(
}
}
type WorktreeSeedValidationExpectation = {
adminUserId: string;
representativeCompanyId: string;
representativeIssueId: string;
};
async function inspectVerifiedSeedDatabase(
connectionString: string,
expected?: WorktreeSeedValidationExpectation,
): Promise<{ summary: WorktreeSeedValidationSummary; expectation: WorktreeSeedValidationExpectation }> {
const migrationState = await inspectMigrations(connectionString);
if (migrationState.status !== "upToDate") {
throw new Error(
`Migration journal is not current (${migrationState.pendingMigrations.length} pending migration(s)).`,
);
}
const migrationRevision = migrationState.appliedMigrations.at(-1);
if (!migrationRevision) {
throw new Error("Migration journal has no applied revision.");
}
const db = createDb(connectionString);
try {
const [counts] = await db
.select({
authUserCount: sql<number>`count(distinct ${authUsers.id})::int`,
credentialAccountCount: sql<number>`count(distinct ${authAccounts.id})::int`,
instanceAdminCount: sql<number>`count(distinct ${instanceUserRoles.userId})::int`,
activeMembershipCount: sql<number>`count(distinct ${companyMemberships.id})::int`,
companyCount: sql<number>`count(distinct ${companies.id})::int`,
issueCount: sql<number>`count(distinct ${issues.id})::int`,
})
.from(authUsers)
.leftJoin(authAccounts, eq(authAccounts.userId, authUsers.id))
.leftJoin(
instanceUserRoles,
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
)
.leftJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.status, "active"),
),
)
.leftJoin(companies, eq(companies.id, companyMemberships.companyId))
.leftJoin(issues, eq(issues.companyId, companies.id));
const admin = await db
.select({ userId: authUsers.id })
.from(authUsers)
.innerJoin(
instanceUserRoles,
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
)
.innerJoin(
authAccounts,
and(
eq(authAccounts.userId, authUsers.id),
sql`length(trim(${authAccounts.providerId})) > 0`,
sql`length(trim(${authAccounts.accountId})) > 0`,
),
)
.innerJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.status, "active"),
),
)
.where(expected ? eq(authUsers.id, expected.adminUserId) : undefined)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!admin) {
throw new Error(
"No auth user has a non-empty credential account, instance-admin role, and active company membership.",
);
}
const representative = await db
.select({ companyId: companies.id, issueId: issues.id })
.from(companies)
.innerJoin(issues, eq(issues.companyId, companies.id))
.where(
expected
? and(
eq(companies.id, expected.representativeCompanyId),
eq(issues.id, expected.representativeIssueId),
)
: undefined,
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!representative) {
throw new Error("No representative cloned company and issue pair is readable.");
}
const summary: WorktreeSeedValidationSummary = {
authUserCount: counts?.authUserCount ?? 0,
credentialAccountCount: counts?.credentialAccountCount ?? 0,
instanceAdminCount: counts?.instanceAdminCount ?? 0,
activeMembershipCount: counts?.activeMembershipCount ?? 0,
companyCount: counts?.companyCount ?? 0,
issueCount: counts?.issueCount ?? 0,
representativeCompanyId: representative.companyId,
representativeIssueId: representative.issueId,
migrationRevision,
};
if (
summary.authUserCount < 1
|| summary.credentialAccountCount < 1
|| summary.instanceAdminCount < 1
|| summary.activeMembershipCount < 1
|| summary.companyCount < 1
|| summary.issueCount < 1
) {
throw new Error("Seed validation found an incomplete auth, membership, company, or issue shape.");
}
return {
summary,
expectation: {
adminUserId: admin.userId,
representativeCompanyId: representative.companyId,
representativeIssueId: representative.issueId,
},
};
} finally {
await db.$client?.end?.({ timeout: 5 }).catch(() => undefined);
}
}
async function seedWorktreeDatabase(input: {
sourceConfigPath: string;
sourceConfig: PaperclipConfig;
@@ -1401,6 +1544,7 @@ async function seedWorktreeDatabase(input: {
instanceId: string;
seedMode: WorktreeSeedMode;
preserveLiveWork?: boolean;
onPhase?: (phase: WorktreeSeedPhase, status: "started" | "succeeded", message?: string) => void;
}): Promise<SeedWorktreeDatabaseResult> {
const seedPlan = resolveWorktreeSeedPlan(input.seedMode);
const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath);
@@ -1428,6 +1572,16 @@ async function seedWorktreeDatabase(input: {
sourceEnvEntries,
sourceHandle?.port,
);
input.onPhase?.("source_validation", "started");
const sourceValidation = await inspectVerifiedSeedDatabase(sourceConnectionString);
input.onPhase?.(
"source_validation",
"succeeded",
`Validated ${sourceValidation.summary.companyCount} company record(s) and ${sourceValidation.summary.issueCount} issue record(s).`,
);
const snapshotAt = new Date().toISOString();
input.onPhase?.("snapshot", "started");
const backup = await runDatabaseBackup({
connectionString: sourceConnectionString,
backupDir: path.resolve(input.targetPaths.backupDir, "seed"),
@@ -1438,6 +1592,7 @@ async function seedWorktreeDatabase(input: {
excludeTables: seedPlan.excludedTables,
nullifyColumns: seedPlan.nullifyColumns,
});
input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`);
targetHandle = await ensureEmbeddedPostgres(
input.targetConfig.database.embeddedPostgresDataDir,
@@ -1445,27 +1600,56 @@ async function seedWorktreeDatabase(input: {
);
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
input.onPhase?.("restore", "started");
await resetPostgresDatabase(adminConnectionString, "paperclip");
const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`;
await runDatabaseRestore({
connectionString: targetConnectionString,
backupFile: backup.backupFile,
});
input.onPhase?.("restore", "succeeded");
input.onPhase?.("migrations", "started");
await applyPendingMigrations(targetConnectionString);
input.onPhase?.("migrations", "succeeded");
input.onPhase?.("execution_quarantine", "started");
const executionQuarantine = input.preserveLiveWork
? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }
: await quarantineSeededWorktreeExecutionState(targetConnectionString);
input.onPhase?.(
"execution_quarantine",
"succeeded",
input.preserveLiveWork
? "Preserved copied live work by explicit request."
: formatSeededWorktreeExecutionQuarantineSummary(executionQuarantine),
);
input.onPhase?.("routine_pause", "started");
const pausedScheduledRoutines = await pauseSeededScheduledRoutines(targetConnectionString);
input.onPhase?.("routine_pause", "succeeded", `Paused ${pausedScheduledRoutines} scheduled routine(s).`);
input.onPhase?.("workspace_rebind", "started");
const reboundWorkspaces = await rebindSeededProjectWorkspaces({
targetConnectionString,
currentCwd: input.targetPaths.cwd,
});
input.onPhase?.("workspace_rebind", "succeeded", `Rebound ${reboundWorkspaces.length} workspace path(s).`);
input.onPhase?.("post_restore_validation", "started");
const targetValidation = await inspectVerifiedSeedDatabase(
targetConnectionString,
sourceValidation.expectation,
);
input.onPhase?.(
"post_restore_validation",
"succeeded",
`Validated migration ${targetValidation.summary.migrationRevision}.`,
);
return {
backupSummary: formatDatabaseBackupResult(backup),
snapshotAt,
migrationRevision: targetValidation.summary.migrationRevision,
pausedScheduledRoutines,
executionQuarantine,
reboundWorkspaces,
validation: targetValidation.summary,
};
} finally {
if (targetHandle?.startedByThisProcess) {
@@ -1477,46 +1661,184 @@ async function seedWorktreeDatabase(input: {
}
}
function writeWorktreeSeedMarker(
filePath: string,
marker: WorktreeSeedPendingMarker | WorktreeSeedCompleteMarker,
): void {
const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32;
const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512;
const activeSeedInterruptHandlers = new Map<string, (signal: NodeJS.Signals) => void>();
function dispatchSeedInterruption(signal: NodeJS.Signals): void {
for (const handler of activeSeedInterruptHandlers.values()) {
try {
handler(signal);
} catch {
// Continue terminalizing the other active manifests before exiting.
}
}
process.exit(signal === "SIGINT" ? 130 : 143);
}
const dispatchSeedSigint = () => dispatchSeedInterruption("SIGINT");
const dispatchSeedSigterm = () => dispatchSeedInterruption("SIGTERM");
function registerSeedInterruptHandler(handler: (signal: NodeJS.Signals) => void): () => void {
const id = randomUUID();
if (activeSeedInterruptHandlers.size === 0) {
process.once("SIGINT", dispatchSeedSigint);
process.once("SIGTERM", dispatchSeedSigterm);
}
activeSeedInterruptHandlers.set(id, handler);
return () => {
activeSeedInterruptHandlers.delete(id);
if (activeSeedInterruptHandlers.size === 0) {
process.off("SIGINT", dispatchSeedSigint);
process.off("SIGTERM", dispatchSeedSigterm);
}
};
}
type LegacyWorktreeSeedPendingMarker = {
version: 1;
state: "pending";
sourceConfigPath: string;
};
function resolveSeedInstanceId(configPath: string): string {
const envEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(configPath));
return nonEmpty(envEntries.PAPERCLIP_INSTANCE_ID)
?? sanitizeWorktreeInstanceId(path.basename(path.dirname(path.resolve(configPath))));
}
function writeWorktreeSeedManifest(filePath: string, manifest: WorktreeSeedManifest): void {
mkdirSync(path.dirname(filePath), { recursive: true });
writeFileSync(filePath, `${JSON.stringify(marker, null, 2)}\n`, { mode: 0o600 });
const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
renameSync(temporaryPath, filePath);
}
export function readWorktreeSeedManifest(configPath: string): WorktreeSeedManifest | null {
const manifestPath = resolveWorktreeSeedMarkerPaths(configPath).manifest;
if (!existsSync(manifestPath)) return null;
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
} catch (error) {
throw new Error(
`Invalid worktree seed manifest at ${manifestPath}: ${error instanceof Error ? error.message : String(error)}`,
);
}
const value = parsed as Partial<WorktreeSeedManifest>;
const diagnosticsValid = Array.isArray(value.diagnostics) && value.diagnostics.every((diagnostic) => (
diagnostic
&& typeof diagnostic === "object"
&& WORKTREE_SEED_PHASES.includes(diagnostic.phase)
&& ["started", "succeeded", "failed"].includes(diagnostic.status)
&& typeof diagnostic.at === "string"
&& (diagnostic.message === undefined || typeof diagnostic.message === "string")
));
const verifiedTerminalValid = value.state !== "verified" || (
value.phase === "complete"
&& typeof value.snapshotAt === "string"
&& value.snapshotAt.length > 0
&& typeof value.migrationRevision === "string"
&& value.migrationRevision.length > 0
&& typeof value.startedAt === "string"
&& typeof value.finishedAt === "string"
&& value.diagnostics?.some((diagnostic) => (
diagnostic.phase === "complete" && diagnostic.status === "succeeded"
)) === true
);
if (
!value
|| typeof value !== "object"
|| value.version !== 2
|| !value.source
|| typeof value.source.instanceId !== "string"
|| typeof value.source.configPath !== "string"
|| typeof value.targetInstanceId !== "string"
|| value.targetInstanceId.length === 0
|| !isWorktreeSeedMode(String(value.seedMode ?? ""))
|| !WORKTREE_SEED_PHASES.includes(value.phase as WorktreeSeedPhase)
|| !["pending", "running", "verified", "failed"].includes(String(value.state ?? ""))
|| typeof value.attemptId !== "string"
|| value.attemptId.length === 0
|| !diagnosticsValid
|| !verifiedTerminalValid
) {
throw new Error(`Invalid worktree seed manifest at ${manifestPath}.`);
}
return value as WorktreeSeedManifest;
}
export function markWorktreeSeedPending(input: {
configPath: string;
sourceConfigPath: string;
now?: Date;
}): void {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
rmSync(markers.complete, { force: true });
writeWorktreeSeedMarker(markers.pending, {
version: 1,
state: "pending",
sourceConfigPath: path.resolve(input.sourceConfigPath),
seedMode: "minimal",
createdAt: (input.now ?? new Date()).toISOString(),
});
}
export function markWorktreeSeedComplete(input: {
configPath: string;
targetInstanceId?: string;
seedMode?: WorktreeSeedMode;
now?: Date;
}): void {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
writeWorktreeSeedMarker(markers.complete, {
version: 1,
state: "complete",
const at = (input.now ?? new Date()).toISOString();
writeWorktreeSeedManifest(markers.manifest, {
version: 2,
source: {
instanceId: resolveSeedInstanceId(input.sourceConfigPath),
configPath: path.resolve(input.sourceConfigPath),
},
snapshotAt: null,
seedMode: input.seedMode ?? "minimal",
completedAt: (input.now ?? new Date()).toISOString(),
migrationRevision: null,
targetInstanceId: input.targetInstanceId ?? resolveSeedInstanceId(input.configPath),
phase: "pending",
state: "pending",
attemptId: randomUUID(),
startedAt: null,
finishedAt: null,
diagnostics: [{ phase: "pending", status: "succeeded", at }],
});
// New manifests are authoritative. Legacy files are removed so no caller can
// mistake a stale binary marker for current verified seed state.
rmSync(markers.complete, { force: true });
rmSync(markers.pending, { force: true });
}
function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMarker {
function updateWorktreeSeedManifest(input: {
configPath: string;
phase: WorktreeSeedPhase;
status: "started" | "succeeded" | "failed";
state?: WorktreeSeedManifest["state"];
message?: string;
snapshotAt?: string | null;
migrationRevision?: string | null;
now?: Date;
}): WorktreeSeedManifest {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
const current = readWorktreeSeedManifest(input.configPath);
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
const at = (input.now ?? new Date()).toISOString();
const nextState = input.state ?? current.state;
const diagnostic = {
phase: input.phase,
status: input.status,
at,
...(input.message
? { message: input.message.slice(0, WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT) }
: {}),
};
const next: WorktreeSeedManifest = {
...current,
phase: input.phase,
state: nextState,
snapshotAt: input.snapshotAt === undefined ? current.snapshotAt : input.snapshotAt,
migrationRevision:
input.migrationRevision === undefined ? current.migrationRevision : input.migrationRevision,
startedAt: current.startedAt ?? (input.status === "started" ? at : null),
finishedAt: nextState === "verified" || nextState === "failed" ? at : null,
diagnostics: [...current.diagnostics, diagnostic].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
};
writeWorktreeSeedManifest(markers.manifest, next);
return next;
}
function readLegacyWorktreeSeedPendingMarker(filePath: string): LegacyWorktreeSeedPendingMarker {
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(filePath, "utf8"));
@@ -1537,7 +1859,7 @@ function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMar
throw new Error(`Invalid worktree seed-pending marker at ${filePath}.`);
}
return parsed as WorktreeSeedPendingMarker;
return parsed as LegacyWorktreeSeedPendingMarker;
}
const WORKTREE_SEED_LOCK_POLL_MS = 50;
@@ -1631,6 +1953,117 @@ async function acquireWorktreeSeedLock(lockPath: string): Promise<() => Promise<
}
}
function startWorktreeSeedAttempt(configPath: string, now = new Date()): WorktreeSeedManifest {
const markers = resolveWorktreeSeedMarkerPaths(configPath);
const current = readWorktreeSeedManifest(configPath);
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
const at = now.toISOString();
const next: WorktreeSeedManifest = {
...current,
state: "running",
phase: "pending",
attemptId: randomUUID(),
snapshotAt: null,
migrationRevision: null,
startedAt: at,
finishedAt: null,
diagnostics: [
...current.diagnostics,
{ phase: "pending" as const, status: "started" as const, at },
].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
};
writeWorktreeSeedManifest(markers.manifest, next);
return next;
}
async function runVerifiedWorktreeSeed(input: {
configPath: string;
sourceConfigPath: string;
sourceConfig: PaperclipConfig;
targetConfig: PaperclipConfig;
targetPaths: WorktreeLocalPaths;
instanceId: string;
seedMode: WorktreeSeedMode;
preserveLiveWork?: boolean;
seedDatabase: SeedWorktreeDatabase;
}): Promise<SeedWorktreeDatabaseResult> {
let activePhase: WorktreeSeedPhase = "pending";
const previous = readWorktreeSeedManifest(input.configPath);
if (previous?.state === "running") {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: previous.phase,
status: "failed",
state: "failed",
message: "The previous seed attempt ended without a terminal result.",
});
}
startWorktreeSeedAttempt(input.configPath);
const unregisterInterruption = registerSeedInterruptHandler((signal) => {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: activePhase,
status: "failed",
state: "failed",
message: `Seed interrupted by ${signal} during ${activePhase}.`,
});
});
try {
const details = await input.seedDatabase({
sourceConfigPath: input.sourceConfigPath,
sourceConfig: input.sourceConfig,
targetConfig: input.targetConfig,
targetPaths: input.targetPaths,
instanceId: input.instanceId,
seedMode: input.seedMode,
preserveLiveWork: input.preserveLiveWork,
onPhase: (phase, status, message) => {
activePhase = phase;
updateWorktreeSeedManifest({
configPath: input.configPath,
phase,
status,
state: "running",
message,
...(phase === "snapshot" && status === "started"
? { snapshotAt: new Date().toISOString() }
: {}),
});
},
});
if (!details.snapshotAt || !details.migrationRevision || !details.validation) {
throw new Error("Seed implementation returned without required validation evidence.");
}
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: "complete",
status: "succeeded",
state: "verified",
snapshotAt: details.snapshotAt,
migrationRevision: details.migrationRevision,
message:
`Verified ${details.validation.companyCount} company record(s), `
+ `${details.validation.issueCount} issue record(s), auth, admin, membership, and migration state.`,
});
return details;
} catch (error) {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: activePhase,
status: "failed",
state: "failed",
// Do not persist the underlying error: database/driver errors may contain
// connection credentials. The CLI still returns the exact error to its caller.
message: `Seed failed during ${activePhase}.`,
});
throw error;
} finally {
unregisterInterruption();
}
}
export async function ensureWorktreeSeeded(
opts: WorktreeEnsureSeededOptions = {},
dependencies: { seedDatabase?: SeedWorktreeDatabase } = {},
@@ -1642,24 +2075,36 @@ export async function ensureWorktreeSeeded(
try {
// These checks deliberately happen under the cross-process lock. A second
// service process waits for the first seed transaction, then observes the
// complete marker instead of cloning the same database concurrently.
if (existsSync(markers.complete)) {
// verified manifest instead of cloning the same database concurrently.
let manifest = readWorktreeSeedManifest(configPath);
if (manifest?.state === "verified") {
return { seeded: false, reason: "verified_manifest" };
}
if (!manifest && existsSync(markers.complete)) {
return { seeded: false, reason: "complete_marker" };
}
if (!existsSync(markers.pending)) {
if (!manifest && existsSync(markers.pending)) {
const legacyPending = readLegacyWorktreeSeedPendingMarker(markers.pending);
markWorktreeSeedPending({
configPath,
sourceConfigPath: legacyPending.sourceConfigPath,
seedMode: "minimal",
});
manifest = readWorktreeSeedManifest(configPath);
}
if (!manifest) {
// Worktrees created before lazy seeding shipped were seeded eagerly and
// have neither marker. Preserve that compatibility without re-cloning.
return { seeded: false, reason: "legacy_unmarked" };
}
const pending = readWorktreeSeedPendingMarker(markers.pending);
const sourceConfigPath = opts.fromConfig || opts.fromDataDir || opts.fromInstance
? resolveSourceConfigPath({
fromConfig: opts.fromConfig,
fromDataDir: opts.fromDataDir,
fromInstance: opts.fromInstance,
})
: path.resolve(pending.sourceConfigPath);
: path.resolve(manifest.source.configPath);
if (path.resolve(sourceConfigPath) === path.resolve(configPath)) {
throw new Error(
@@ -1679,16 +2124,17 @@ export async function ensureWorktreeSeeded(
const targetRoot = path.dirname(path.dirname(configPath));
const targetPaths = resolveWorktreeReseedTargetPaths({ configPath, rootPath: targetRoot });
const seedDatabase = dependencies.seedDatabase ?? seedWorktreeDatabase;
const details = await seedDatabase({
const details = await runVerifiedWorktreeSeed({
configPath,
sourceConfigPath,
sourceConfig,
targetConfig,
targetPaths,
instanceId: targetPaths.instanceId,
seedMode: "minimal",
seedMode: manifest.seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase,
});
markWorktreeSeedComplete({ configPath });
return { seeded: true, reason: "seeded", details };
} finally {
await releaseLock();
@@ -1762,6 +2208,8 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
markWorktreeSeedPending({
configPath: paths.configPath,
sourceConfigPath,
targetInstanceId: instanceId,
seedMode,
});
const sourceEnvEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(sourceConfigPath));
const existingAgentJwtSecret =
@@ -1790,8 +2238,11 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
}
const spinner = p.spinner();
spinner.start(`Seeding isolated worktree database from source instance (${seedMode})...`);
const markers = resolveWorktreeSeedMarkerPaths(paths.configPath);
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
try {
const seeded = await seedWorktreeDatabase({
const seeded = await runVerifiedWorktreeSeed({
configPath: paths.configPath,
sourceConfigPath,
sourceConfig,
targetConfig,
@@ -1799,16 +2250,18 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
instanceId,
seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase: seedWorktreeDatabase,
});
seedSummary = seeded.backupSummary;
seedExecutionQuarantineSummary = seeded.executionQuarantine;
pausedScheduledRoutineCount = seeded.pausedScheduledRoutines;
reboundWorkspaceSummary = seeded.reboundWorkspaces;
markWorktreeSeedComplete({ configPath: paths.configPath, seedMode });
spinner.stop(`Seeded isolated worktree database (${seedMode}).`);
} catch (error) {
spinner.stop(pc.red("Failed to seed worktree database."));
throw error;
} finally {
await releaseSeedLock();
}
}
@@ -3480,6 +3933,34 @@ export async function worktreeMergeHistoryCommand(sourceArg: string | undefined,
}
}
async function backupWorktreeReseedTarget(input: {
targetConfig: PaperclipConfig;
targetPaths: WorktreeLocalPaths;
}): Promise<string> {
if (input.targetConfig.database.mode !== "embedded-postgres") {
throw new Error("Managed worktree repair requires an embedded PostgreSQL target.");
}
const targetHandle = await ensureEmbeddedPostgres(
input.targetConfig.database.embeddedPostgresDataDir,
input.targetConfig.database.embeddedPostgresPort,
);
try {
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const result = await runDatabaseBackup({
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`,
backupDir: path.resolve(input.targetPaths.backupDir, "repair"),
retention: { dailyDays: 30, weeklyWeeks: 12, monthlyMonths: 12 },
filenamePrefix: `${input.targetPaths.instanceId}-pre-repair`,
backupEngine: "auto",
includeMigrationJournal: true,
});
return formatDatabaseBackupResult(result);
} finally {
if (targetHandle.startedByThisProcess) await targetHandle.stop();
}
}
async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
const seedMode = opts.seedMode ?? "full";
if (!isWorktreeSeedMode(seedMode)) {
@@ -3535,8 +4016,23 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
const spinner = p.spinner();
spinner.start(`Reseeding ${targetEndpoint.label} from ${source.label} (${seedMode})...`);
const markers = resolveWorktreeSeedMarkerPaths(targetEndpoint.configPath);
mkdirSync(path.dirname(markers.lock), { recursive: true });
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
try {
const seeded = await seedWorktreeDatabase({
let targetBackupSummary: string | null = null;
if (opts.backupTarget) {
targetBackupSummary = await backupWorktreeReseedTarget({ targetConfig, targetPaths });
p.log.message(pc.dim(`Recoverable pre-repair backup: ${targetBackupSummary}`));
}
markWorktreeSeedPending({
configPath: targetEndpoint.configPath,
sourceConfigPath: source.configPath,
targetInstanceId: targetPaths.instanceId,
seedMode,
});
const seeded = await runVerifiedWorktreeSeed({
configPath: targetEndpoint.configPath,
sourceConfigPath: source.configPath,
sourceConfig,
targetConfig,
@@ -3544,8 +4040,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
instanceId: targetPaths.instanceId,
seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase: seedWorktreeDatabase,
});
markWorktreeSeedComplete({ configPath: targetEndpoint.configPath, seedMode });
spinner.stop(`Reseeded ${targetEndpoint.label} (${seedMode}).`);
p.log.message(pc.dim(`Source: ${source.configPath}`));
p.log.message(pc.dim(`Target: ${targetEndpoint.configPath}`));
@@ -3567,6 +4063,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
} catch (error) {
spinner.stop(pc.red("Failed to reseed worktree database."));
throw error;
} finally {
await releaseSeedLock();
}
}
@@ -3748,6 +4246,7 @@ export function registerWorktreeCommands(program: Command): void {
.option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
.option("--yes", "Skip the destructive confirmation prompt", false)
.option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false)
.option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false)
.action(worktreeReseedCommand);
worktree
+11 -9
View File
@@ -479,25 +479,24 @@ After `worktree init`, both the server and the CLI auto-load the repo-local `.pa
Seeding a worktree database is the heaviest part of `worktree init`. That work can be deferred so a worktree is cheap to create and only pays the seed cost the first time it is actually used — the CLI/dev-time analog of the server's lazy runtime provisioning (see the board-operator guide's "Lazy runtime provisioning" section).
Seeding state is tracked with two marker files under the worktree's `.paperclip/` directory:
Seeding state is tracked in `.paperclip/seed-manifest.json`. The versioned manifest records only non-secret evidence: source instance id/config path, target instance id, seed mode, snapshot time, migration revision, attempt timestamps, current phase, terminal state, and a bounded phase diagnostic history. It never stores database credentials or auth credential material. The legacy `seed-pending` and `seed-complete` files remain read-only compatibility signals for worktrees created before the manifest shipped.
- `seed-pending` — the isolated database has not been seeded yet (a **lean** worktree). Written by `worktree init` before any seed runs.
- `seed-complete` — the database was seeded; the pending marker is removed.
The default `worktree init` still seeds eagerly. A lean worktree (created without an eager seed) has a `pending` manifest until something seeds it on demand:
The default `worktree init` still seeds eagerly and writes `seed-complete` immediately. A lean worktree (created without an eager seed) keeps its `seed-pending` marker until something seeds it on demand:
- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: a present `seed-complete` marker or a missing `seed-pending` marker short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source instance from the `seed-pending` marker unless you pass `--from-config`.
- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: only a complete `verified` manifest short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source identity from the manifest unless you pass `--from-config`.
- `paperclipai run` calls `ensureWorktreeSeeded` automatically before doctor/boot, so `run` transparently seeds a lean worktree on first launch.
- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the worktree is still `seed-pending`. An explicitly configured runtime provision command always takes precedence.
- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the manifest is not verified. An explicitly configured runtime provision command always takes precedence.
- Worktrees created before lazy seeding shipped have neither marker; they are treated as already-seeded for backward compatibility (never re-cloned).
**Seed-pending guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose database is still `seed-pending` and points you at the fix:
Both `minimal` and `full` modes use the same terminal validation contract. Before a manifest can become `verified`, Paperclip proves the migration journal is current and reads a credential-backed auth user, instance administrator role, active company membership, and representative cloned company/issue pair in both source and target. Restore, migrations, execution quarantine, routine pausing, workspace rebinding, and post-restore validation all run under the seed lock. An interruption leaves the exact active phase in terminal `failed` state; it cannot produce readiness evidence.
**Unverified-seed guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose manifest is pending, running, failed, malformed, or missing required verification evidence and points you at the fix:
```
[paperclip] this worktree database is seed-pending. Run `pnpm paperclipai worktree ensure-seeded` before `pnpm dev`.
```
This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty, unseeded database — run `worktree ensure-seeded` once and re-run `pnpm dev`.
This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty or partially restored database — run `worktree ensure-seeded` once and re-run `pnpm dev`.
Provisioned git worktrees also pause seeded routines that still have enabled schedule triggers in the isolated worktree database by default. This prevents copied daily/cron routines from firing unexpectedly inside the new workspace instance during development without disabling webhook/API-only routines.
@@ -603,6 +602,7 @@ For an already-created worktree where you want to keep the existing repo-local c
| `--seed-mode <mode>` | Seed profile: `minimal` or `full` (default: `full`) |
| `--yes` | Skip the destructive confirmation prompt |
| `--allow-live-target` | Override the guard that requires the target worktree DB to be stopped first |
| `--backup-target` | Retain a recoverable full target-DB backup before reseeding |
Examples:
@@ -622,6 +622,8 @@ npx paperclipai worktree reseed \
--seed-mode full
```
Managed workspace repair uses this same verified full-reseed contract through `POST /api/execution-workspaces/:id/runtime-commands/repair`. The exclusive, audited operation stops managed services, writes a recoverable pre-repair database backup under the isolated instance's backup directory, performs the full seed/migration/quarantine/rebinding sequence, and restarts only after terminal manifest and service-health validation. It preserves the worktree filesystem. On failure, services remain stopped while the database backup, seed manifest, bounded phase diagnostics, and operation log are retained for inspection; repair never retries itself in a loop.
**`npx paperclipai worktree:make <name> [options]`** — Create `~/NAME` as a git worktree, then initialize an isolated Paperclip instance inside it. This combines `git worktree add` with `worktree init` in a single step.
| Option | Description |
@@ -3,6 +3,7 @@ export type WorkspaceOperationPhase =
| "workspace_config_freshness"
| "workspace_provision"
| "workspace_runtime_provision"
| "workspace_repair"
| "workspace_teardown"
| "worktree_cleanup"
| "workspace_finalize";
@@ -137,7 +137,10 @@ test("uses the base CLI when its import graph boots", () => {
assert.equal(result.status, 0, result.stderr);
const config = readWorktreeConfig(worktreeCwd);
assert.equal(config.$meta.source, "fake-cli");
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
assert.equal(
JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state,
"pending",
);
const initInvocation = readCliInvocations(baseCwd).find(
(args) => args[0] === "worktree" && args[1] === "init",
);
@@ -166,7 +169,10 @@ test("falls back to an isolated config when the base CLI cannot boot", () => {
);
const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8");
assert.match(env, /PAPERCLIP_IN_WORKTREE=true/);
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
assert.equal(
JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state,
"pending",
);
});
test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => {
@@ -271,7 +277,7 @@ test("runtime provisioning invokes ensure-seeded once and fast-exits after succe
const second = runRuntimeProvision(baseCwd, worktreeCwd);
assert.equal(second.status, 0, second.stderr);
assert.match(second.stderr, /already seeded; skipping/);
assert.match(second.stderr, /already seeded.*skipping/);
const ensureCallsAfterSecond = readCliInvocations(baseCwd)
.filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded");
assert.equal(ensureCallsAfterSecond.length, 1);
@@ -290,3 +296,22 @@ test("runtime provisioning leaves seed-pending in place when ensure-seeded fails
assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending")));
assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-complete")));
});
test("runtime provisioning does not trust a truncated verified manifest", () => {
const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0, ensureExit: 4 });
const worktreeCwd = makeTempDir("paperclip-provision-runtime-truncated-");
fs.mkdirSync(path.join(worktreeCwd, ".paperclip"), { recursive: true });
fs.writeFileSync(path.join(worktreeCwd, ".paperclip", "config.json"), "{}\n");
fs.writeFileSync(
path.join(worktreeCwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "verified" }),
);
const result = runRuntimeProvision(baseCwd, worktreeCwd);
assert.equal(result.status, 4, result.stderr);
assert.equal(
readCliInvocations(baseCwd)
.filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded").length,
1,
);
});
+44 -3
View File
@@ -7,6 +7,7 @@ paperclip_home="${PAPERCLIP_HOME:-$HOME/.paperclip}"
paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}"
paperclip_dir="$worktree_cwd/.paperclip"
worktree_config_path="$paperclip_dir/config.json"
seed_manifest_path="$paperclip_dir/seed-manifest.json"
seed_pending_marker_path="$paperclip_dir/seed-pending"
seed_complete_marker_path="$paperclip_dir/seed-complete"
@@ -20,8 +21,37 @@ if [[ ! -d "$worktree_cwd" ]]; then
exit 1
fi
if [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then
echo "Worktree database is already seeded; skipping runtime provisioning." >&2
if [[ -e "$seed_manifest_path" ]]; then
seed_manifest_state="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF'
const fs = require("node:fs");
try {
const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8"));
const complete = value?.version === 2
&& value?.state === "verified"
&& value?.phase === "complete"
&& typeof value?.source?.instanceId === "string" && value.source.instanceId.length > 0
&& typeof value?.source?.configPath === "string" && value.source.configPath.length > 0
&& (value?.seedMode === "minimal" || value?.seedMode === "full")
&& typeof value?.snapshotAt === "string" && value.snapshotAt.length > 0
&& typeof value?.migrationRevision === "string" && value.migrationRevision.length > 0
&& typeof value?.targetInstanceId === "string" && value.targetInstanceId.length > 0
&& typeof value?.attemptId === "string" && value.attemptId.length > 0
&& typeof value?.startedAt === "string"
&& typeof value?.finishedAt === "string"
&& Array.isArray(value?.diagnostics)
&& value.diagnostics.some((entry) => entry?.phase === "complete" && entry?.status === "succeeded" && typeof entry?.at === "string");
process.stdout.write(complete ? "verified" : "incomplete");
} catch {
process.stdout.write("invalid");
}
EOF
)"
if [[ "$seed_manifest_state" == "verified" ]]; then
echo "Worktree database has a verified seed manifest; skipping runtime provisioning." >&2
exit 0
fi
elif [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then
echo "Worktree database is already seeded by a legacy marker; skipping runtime provisioning." >&2
exit 0
fi
@@ -31,6 +61,17 @@ if [[ ! -f "$worktree_config_path" ]]; then
fi
source_config_path="${PAPERCLIP_CONFIG:-}"
if [[ -e "$seed_manifest_path" ]]; then
manifest_source_config_path="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF'
const fs = require("node:fs");
const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8"));
process.stdout.write(typeof value?.source?.configPath === "string" ? value.source.configPath : "");
EOF
)"
if [[ -n "$manifest_source_config_path" ]]; then
source_config_path="$manifest_source_config_path"
fi
fi
if [[ -z "$source_config_path" && ( -e "$base_cwd/.paperclip/config.json" || -L "$base_cwd/.paperclip/config.json" ) ]]; then
source_config_path="$base_cwd/.paperclip/config.json"
fi
@@ -41,7 +82,7 @@ source_config_args=(--from-config "$source_config_path")
if [[ "$source_config_path" == "$worktree_config_path" ]]; then
# A human may invoke this after sourcing `worktree env`, which points
# PAPERCLIP_CONFIG at the target. In that case the CLI reads the original
# source config from the seed-pending marker instead.
# source config from the seed manifest or legacy pending marker instead.
source_config_args=()
fi
+32 -8
View File
@@ -8,6 +8,7 @@ paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}"
paperclip_dir="$worktree_cwd/.paperclip"
worktree_config_path="$paperclip_dir/config.json"
worktree_env_path="$paperclip_dir/.env"
seed_manifest_path="$paperclip_dir/seed-manifest.json"
seed_pending_marker_path="$paperclip_dir/seed-pending"
seed_complete_marker_path="$paperclip_dir/seed-complete"
worktree_name="${PAPERCLIP_WORKSPACE_BRANCH:-$(basename "$worktree_cwd")}"
@@ -237,25 +238,48 @@ for (const rawValue of runtimePaths) {
EOF
}
write_seed_pending_marker() {
write_seed_pending_manifest() {
SEED_MANIFEST_PATH="$seed_manifest_path" \
SEED_PENDING_MARKER_PATH="$seed_pending_marker_path" \
SEED_COMPLETE_MARKER_PATH="$seed_complete_marker_path" \
SOURCE_CONFIG_PATH="$source_config_path" \
TARGET_INSTANCE_ID="$worktree_instance_id" \
node <<'EOF'
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const manifestPath = process.env.SEED_MANIFEST_PATH;
const pendingPath = process.env.SEED_PENDING_MARKER_PATH;
const completePath = process.env.SEED_COMPLETE_MARKER_PATH;
const sourceConfigPath = path.resolve(process.env.SOURCE_CONFIG_PATH);
const sourceEnvPath = path.join(path.dirname(sourceConfigPath), ".env");
let sourceInstanceId = path.basename(path.dirname(sourceConfigPath));
if (fs.existsSync(sourceEnvPath)) {
const match = fs.readFileSync(sourceEnvPath, "utf8").match(/^\s*(?:export\s+)?PAPERCLIP_INSTANCE_ID\s*=\s*["']?([^\s"'#]+)["']?/m);
if (match?.[1]) sourceInstanceId = match[1];
}
fs.rmSync(completePath, { force: true });
fs.rmSync(pendingPath, { force: true });
const at = new Date().toISOString();
fs.writeFileSync(
pendingPath,
manifestPath,
`${JSON.stringify({
version: 1,
state: "pending",
sourceConfigPath: path.resolve(process.env.SOURCE_CONFIG_PATH),
version: 2,
source: {
instanceId: sourceInstanceId,
configPath: sourceConfigPath,
},
snapshotAt: null,
seedMode: "minimal",
createdAt: new Date().toISOString(),
migrationRevision: null,
targetInstanceId: process.env.TARGET_INSTANCE_ID,
phase: "pending",
state: "pending",
attemptId: crypto.randomUUID(),
startedAt: null,
finishedAt: null,
diagnostics: [{ phase: "pending", status: "succeeded", at }],
}, null, 2)}\n`,
{ mode: 0o600 },
);
@@ -551,8 +575,8 @@ else
created_worktree_config=1
fi
if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then
write_seed_pending_marker
if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_manifest_path" && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then
write_seed_pending_manifest
fi
list_base_node_modules_paths() {
@@ -36,6 +36,36 @@ describe("dev-runner worktree env bootstrap", () => {
expect(isWorktreeSeedPending(root)).toBe(false);
});
it("guards every manifest state except a complete verified manifest", () => {
const root = createTempRoot("paperclip-dev-runner-seed-manifest-");
const manifestPath = path.join(root, ".paperclip", "seed-manifest.json");
fs.mkdirSync(path.dirname(manifestPath), { recursive: true });
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "failed" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "verified" }), "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
fs.writeFileSync(manifestPath, JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "minimal",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}), "utf8");
expect(isWorktreeSeedPending(root)).toBe(false);
fs.writeFileSync(manifestPath, "not-json", "utf8");
expect(isWorktreeSeedPending(root)).toBe(true);
});
it("detects linked git worktrees from .git files", () => {
const root = createTempRoot("paperclip-dev-runner-worktree-");
fs.writeFileSync(path.join(root, ".git"), "gitdir: /tmp/paperclip/.git/worktrees/feature\n", "utf8");
@@ -1,3 +1,8 @@
import { EventEmitter } from "node:events";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import express from "express";
import request from "supertest";
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -29,6 +34,12 @@ const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn());
// `execution-workspace-runtime-lease-route.test.ts` exercises the lease itself against a real db.
const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null));
const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined));
const mockSpawn = vi.hoisted(() => vi.fn());
vi.mock("node:child_process", async () => ({
...(await vi.importActual<typeof import("node:child_process")>("node:child_process")),
spawn: mockSpawn,
}));
vi.mock("../telemetry.js", () => ({ getTelemetryClient: mockGetTelemetryClient }));
@@ -45,7 +56,7 @@ vi.mock("../services/index.js", () => ({
claim: mockClaimRuntimeLease,
release: mockReleaseRuntimeLease,
}),
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
@@ -135,6 +146,7 @@ async function createApp() {
describe.sequential("execution workspace runtime control conflict and failure reconciliation", () => {
beforeEach(() => {
vi.clearAllMocks();
mockSpawn.mockReset();
mockAccessService.decide.mockResolvedValue({
allowed: true,
action: "runtime:manage",
@@ -165,7 +177,7 @@ describe.sequential("execution workspace runtime control conflict and failure re
// Mirror the real recorder: a throwing `run` becomes a terminal failed operation and
// the error still propagates to the caller.
try {
await input.run();
await input.run(async () => undefined);
} catch (error) {
(error as any).recordedOperationStatus = "failed";
throw error;
@@ -247,6 +259,156 @@ describe.sequential("execution workspace runtime control conflict and failure re
);
});
it("runs an audited full repair through terminal seed and service readiness validation", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-success-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
fs.mkdirSync(configDir, { recursive: true });
fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true });
fs.mkdirSync(path.dirname(cliRunner), { recursive: true });
fs.mkdirSync(path.dirname(cliEntry), { recursive: true });
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(sourceConfigPath, "{}\n");
fs.writeFileSync(cliRunner, "// test runner\n");
fs.writeFileSync(cliEntry, "// test entry\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
attemptId: "previous",
}));
const { deriveWorktreeInstanceId } = await import("../services/workspace-instance-cleanup.js");
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
mockStartRuntimeServices.mockResolvedValue([{
id: "service-1",
status: "running",
healthStatus: "healthy",
}]);
mockSpawn.mockImplementation((_command: string, args: string[]) => {
expect(args).toEqual(expect.arrayContaining([
"worktree", "reseed", "--seed-mode", "full", "--yes", "--backup-target",
]));
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0142_test.sql",
targetInstanceId: deriveWorktreeInstanceId(workspaceCwd),
phase: "complete",
state: "verified",
attemptId: "repair-attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}));
child.emit("exit", 0);
});
return child;
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBe(200);
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledWith(
expect.objectContaining({ executionWorkspaceId, workspaceCwd }),
);
expect(mockStartRuntimeServices).toHaveBeenCalledTimes(1);
expect(mockLogActivity).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ action: "execution_workspace.runtime_repair" }),
);
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it("fails repair at the exact seed phase and leaves managed services stopped", async () => {
const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-failure-"));
try {
const configDir = path.join(workspaceCwd, ".paperclip");
const sourceConfigPath = path.join(workspaceCwd, "source", "config.json");
const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts");
for (const dir of [configDir, path.dirname(sourceConfigPath), path.dirname(cliRunner), path.dirname(cliEntry)]) {
fs.mkdirSync(dir, { recursive: true });
}
fs.writeFileSync(path.join(configDir, "config.json"), "{}\n");
fs.writeFileSync(sourceConfigPath, "{}\n");
fs.writeFileSync(cliRunner, "// test runner\n");
fs.writeFileSync(cliEntry, "// test entry\n");
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
attemptId: "previous",
}));
mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd }));
mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd });
mockSpawn.mockImplementation(() => {
const child = new EventEmitter() as EventEmitter & {
stdout: PassThrough;
stderr: PassThrough;
kill: ReturnType<typeof vi.fn>;
};
child.stdout = new PassThrough();
child.stderr = new PassThrough();
child.kill = vi.fn();
queueMicrotask(() => {
fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: sourceConfigPath },
state: "failed",
phase: "restore",
attemptId: "repair-attempt",
}));
child.emit("exit", 4);
});
return child;
});
const progress: Array<Record<string, unknown>> = [];
mockWorkspaceOperationService.createRecorder.mockReturnValue({
attachExecutionWorkspaceId: vi.fn(),
recordOperation: async (input: any) => {
await input.run(async (entry: Record<string, unknown>) => {
progress.push(entry);
});
},
});
const app = await createApp();
const res = await request(app)
.post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`)
.send({});
expect(res.status).toBeGreaterThanOrEqual(400);
expect(progress).toEqual(expect.arrayContaining([
expect.objectContaining({ metadata: { seedFailurePhase: "restore" } }),
]));
expect(mockStartRuntimeServices).not.toHaveBeenCalled();
expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(2);
} finally {
fs.rmSync(workspaceCwd, { recursive: true, force: true });
}
});
it("reconciles once when the operation's own time budget fails a start that never settles", async () => {
const { WorkspaceOperationTimeoutError } = await import("../services/workspace-operations.js");
// The recorder's ceiling fires outside `run`, so only the outer handler can reconcile.
@@ -19,6 +19,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({
const mockWorkspaceOperationService = vi.hoisted(() => ({
listForExecutionWorkspace: vi.fn(),
createRecorder: vi.fn(),
assertRuntimeControlAvailable: vi.fn(async () => undefined),
}));
const mockWorkspaceRuntimeLeaseService = vi.hoisted(() => ({
@@ -47,7 +48,7 @@ vi.mock("../services/index.js", () => ({
logActivity: mockLogActivity,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/environment-runtime.js", () => ({
@@ -44,14 +44,14 @@ describe("workspace runtime control serialization", () => {
await expect(runExclusiveWorkspaceRuntimeControl({
executionWorkspaceId: "workspace-1",
action: "restart",
run: async () => "restarted",
action: "repair",
run: async () => "repaired",
})).rejects.toMatchObject({
status: 409,
details: {
code: "workspace_runtime_control_in_progress",
activeAction: "start",
requestedAction: "restart",
requestedAction: "repair",
},
});
@@ -204,4 +204,71 @@ describeEmbeddedPostgres("workspace operation reconciliation", () => {
.then((rows) => rows[0]);
expect(unrelated?.status).toBe("running");
});
it("persists bounded repair phase progress on the operation row and log", async () => {
const companyId = randomUUID();
const projectId = randomUUID();
const executionWorkspaceId = randomUUID();
await db.insert(companies).values({
id: companyId,
name: "Workspace repair diagnostics",
issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`,
requireBoardApprovalForNewAgents: false,
});
await db.insert(projects).values({
id: projectId,
companyId,
name: "Repair diagnostics",
status: "in_progress",
});
await db.insert(executionWorkspaces).values({
id: executionWorkspaceId,
companyId,
projectId,
mode: "isolated_workspace",
strategyType: "git_worktree",
name: "Repair diagnostics workspace",
status: "active",
cwd: "/tmp/repair-diagnostics-workspace",
});
const operation = await workspaceOperationService(db)
.createRecorder({ companyId, executionWorkspaceId })
.recordOperation({
phase: "workspace_repair",
command: "workspace command repair",
cwd: "/tmp/repair-diagnostics-workspace",
metadata: { action: "repair" },
run: async (reportProgress) => {
await reportProgress({
metadata: {
repairPhase: "target_backup",
repairDiagnostics: [{
phase: "target_backup",
status: "succeeded",
at: "2026-08-18T00:00:00.000Z",
}],
databaseOnly: true,
worktreePreserved: true,
},
system: "Workspace repair target_backup: succeeded.\n",
});
return { status: "succeeded", metadata: { backupRetained: true } };
},
});
expect(operation).toMatchObject({
status: "succeeded",
metadata: {
action: "repair",
repairPhase: "target_backup",
databaseOnly: true,
worktreePreserved: true,
backupRetained: true,
},
});
expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain(
"Workspace repair target_backup: succeeded.",
);
});
});
@@ -54,7 +54,7 @@ vi.mock("../services/index.js", () => ({
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.mock("../services/workspace-runtime.js", () => ({
@@ -84,7 +84,7 @@ function registerWorkspaceRouteMocks() {
secretService: () => mockSecretService,
workspaceOperationService: () => mockWorkspaceOperationService,
workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService,
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"],
LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"],
}));
vi.doMock("../services/workspace-runtime.js", () => ({
@@ -456,6 +456,39 @@ describe("resolveRuntimeProvisionCommand", () => {
await fs.writeFile(path.join(cwd, ".paperclip", "seed-complete"), "{}\n");
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "failed" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({ version: 2, state: "verified" }),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(
"bash ./scripts/provision-worktree-runtime.sh",
);
await fs.writeFile(
path.join(cwd, ".paperclip", "seed-manifest.json"),
JSON.stringify({
version: 2,
source: { instanceId: "source", configPath: "/source/config.json" },
snapshotAt: "2026-08-18T00:00:00.000Z",
seedMode: "full",
migrationRevision: "0001",
targetInstanceId: "target",
phase: "complete",
state: "verified",
attemptId: "attempt",
startedAt: "2026-08-18T00:00:00.000Z",
finishedAt: "2026-08-18T00:01:00.000Z",
diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }],
}),
);
expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe("");
} finally {
await fs.rm(baseCwd, { recursive: true, force: true });
}
+5
View File
@@ -1,6 +1,7 @@
import { existsSync, lstatSync, readFileSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { hasVerifiedWorktreeSeedManifest } from "./worktree-seed-manifest.js";
function parseEnvFile(contents: string): Record<string, string> {
const entries: Record<string, string> = {};
@@ -58,6 +59,10 @@ export function resolveWorktreeEnvFilePath(rootDir: string): string {
export function isWorktreeSeedPending(rootDir: string): boolean {
const markerDir = path.resolve(rootDir, ".paperclip");
const manifestPath = path.resolve(markerDir, "seed-manifest.json");
if (existsSync(manifestPath)) {
return !hasVerifiedWorktreeSeedManifest(manifestPath);
}
return existsSync(path.resolve(markerDir, "seed-pending"))
&& !existsSync(path.resolve(markerDir, "seed-complete"));
}
+248 -8
View File
@@ -1,3 +1,6 @@
import { spawn } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import path from "node:path";
import { and, eq } from "drizzle-orm";
import { Router, type Request, type Response } from "express";
import type { Db } from "@paperclipai/db";
@@ -45,6 +48,8 @@ import { appendWithCap } from "../adapters/utils.js";
import { environmentRuntimeService } from "../services/environment-runtime.js";
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js";
import { deriveWorktreeInstanceId } from "../services/workspace-instance-cleanup.js";
import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024;
@@ -151,7 +156,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
async function handleExecutionWorkspaceRuntimeCommand(req: Request, res: Response) {
const id = req.params.id as string;
const action = String(req.params.action ?? "").trim().toLowerCase();
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "run") {
if (action !== "start" && action !== "stop" && action !== "restart" && action !== "repair" && action !== "run") {
res.status(404).json({ error: "Workspace command action not found" });
return;
}
@@ -262,7 +267,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
return;
}
if ((action === "start" || action === "restart") && !effectiveRuntimeConfig) {
if ((action === "start" || action === "restart" || action === "repair") && !effectiveRuntimeConfig) {
res.status(422).json({ error: "Execution workspace has no workspace command configuration or inherited project workspace default" });
return;
}
@@ -327,7 +332,11 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
}
const recordRuntimeControlOperation = () => recorder.recordOperation({
phase: action === "stop" ? "workspace_teardown" : "workspace_provision",
phase: action === "stop"
? "workspace_teardown"
: action === "repair"
? "workspace_repair"
: "workspace_provision",
command: workspaceCommand?.command ?? `workspace command ${action}`,
cwd: existing.cwd,
metadata: {
@@ -339,7 +348,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
runtimeServiceId: selectedRuntimeServiceId,
serviceIndex: selectedServiceIndex,
},
run: async () => {
run: async (reportProgress) => {
const ensureWorkspaceAvailable = async () =>
await ensurePersistedExecutionWorkspaceAvailable({
base: {
@@ -429,7 +438,234 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
else stderr = appendWithCap(stderr, chunk, WORKSPACE_CONTROL_OUTPUT_MAX_CHARS);
};
if (action === "stop" || action === "restart") {
if (action === "repair") {
type RepairPhase =
| "managed_stop"
| "target_backup"
| "full_reseed"
| "managed_restart"
| "readiness_validation";
const repairDiagnostics: Array<{
phase: RepairPhase;
status: "started" | "succeeded" | "failed";
at: string;
}> = [];
let repairPhase: RepairPhase = "managed_stop";
const reportRepairPhase = async (
phase: RepairPhase,
status: "started" | "succeeded" | "failed",
) => {
repairPhase = phase;
repairDiagnostics.push({ phase, status, at: new Date().toISOString() });
if (repairDiagnostics.length > 32) repairDiagnostics.splice(0, repairDiagnostics.length - 32);
await reportProgress({
metadata: {
repairPhase,
repairDiagnostics: [...repairDiagnostics],
databaseOnly: true,
worktreePreserved: true,
},
system: `Workspace repair ${phase}: ${status}.\n`,
});
};
try {
await reportRepairPhase("managed_stop", "started");
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
workspaceCwd,
});
await reportRepairPhase("managed_stop", "succeeded");
const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json");
const targetConfigPath = path.join(workspaceCwd, ".paperclip", "config.json");
let sourceConfigPath: string | null = null;
let previousAttemptId: string | null = null;
if (existsSync(manifestPath)) {
try {
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
attemptId?: unknown;
source?: { configPath?: unknown };
};
previousAttemptId = typeof manifest.attemptId === "string" ? manifest.attemptId : null;
sourceConfigPath = typeof manifest.source?.configPath === "string"
? path.resolve(manifest.source.configPath)
: null;
} catch {
throw new Error("Workspace seed manifest is malformed; repair source identity cannot be trusted.");
}
}
const baseWorkspaceCwd = projectWorkspace?.cwd ?? workspaceCwd;
const baseConfigPath = path.join(baseWorkspaceCwd, ".paperclip", "config.json");
if (!sourceConfigPath && existsSync(baseConfigPath) && path.resolve(baseConfigPath) !== path.resolve(targetConfigPath)) {
sourceConfigPath = baseConfigPath;
}
if (!sourceConfigPath || !existsSync(sourceConfigPath)) {
throw new Error("Workspace repair has no readable configured source instance.");
}
const cliRunner = path.join(baseWorkspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
const cliEntry = path.join(baseWorkspaceCwd, "cli", "src", "index.ts");
const cliDist = path.join(baseWorkspaceCwd, "cli", "dist", "index.js");
const cliArgs = existsSync(cliRunner) && existsSync(cliEntry)
? [cliRunner, cliEntry]
: existsSync(cliDist)
? [cliDist]
: null;
if (!cliArgs) {
throw new Error("Workspace repair cannot find a runnable Paperclip CLI in the base workspace.");
}
await reportRepairPhase("target_backup", "started");
const child = spawn(process.execPath, [
...cliArgs,
"worktree",
"reseed",
"--from-config",
sourceConfigPath,
"--to",
workspaceCwd,
"--seed-mode",
"full",
"--yes",
"--backup-target",
], {
cwd: baseWorkspaceCwd,
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
});
child.stdout?.on("data", (chunk: Buffer) => {
void onLog("stdout", chunk.toString("utf8"));
});
child.stderr?.on("data", (chunk: Buffer) => {
void onLog("stderr", chunk.toString("utf8"));
});
let repairCommandTimedOut = false;
let repairCommandForceTimeout: NodeJS.Timeout | null = null;
const repairCommandTimeout = setTimeout(() => {
repairCommandTimedOut = true;
child.kill("SIGTERM");
repairCommandForceTimeout = setTimeout(() => child.kill("SIGKILL"), 5_000);
repairCommandForceTimeout.unref?.();
}, 20 * 60_000);
repairCommandTimeout.unref?.();
let reseedObserved = false;
const manifestPoll = setInterval(() => {
if (reseedObserved || !existsSync(manifestPath)) return;
try {
const current = JSON.parse(readFileSync(manifestPath, "utf8")) as {
attemptId?: unknown;
state?: unknown;
};
if (
typeof current.attemptId === "string"
&& current.attemptId !== previousAttemptId
&& (current.state === "pending" || current.state === "running" || current.state === "failed")
) {
reseedObserved = true;
}
} catch {
// The atomic manifest writer makes this unlikely; the terminal
// read below remains authoritative.
}
}, 100);
manifestPoll.unref?.();
const exitCode = await new Promise<number>((resolve, reject) => {
child.once("error", reject);
child.once("exit", (code) => resolve(code ?? 1));
}).finally(() => {
clearInterval(manifestPoll);
clearTimeout(repairCommandTimeout);
if (repairCommandForceTimeout) clearTimeout(repairCommandForceTimeout);
});
if (reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
if (exitCode !== 0) {
let seedFailurePhase: string | null = null;
try {
const failedManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as {
phase?: unknown;
state?: unknown;
};
seedFailurePhase = failedManifest.state === "failed" && typeof failedManifest.phase === "string"
? failedManifest.phase
: null;
} catch {
// The outer repair phase remains exact when no seed phase was persisted.
}
await reportProgress({
metadata: { seedFailurePhase },
system: seedFailurePhase ? `Workspace seed failed during ${seedFailurePhase}.\n` : null,
});
throw new Error(
repairCommandTimedOut
? `Workspace database repair command timed out during ${repairPhase}.`
: `Workspace database repair command failed during ${repairPhase}.`,
);
}
if (!reseedObserved) {
await reportRepairPhase("target_backup", "succeeded");
await reportRepairPhase("full_reseed", "started");
}
const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record<string, unknown>;
if (!isVerifiedWorktreeSeedManifest(manifest)) {
throw new Error("Workspace reseed returned without a verified terminal manifest.");
}
const expectedInstanceId = deriveWorktreeInstanceId(workspaceCwd);
if (manifest.targetInstanceId !== expectedInstanceId) {
throw new Error("Verified seed manifest belongs to a different workspace instance.");
}
await reportRepairPhase("full_reseed", "succeeded");
await reportRepairPhase("managed_restart", "started");
const availableWorkspace = await ensureWorkspaceAvailable();
if (!availableWorkspace) {
throw new Error("Execution workspace needs a local path before Paperclip can restart it.");
}
const startedServices = await startRuntimeServicesForWorkspaceControl({
db,
actor: {
id: actor.agentId ?? null,
name: actor.actorType === "user" ? "Board" : "Agent",
companyId: existing.companyId,
},
issue: existing.sourceIssueId
? { id: existing.sourceIssueId, identifier: null, title: existing.name }
: null,
workspace: availableWorkspace,
executionWorkspaceId: existing.id,
config: {
workspaceRuntime: effectiveRuntimeConfig,
runtimeProvisionCommand:
existing.config?.runtimeProvisionCommand
?? projectPolicy?.workspaceStrategy?.runtimeProvisionCommand
?? null,
},
adapterEnv: {},
onLog,
recorder,
});
runtimeServiceCount = startedServices.length;
await reportRepairPhase("managed_restart", "succeeded");
await reportRepairPhase("readiness_validation", "started");
if (
startedServices.length === 0
|| startedServices.some((service) => service.status !== "running" || service.healthStatus !== "healthy")
) {
throw new Error("Managed restart did not produce healthy runtime services.");
}
await reportRepairPhase("readiness_validation", "succeeded");
} catch (error) {
await reportRepairPhase(repairPhase, "failed");
throw error;
}
} else if (action === "stop" || action === "restart") {
await stopRuntimeServicesForExecutionWorkspace({
db,
executionWorkspaceId: existing.id,
@@ -481,7 +717,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
throw error;
}
runtimeServiceCount = startedServices.length;
} else {
} else if (action !== "repair") {
runtimeServiceCount = selectedRuntimeServiceId ? Math.max(0, (existing.runtimeServices?.length ?? 1) - 1) : 0;
}
@@ -504,7 +740,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
config: { workspaceRuntime: effectiveRuntimeConfig },
currentDesiredState,
currentServiceStates: existing.config?.serviceStates ?? null,
action,
action: action === "repair" ? "start" : action,
serviceIndex: selectedServiceIndex,
});
const metadata = mergeExecutionWorkspaceConfig(existing.metadata as Record<string, unknown> | null, {
@@ -522,6 +758,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
? "Stopped execution workspace runtime services.\n"
: action === "restart"
? "Restarted execution workspace runtime services.\n"
: action === "repair"
? "Repaired the isolated workspace database and restarted healthy runtime services.\n"
: "Started execution workspace runtime services.\n",
metadata: {
runtimeServiceCount,
@@ -565,7 +803,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
// The operation is already terminal here. This also catches the recorder's own time
// budget expiring on a start that never settles, which is the one failure the inner
// handler cannot see — reconcile there too so no listener or desired state is left over.
if (action === "start" || action === "restart") await reconcileFailedRuntimeStart(error);
if (action === "start" || action === "restart" || action === "repair") {
await reconcileFailedRuntimeStart(error);
}
throw error;
}
},
+43 -16
View File
@@ -14,7 +14,7 @@ type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect;
* Managed runtime control actions. Every one of these mutates the runtime rows and
* local listeners of a single execution workspace, so only one may be live at a time.
*/
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "run"]);
const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]);
/**
* Identity of this server process. A `running` runtime-control operation stamped with a
@@ -234,7 +234,10 @@ export interface WorkspaceOperationRecorder {
* timeout error is rethrown, so a hung provider can never leave an active operation.
*/
timeoutMs?: number | null;
run: () => Promise<{
run: (reportProgress: (input: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => Promise<void>) => Promise<{
status?: WorkspaceOperationStatus;
exitCode?: number | null;
stdout?: string | null;
@@ -484,7 +487,7 @@ export function workspaceOperationService(db: Db) {
// Managed runtime controls get an ownership stamp so bounded recovery can tell a
// slow-but-live operation from one abandoned by a dead request or server process.
const runtimeControlAction = readRuntimeControlAction(recordInput.metadata);
const insertedMetadata = runtimeControlAction
let currentMetadata = runtimeControlAction
? {
...(recordInput.metadata ?? {}),
runtimeControlOwner: {
@@ -515,7 +518,7 @@ export function workspaceOperationService(db: Db) {
logStore: handle.store,
logRef: handle.logRef,
metadata: redactCurrentUserValue(
insertedMetadata,
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
startedAt,
@@ -532,17 +535,21 @@ export function workspaceOperationService(db: Db) {
if (runtimeControlAction) {
heartbeatTimer = setInterval(() => {
const heartbeatAt = new Date();
currentMetadata = combineMetadata(currentMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
});
void db
.update(workspaceOperations)
.set({
metadata: combineMetadata(insertedMetadata, {
runtimeControlOwner: {
ownerId: RUNTIME_CONTROL_OWNER_ID,
pid: process.pid,
action: runtimeControlAction,
heartbeatAt: heartbeatAt.toISOString(),
} satisfies RuntimeControlOwnerStamp,
}),
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
updatedAt: heartbeatAt,
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")))
@@ -551,9 +558,29 @@ export function workspaceOperationService(db: Db) {
heartbeatTimer.unref?.();
}
const reportProgress = async (progress: {
metadata?: Record<string, unknown> | null;
system?: string | null;
}) => {
await append("system", progress.system ?? null);
currentMetadata = combineMetadata(currentMetadata, progress.metadata);
await db
.update(workspaceOperations)
.set({
metadata: redactCurrentUserValue(
currentMetadata,
currentUserRedactionOptions,
) as Record<string, unknown> | null,
stdoutExcerpt: stdoutExcerpt || null,
stderrExcerpt: stderrExcerpt || null,
updatedAt: new Date(),
})
.where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running")));
};
const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction);
const settle = async () => {
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run();
if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress);
const timeout = new Promise<never>((_resolve, reject) => {
timeoutTimer = setTimeout(
() => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)),
@@ -561,7 +588,7 @@ export function workspaceOperationService(db: Db) {
);
timeoutTimer.unref?.();
});
return await Promise.race([recordInput.run(), timeout]);
return await Promise.race([recordInput.run(reportProgress), timeout]);
};
try {
@@ -583,7 +610,7 @@ export function workspaceOperationService(db: Db) {
logSha256: finalized.sha256,
logCompressed: finalized.compressed,
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, result.metadata),
combineMetadata(currentMetadata, result.metadata),
currentUserRedactionOptions,
) as Record<string, unknown> | null,
finishedAt,
@@ -613,7 +640,7 @@ export function workspaceOperationService(db: Db) {
...(runtimeControlAction
? {
metadata: redactCurrentUserValue(
combineMetadata(insertedMetadata, {
combineMetadata(currentMetadata, {
failureReason: error instanceof WorkspaceOperationTimeoutError
? "runtime_control_timeout"
: "runtime_control_error",
@@ -28,7 +28,7 @@ const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([
]);
/** Runtime control actions that mutate the workspace and therefore need the lease. */
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart"];
export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart", "repair"];
/**
* Upper bound on how long a lease survives without the owner touching it. Recovery
+7 -5
View File
@@ -36,6 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm";
import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js";
import { conflict } from "../errors.js";
import { resolveHomeAwarePath } from "../home-paths.js";
import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js";
import {
createLocalServiceKey,
findLocalServiceRegistryRecordByRuntimeServiceId,
@@ -5015,6 +5016,7 @@ export function resolveRuntimeProvisionCommand(input: {
if (input.workspace.strategy !== "git_worktree") return "";
const stateDir = path.join(input.workspace.cwd, ".paperclip");
const manifestPath = path.join(stateDir, "seed-manifest.json");
const pendingMarker = path.join(stateDir, "seed-pending");
const completeMarker = path.join(stateDir, "seed-complete");
const provisionScript = path.join(
@@ -5022,11 +5024,11 @@ export function resolveRuntimeProvisionCommand(input: {
"scripts",
"provision-worktree-runtime.sh",
);
if (
!existsSync(pendingMarker)
|| existsSync(completeMarker)
|| !existsSync(provisionScript)
) {
let needsSeed = existsSync(pendingMarker) && !existsSync(completeMarker);
if (existsSync(manifestPath)) {
needsSeed = !hasVerifiedWorktreeSeedManifest(manifestPath);
}
if (!needsSeed || !existsSync(provisionScript)) {
return "";
}
+54
View File
@@ -0,0 +1,54 @@
import { existsSync, readFileSync } from "node:fs";
const WORKTREE_SEED_MODES = new Set(["minimal", "full"]);
type SeedDiagnostic = {
phase?: unknown;
status?: unknown;
at?: unknown;
};
/**
* A manifest is readiness evidence only when the complete validation phase was
* durably recorded. A bare `state: verified` is deliberately insufficient so
* truncated or hand-edited files cannot start a partially restored workspace.
*/
export function isVerifiedWorktreeSeedManifest(value: unknown): boolean {
if (!value || typeof value !== "object") return false;
const manifest = value as Record<string, unknown>;
const source = manifest.source as Record<string, unknown> | null;
const diagnostics = manifest.diagnostics as SeedDiagnostic[] | null;
return manifest.version === 2
&& manifest.state === "verified"
&& manifest.phase === "complete"
&& typeof source?.instanceId === "string"
&& source.instanceId.length > 0
&& typeof source.configPath === "string"
&& source.configPath.length > 0
&& WORKTREE_SEED_MODES.has(String(manifest.seedMode ?? ""))
&& typeof manifest.snapshotAt === "string"
&& manifest.snapshotAt.length > 0
&& typeof manifest.migrationRevision === "string"
&& manifest.migrationRevision.length > 0
&& typeof manifest.targetInstanceId === "string"
&& manifest.targetInstanceId.length > 0
&& typeof manifest.attemptId === "string"
&& manifest.attemptId.length > 0
&& typeof manifest.startedAt === "string"
&& typeof manifest.finishedAt === "string"
&& Array.isArray(diagnostics)
&& diagnostics.some((diagnostic) => (
diagnostic?.phase === "complete"
&& diagnostic.status === "succeeded"
&& typeof diagnostic.at === "string"
));
}
export function hasVerifiedWorktreeSeedManifest(manifestPath: string): boolean {
if (!existsSync(manifestPath)) return false;
try {
return isVerifiedWorktreeSeedManifest(JSON.parse(readFileSync(manifestPath, "utf8")));
} catch {
return false;
}
}
+5
View File
@@ -114,6 +114,11 @@ export const executionWorkspacesApi = {
`/execution-workspaces/${id}/runtime-commands/${action}`,
sanitizeWorkspaceRuntimeControlTarget(target),
),
repair: (id: string) =>
api.post<{ workspace: ExecutionWorkspace; operation: WorkspaceOperation }>(
`/execution-workspaces/${id}/runtime-commands/repair`,
{},
),
update: (id: string, data: Record<string, unknown>) => api.patch<ExecutionWorkspace>(`/execution-workspaces/${id}`, data),
/**
* Reconcile a git-worktree branch divergence via the S4 (`PAP-1586`) op.