feat(workspaces): verify seeds and add managed repair

Replace binary readiness markers with an evidence-bearing seed manifest and enforce a shared validation contract for minimal and full clones. Add an exclusive audited database repair path with recoverable backup, bounded phase diagnostics, and healthy restart validation.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-08-18 23:20:22 +00:00
1 parent 4b968d8c05
commit 53a0aa3020
21 files changed
+1568 -154

No files matched your search

+42
View File
@@ -5,12 +5,52 @@ import { expandHomePrefix } from "../config/home.js";
export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees";
export const WORKTREE_SEED_MODES = ["minimal", "full"] as const;
export const WORKTREE_SEED_MANIFEST = "seed-manifest.json";
export const WORKTREE_SEED_PENDING_MARKER = "seed-pending";
export const WORKTREE_SEED_COMPLETE_MARKER = "seed-complete";
export const WORKTREE_SEED_LOCK_MARKER = "seed.lock";
export type WorktreeSeedMode = (typeof WORKTREE_SEED_MODES)[number];
export const WORKTREE_SEED_PHASES = [
"pending",
"source_validation",
"snapshot",
"restore",
"migrations",
"execution_quarantine",
"routine_pause",
"workspace_rebind",
"post_restore_validation",
"complete",
] as const;
export type WorktreeSeedPhase = (typeof WORKTREE_SEED_PHASES)[number];
export type WorktreeSeedState = "pending" | "running" | "verified" | "failed";
export type WorktreeSeedManifest = {
version: 2;
source: {
instanceId: string;
configPath: string;
};
snapshotAt: string | null;
seedMode: WorktreeSeedMode;
migrationRevision: string | null;
targetInstanceId: string;
phase: WorktreeSeedPhase;
state: WorktreeSeedState;
attemptId: string;
startedAt: string | null;
finishedAt: string | null;
diagnostics: Array<{
phase: WorktreeSeedPhase;
status: "started" | "succeeded" | "failed";
at: string;
message?: string;
}>;
};
export type WorktreeSeedPlan = {
mode: WorktreeSeedMode;
excludedTables: string[];
@@ -54,6 +94,7 @@ export type WorktreeUiBranding = {
};
export type WorktreeSeedMarkerPaths = {
manifest: string;
pending: string;
complete: string;
lock: string;
@@ -62,6 +103,7 @@ export type WorktreeSeedMarkerPaths = {
export function resolveWorktreeSeedMarkerPaths(configPath: string): WorktreeSeedMarkerPaths {
const configDir = path.dirname(path.resolve(configPath));
return {
manifest: path.resolve(configDir, WORKTREE_SEED_MANIFEST),
pending: path.resolve(configDir, WORKTREE_SEED_PENDING_MARKER),
complete: path.resolve(configDir, WORKTREE_SEED_COMPLETE_MARKER),
lock: path.resolve(configDir, WORKTREE_SEED_LOCK_MARKER),
+551 -52
View File
@@ -7,6 +7,7 @@ import {
readdirSync,
readFileSync,
readlinkSync,
renameSync,
rmSync,
statSync,
symlinkSync,
@@ -24,8 +25,11 @@ import { and, eq, inArray, sql } from "drizzle-orm";
import {
applyPendingMigrations,
agents,
authAccounts,
authUsers,
assets,
companies,
companyMemberships,
createDb,
documentRevisions,
documents,
@@ -39,6 +43,7 @@ import {
issueComments,
issueDocuments,
issues,
instanceUserRoles,
projectWorkspaces,
projects,
routines,
@@ -65,13 +70,16 @@ import {
formatShellExports,
generateWorktreeColor,
isWorktreeSeedMode,
WORKTREE_SEED_PHASES,
resolveSuggestedWorktreeName,
resolveWorktreeSeedPlan,
resolveWorktreeSeedMarkerPaths,
resolveWorktreeLocalPaths,
sanitizeWorktreeInstanceId,
type WorktreeSeedPlan,
type WorktreeSeedManifest,
type WorktreeSeedMode,
type WorktreeSeedPhase,
type WorktreeLocalPaths,
} from "./worktree-lib.js";
import {
@@ -137,6 +145,7 @@ type WorktreeReseedOptions = {
preserveLiveWork?: boolean;
yes?: boolean;
allowLiveTarget?: boolean;
backupTarget?: boolean;
};
type WorktreeRepairOptions = {
@@ -197,6 +206,8 @@ type CopiedGitHooksResult = {
type SeedWorktreeDatabaseResult = {
backupSummary: string;
snapshotAt: string;
migrationRevision: string;
pausedScheduledRoutines: number;
executionQuarantine: SeededWorktreeExecutionQuarantineSummary;
reboundWorkspaces: Array<{
@@ -204,28 +215,26 @@ type SeedWorktreeDatabaseResult = {
fromCwd: string;
toCwd: string;
}>;
validation: WorktreeSeedValidationSummary;
};
type WorktreeSeedPendingMarker = {
version: 1;
state: "pending";
sourceConfigPath: string;
seedMode: "minimal";
createdAt: string;
};
type WorktreeSeedCompleteMarker = {
version: 1;
state: "complete";
seedMode: WorktreeSeedMode;
completedAt: string;
export type WorktreeSeedValidationSummary = {
authUserCount: number;
credentialAccountCount: number;
instanceAdminCount: number;
activeMembershipCount: number;
companyCount: number;
issueCount: number;
representativeCompanyId: string;
representativeIssueId: string;
migrationRevision: string;
};
type SeedWorktreeDatabase = typeof seedWorktreeDatabase;
export type EnsureWorktreeSeededResult = {
seeded: boolean;
reason: "seeded" | "complete_marker" | "legacy_unmarked";
reason: "seeded" | "verified_manifest" | "complete_marker" | "legacy_unmarked";
details?: SeedWorktreeDatabaseResult;
};
@@ -1393,6 +1402,140 @@ export async function quarantineSeededWorktreeExecutionState(
}
}
type WorktreeSeedValidationExpectation = {
adminUserId: string;
representativeCompanyId: string;
representativeIssueId: string;
};
async function inspectVerifiedSeedDatabase(
connectionString: string,
expected?: WorktreeSeedValidationExpectation,
): Promise<{ summary: WorktreeSeedValidationSummary; expectation: WorktreeSeedValidationExpectation }> {
const migrationState = await inspectMigrations(connectionString);
if (migrationState.status !== "upToDate") {
throw new Error(
`Migration journal is not current (${migrationState.pendingMigrations.length} pending migration(s)).`,
);
}
const migrationRevision = migrationState.appliedMigrations.at(-1);
if (!migrationRevision) {
throw new Error("Migration journal has no applied revision.");
}
const db = createDb(connectionString);
try {
const [counts] = await db
.select({
authUserCount: sql<number>`count(distinct ${authUsers.id})::int`,
credentialAccountCount: sql<number>`count(distinct ${authAccounts.id})::int`,
instanceAdminCount: sql<number>`count(distinct ${instanceUserRoles.userId})::int`,
activeMembershipCount: sql<number>`count(distinct ${companyMemberships.id})::int`,
companyCount: sql<number>`count(distinct ${companies.id})::int`,
issueCount: sql<number>`count(distinct ${issues.id})::int`,
})
.from(authUsers)
.leftJoin(authAccounts, eq(authAccounts.userId, authUsers.id))
.leftJoin(
instanceUserRoles,
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
)
.leftJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.status, "active"),
),
)
.leftJoin(companies, eq(companies.id, companyMemberships.companyId))
.leftJoin(issues, eq(issues.companyId, companies.id));
const admin = await db
.select({ userId: authUsers.id })
.from(authUsers)
.innerJoin(
instanceUserRoles,
and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")),
)
.innerJoin(
authAccounts,
and(
eq(authAccounts.userId, authUsers.id),
sql`length(trim(${authAccounts.providerId})) > 0`,
sql`length(trim(${authAccounts.accountId})) > 0`,
),
)
.innerJoin(
companyMemberships,
and(
eq(companyMemberships.principalType, "user"),
eq(companyMemberships.principalId, authUsers.id),
eq(companyMemberships.status, "active"),
),
)
.where(expected ? eq(authUsers.id, expected.adminUserId) : undefined)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!admin) {
throw new Error(
"No auth user has a non-empty credential account, instance-admin role, and active company membership.",
);
}
const representative = await db
.select({ companyId: companies.id, issueId: issues.id })
.from(companies)
.innerJoin(issues, eq(issues.companyId, companies.id))
.where(
expected
? and(
eq(companies.id, expected.representativeCompanyId),
eq(issues.id, expected.representativeIssueId),
)
: undefined,
)
.limit(1)
.then((rows) => rows[0] ?? null);
if (!representative) {
throw new Error("No representative cloned company and issue pair is readable.");
}
const summary: WorktreeSeedValidationSummary = {
authUserCount: counts?.authUserCount ?? 0,
credentialAccountCount: counts?.credentialAccountCount ?? 0,
instanceAdminCount: counts?.instanceAdminCount ?? 0,
activeMembershipCount: counts?.activeMembershipCount ?? 0,
companyCount: counts?.companyCount ?? 0,
issueCount: counts?.issueCount ?? 0,
representativeCompanyId: representative.companyId,
representativeIssueId: representative.issueId,
migrationRevision,
};
if (
summary.authUserCount < 1
|| summary.credentialAccountCount < 1
|| summary.instanceAdminCount < 1
|| summary.activeMembershipCount < 1
|| summary.companyCount < 1
|| summary.issueCount < 1
) {
throw new Error("Seed validation found an incomplete auth, membership, company, or issue shape.");
}
return {
summary,
expectation: {
adminUserId: admin.userId,
representativeCompanyId: representative.companyId,
representativeIssueId: representative.issueId,
},
};
} finally {
await db.$client?.end?.({ timeout: 5 }).catch(() => undefined);
}
}
async function seedWorktreeDatabase(input: {
sourceConfigPath: string;
sourceConfig: PaperclipConfig;
@@ -1401,6 +1544,7 @@ async function seedWorktreeDatabase(input: {
instanceId: string;
seedMode: WorktreeSeedMode;
preserveLiveWork?: boolean;
onPhase?: (phase: WorktreeSeedPhase, status: "started" | "succeeded", message?: string) => void;
}): Promise<SeedWorktreeDatabaseResult> {
const seedPlan = resolveWorktreeSeedPlan(input.seedMode);
const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath);
@@ -1428,6 +1572,16 @@ async function seedWorktreeDatabase(input: {
sourceEnvEntries,
sourceHandle?.port,
);
input.onPhase?.("source_validation", "started");
const sourceValidation = await inspectVerifiedSeedDatabase(sourceConnectionString);
input.onPhase?.(
"source_validation",
"succeeded",
`Validated ${sourceValidation.summary.companyCount} company record(s) and ${sourceValidation.summary.issueCount} issue record(s).`,
);
const snapshotAt = new Date().toISOString();
input.onPhase?.("snapshot", "started");
const backup = await runDatabaseBackup({
connectionString: sourceConnectionString,
backupDir: path.resolve(input.targetPaths.backupDir, "seed"),
@@ -1438,6 +1592,7 @@ async function seedWorktreeDatabase(input: {
excludeTables: seedPlan.excludedTables,
nullifyColumns: seedPlan.nullifyColumns,
});
input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`);
targetHandle = await ensureEmbeddedPostgres(
input.targetConfig.database.embeddedPostgresDataDir,
@@ -1445,27 +1600,56 @@ async function seedWorktreeDatabase(input: {
);
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
input.onPhase?.("restore", "started");
await resetPostgresDatabase(adminConnectionString, "paperclip");
const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`;
await runDatabaseRestore({
connectionString: targetConnectionString,
backupFile: backup.backupFile,
});
input.onPhase?.("restore", "succeeded");
input.onPhase?.("migrations", "started");
await applyPendingMigrations(targetConnectionString);
input.onPhase?.("migrations", "succeeded");
input.onPhase?.("execution_quarantine", "started");
const executionQuarantine = input.preserveLiveWork
? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }
: await quarantineSeededWorktreeExecutionState(targetConnectionString);
input.onPhase?.(
"execution_quarantine",
"succeeded",
input.preserveLiveWork
? "Preserved copied live work by explicit request."
: formatSeededWorktreeExecutionQuarantineSummary(executionQuarantine),
);
input.onPhase?.("routine_pause", "started");
const pausedScheduledRoutines = await pauseSeededScheduledRoutines(targetConnectionString);
input.onPhase?.("routine_pause", "succeeded", `Paused ${pausedScheduledRoutines} scheduled routine(s).`);
input.onPhase?.("workspace_rebind", "started");
const reboundWorkspaces = await rebindSeededProjectWorkspaces({
targetConnectionString,
currentCwd: input.targetPaths.cwd,
});
input.onPhase?.("workspace_rebind", "succeeded", `Rebound ${reboundWorkspaces.length} workspace path(s).`);
input.onPhase?.("post_restore_validation", "started");
const targetValidation = await inspectVerifiedSeedDatabase(
targetConnectionString,
sourceValidation.expectation,
);
input.onPhase?.(
"post_restore_validation",
"succeeded",
`Validated migration ${targetValidation.summary.migrationRevision}.`,
);
return {
backupSummary: formatDatabaseBackupResult(backup),
snapshotAt,
migrationRevision: targetValidation.summary.migrationRevision,
pausedScheduledRoutines,
executionQuarantine,
reboundWorkspaces,
validation: targetValidation.summary,
};
} finally {
if (targetHandle?.startedByThisProcess) {
@@ -1477,46 +1661,184 @@ async function seedWorktreeDatabase(input: {
}
}
function writeWorktreeSeedMarker(
filePath: string,
marker: WorktreeSeedPendingMarker | WorktreeSeedCompleteMarker,
): void {
const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32;
const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512;
const activeSeedInterruptHandlers = new Map<string, (signal: NodeJS.Signals) => void>();
function dispatchSeedInterruption(signal: NodeJS.Signals): void {
for (const handler of activeSeedInterruptHandlers.values()) {
try {
handler(signal);
} catch {
// Continue terminalizing the other active manifests before exiting.
}
}
process.exit(signal === "SIGINT" ? 130 : 143);
}
const dispatchSeedSigint = () => dispatchSeedInterruption("SIGINT");
const dispatchSeedSigterm = () => dispatchSeedInterruption("SIGTERM");
function registerSeedInterruptHandler(handler: (signal: NodeJS.Signals) => void): () => void {
const id = randomUUID();
if (activeSeedInterruptHandlers.size === 0) {
process.once("SIGINT", dispatchSeedSigint);
process.once("SIGTERM", dispatchSeedSigterm);
}
activeSeedInterruptHandlers.set(id, handler);
return () => {
activeSeedInterruptHandlers.delete(id);
if (activeSeedInterruptHandlers.size === 0) {
process.off("SIGINT", dispatchSeedSigint);
process.off("SIGTERM", dispatchSeedSigterm);
}
};
}
type LegacyWorktreeSeedPendingMarker = {
version: 1;
state: "pending";
sourceConfigPath: string;
};
function resolveSeedInstanceId(configPath: string): string {
const envEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(configPath));
return nonEmpty(envEntries.PAPERCLIP_INSTANCE_ID)
?? sanitizeWorktreeInstanceId(path.basename(path.dirname(path.resolve(configPath))));
}
function writeWorktreeSeedManifest(filePath: string, manifest: WorktreeSeedManifest): void {
mkdirSync(path.dirname(filePath), { recursive: true });
writeFileSync(filePath, `${JSON.stringify(marker, null, 2)}\n`, { mode: 0o600 });
const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`;
writeFileSync(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
renameSync(temporaryPath, filePath);
}
export function readWorktreeSeedManifest(configPath: string): WorktreeSeedManifest | null {
const manifestPath = resolveWorktreeSeedMarkerPaths(configPath).manifest;
if (!existsSync(manifestPath)) return null;
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
} catch (error) {
throw new Error(
`Invalid worktree seed manifest at ${manifestPath}: ${error instanceof Error ? error.message : String(error)}`,
);
}
const value = parsed as Partial<WorktreeSeedManifest>;
const diagnosticsValid = Array.isArray(value.diagnostics) && value.diagnostics.every((diagnostic) => (
diagnostic
&& typeof diagnostic === "object"
&& WORKTREE_SEED_PHASES.includes(diagnostic.phase)
&& ["started", "succeeded", "failed"].includes(diagnostic.status)
&& typeof diagnostic.at === "string"
&& (diagnostic.message === undefined || typeof diagnostic.message === "string")
));
const verifiedTerminalValid = value.state !== "verified" || (
value.phase === "complete"
&& typeof value.snapshotAt === "string"
&& value.snapshotAt.length > 0
&& typeof value.migrationRevision === "string"
&& value.migrationRevision.length > 0
&& typeof value.startedAt === "string"
&& typeof value.finishedAt === "string"
&& value.diagnostics?.some((diagnostic) => (
diagnostic.phase === "complete" && diagnostic.status === "succeeded"
)) === true
);
if (
!value
|| typeof value !== "object"
|| value.version !== 2
|| !value.source
|| typeof value.source.instanceId !== "string"
|| typeof value.source.configPath !== "string"
|| typeof value.targetInstanceId !== "string"
|| value.targetInstanceId.length === 0
|| !isWorktreeSeedMode(String(value.seedMode ?? ""))
|| !WORKTREE_SEED_PHASES.includes(value.phase as WorktreeSeedPhase)
|| !["pending", "running", "verified", "failed"].includes(String(value.state ?? ""))
|| typeof value.attemptId !== "string"
|| value.attemptId.length === 0
|| !diagnosticsValid
|| !verifiedTerminalValid
) {
throw new Error(`Invalid worktree seed manifest at ${manifestPath}.`);
}
return value as WorktreeSeedManifest;
}
export function markWorktreeSeedPending(input: {
configPath: string;
sourceConfigPath: string;
now?: Date;
}): void {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
rmSync(markers.complete, { force: true });
writeWorktreeSeedMarker(markers.pending, {
version: 1,
state: "pending",
sourceConfigPath: path.resolve(input.sourceConfigPath),
seedMode: "minimal",
createdAt: (input.now ?? new Date()).toISOString(),
});
}
export function markWorktreeSeedComplete(input: {
configPath: string;
targetInstanceId?: string;
seedMode?: WorktreeSeedMode;
now?: Date;
}): void {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
writeWorktreeSeedMarker(markers.complete, {
version: 1,
state: "complete",
const at = (input.now ?? new Date()).toISOString();
writeWorktreeSeedManifest(markers.manifest, {
version: 2,
source: {
instanceId: resolveSeedInstanceId(input.sourceConfigPath),
configPath: path.resolve(input.sourceConfigPath),
},
snapshotAt: null,
seedMode: input.seedMode ?? "minimal",
completedAt: (input.now ?? new Date()).toISOString(),
migrationRevision: null,
targetInstanceId: input.targetInstanceId ?? resolveSeedInstanceId(input.configPath),
phase: "pending",
state: "pending",
attemptId: randomUUID(),
startedAt: null,
finishedAt: null,
diagnostics: [{ phase: "pending", status: "succeeded", at }],
});
// New manifests are authoritative. Legacy files are removed so no caller can
// mistake a stale binary marker for current verified seed state.
rmSync(markers.complete, { force: true });
rmSync(markers.pending, { force: true });
}
function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMarker {
function updateWorktreeSeedManifest(input: {
configPath: string;
phase: WorktreeSeedPhase;
status: "started" | "succeeded" | "failed";
state?: WorktreeSeedManifest["state"];
message?: string;
snapshotAt?: string | null;
migrationRevision?: string | null;
now?: Date;
}): WorktreeSeedManifest {
const markers = resolveWorktreeSeedMarkerPaths(input.configPath);
const current = readWorktreeSeedManifest(input.configPath);
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
const at = (input.now ?? new Date()).toISOString();
const nextState = input.state ?? current.state;
const diagnostic = {
phase: input.phase,
status: input.status,
at,
...(input.message
? { message: input.message.slice(0, WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT) }
: {}),
};
const next: WorktreeSeedManifest = {
...current,
phase: input.phase,
state: nextState,
snapshotAt: input.snapshotAt === undefined ? current.snapshotAt : input.snapshotAt,
migrationRevision:
input.migrationRevision === undefined ? current.migrationRevision : input.migrationRevision,
startedAt: current.startedAt ?? (input.status === "started" ? at : null),
finishedAt: nextState === "verified" || nextState === "failed" ? at : null,
diagnostics: [...current.diagnostics, diagnostic].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
};
writeWorktreeSeedManifest(markers.manifest, next);
return next;
}
function readLegacyWorktreeSeedPendingMarker(filePath: string): LegacyWorktreeSeedPendingMarker {
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(filePath, "utf8"));
@@ -1537,7 +1859,7 @@ function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMar
throw new Error(`Invalid worktree seed-pending marker at ${filePath}.`);
}
return parsed as WorktreeSeedPendingMarker;
return parsed as LegacyWorktreeSeedPendingMarker;
}
const WORKTREE_SEED_LOCK_POLL_MS = 50;
@@ -1631,6 +1953,117 @@ async function acquireWorktreeSeedLock(lockPath: string): Promise<() => Promise<
}
}
function startWorktreeSeedAttempt(configPath: string, now = new Date()): WorktreeSeedManifest {
const markers = resolveWorktreeSeedMarkerPaths(configPath);
const current = readWorktreeSeedManifest(configPath);
if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`);
const at = now.toISOString();
const next: WorktreeSeedManifest = {
...current,
state: "running",
phase: "pending",
attemptId: randomUUID(),
snapshotAt: null,
migrationRevision: null,
startedAt: at,
finishedAt: null,
diagnostics: [
...current.diagnostics,
{ phase: "pending" as const, status: "started" as const, at },
].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT),
};
writeWorktreeSeedManifest(markers.manifest, next);
return next;
}
async function runVerifiedWorktreeSeed(input: {
configPath: string;
sourceConfigPath: string;
sourceConfig: PaperclipConfig;
targetConfig: PaperclipConfig;
targetPaths: WorktreeLocalPaths;
instanceId: string;
seedMode: WorktreeSeedMode;
preserveLiveWork?: boolean;
seedDatabase: SeedWorktreeDatabase;
}): Promise<SeedWorktreeDatabaseResult> {
let activePhase: WorktreeSeedPhase = "pending";
const previous = readWorktreeSeedManifest(input.configPath);
if (previous?.state === "running") {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: previous.phase,
status: "failed",
state: "failed",
message: "The previous seed attempt ended without a terminal result.",
});
}
startWorktreeSeedAttempt(input.configPath);
const unregisterInterruption = registerSeedInterruptHandler((signal) => {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: activePhase,
status: "failed",
state: "failed",
message: `Seed interrupted by ${signal} during ${activePhase}.`,
});
});
try {
const details = await input.seedDatabase({
sourceConfigPath: input.sourceConfigPath,
sourceConfig: input.sourceConfig,
targetConfig: input.targetConfig,
targetPaths: input.targetPaths,
instanceId: input.instanceId,
seedMode: input.seedMode,
preserveLiveWork: input.preserveLiveWork,
onPhase: (phase, status, message) => {
activePhase = phase;
updateWorktreeSeedManifest({
configPath: input.configPath,
phase,
status,
state: "running",
message,
...(phase === "snapshot" && status === "started"
? { snapshotAt: new Date().toISOString() }
: {}),
});
},
});
if (!details.snapshotAt || !details.migrationRevision || !details.validation) {
throw new Error("Seed implementation returned without required validation evidence.");
}
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: "complete",
status: "succeeded",
state: "verified",
snapshotAt: details.snapshotAt,
migrationRevision: details.migrationRevision,
message:
`Verified ${details.validation.companyCount} company record(s), `
+ `${details.validation.issueCount} issue record(s), auth, admin, membership, and migration state.`,
});
return details;
} catch (error) {
updateWorktreeSeedManifest({
configPath: input.configPath,
phase: activePhase,
status: "failed",
state: "failed",
// Do not persist the underlying error: database/driver errors may contain
// connection credentials. The CLI still returns the exact error to its caller.
message: `Seed failed during ${activePhase}.`,
});
throw error;
} finally {
unregisterInterruption();
}
}
export async function ensureWorktreeSeeded(
opts: WorktreeEnsureSeededOptions = {},
dependencies: { seedDatabase?: SeedWorktreeDatabase } = {},
@@ -1642,24 +2075,36 @@ export async function ensureWorktreeSeeded(
try {
// These checks deliberately happen under the cross-process lock. A second
// service process waits for the first seed transaction, then observes the
// complete marker instead of cloning the same database concurrently.
if (existsSync(markers.complete)) {
// verified manifest instead of cloning the same database concurrently.
let manifest = readWorktreeSeedManifest(configPath);
if (manifest?.state === "verified") {
return { seeded: false, reason: "verified_manifest" };
}
if (!manifest && existsSync(markers.complete)) {
return { seeded: false, reason: "complete_marker" };
}
if (!existsSync(markers.pending)) {
if (!manifest && existsSync(markers.pending)) {
const legacyPending = readLegacyWorktreeSeedPendingMarker(markers.pending);
markWorktreeSeedPending({
configPath,
sourceConfigPath: legacyPending.sourceConfigPath,
seedMode: "minimal",
});
manifest = readWorktreeSeedManifest(configPath);
}
if (!manifest) {
// Worktrees created before lazy seeding shipped were seeded eagerly and
// have neither marker. Preserve that compatibility without re-cloning.
return { seeded: false, reason: "legacy_unmarked" };
}
const pending = readWorktreeSeedPendingMarker(markers.pending);
const sourceConfigPath = opts.fromConfig || opts.fromDataDir || opts.fromInstance
? resolveSourceConfigPath({
fromConfig: opts.fromConfig,
fromDataDir: opts.fromDataDir,
fromInstance: opts.fromInstance,
})
: path.resolve(pending.sourceConfigPath);
: path.resolve(manifest.source.configPath);
if (path.resolve(sourceConfigPath) === path.resolve(configPath)) {
throw new Error(
@@ -1679,16 +2124,17 @@ export async function ensureWorktreeSeeded(
const targetRoot = path.dirname(path.dirname(configPath));
const targetPaths = resolveWorktreeReseedTargetPaths({ configPath, rootPath: targetRoot });
const seedDatabase = dependencies.seedDatabase ?? seedWorktreeDatabase;
const details = await seedDatabase({
const details = await runVerifiedWorktreeSeed({
configPath,
sourceConfigPath,
sourceConfig,
targetConfig,
targetPaths,
instanceId: targetPaths.instanceId,
seedMode: "minimal",
seedMode: manifest.seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase,
});
markWorktreeSeedComplete({ configPath });
return { seeded: true, reason: "seeded", details };
} finally {
await releaseLock();
@@ -1762,6 +2208,8 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
markWorktreeSeedPending({
configPath: paths.configPath,
sourceConfigPath,
targetInstanceId: instanceId,
seedMode,
});
const sourceEnvEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(sourceConfigPath));
const existingAgentJwtSecret =
@@ -1790,8 +2238,11 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
}
const spinner = p.spinner();
spinner.start(`Seeding isolated worktree database from source instance (${seedMode})...`);
const markers = resolveWorktreeSeedMarkerPaths(paths.configPath);
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
try {
const seeded = await seedWorktreeDatabase({
const seeded = await runVerifiedWorktreeSeed({
configPath: paths.configPath,
sourceConfigPath,
sourceConfig,
targetConfig,
@@ -1799,16 +2250,18 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise<void> {
instanceId,
seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase: seedWorktreeDatabase,
});
seedSummary = seeded.backupSummary;
seedExecutionQuarantineSummary = seeded.executionQuarantine;
pausedScheduledRoutineCount = seeded.pausedScheduledRoutines;
reboundWorkspaceSummary = seeded.reboundWorkspaces;
markWorktreeSeedComplete({ configPath: paths.configPath, seedMode });
spinner.stop(`Seeded isolated worktree database (${seedMode}).`);
} catch (error) {
spinner.stop(pc.red("Failed to seed worktree database."));
throw error;
} finally {
await releaseSeedLock();
}
}
@@ -3480,6 +3933,34 @@ export async function worktreeMergeHistoryCommand(sourceArg: string | undefined,
}
}
async function backupWorktreeReseedTarget(input: {
targetConfig: PaperclipConfig;
targetPaths: WorktreeLocalPaths;
}): Promise<string> {
if (input.targetConfig.database.mode !== "embedded-postgres") {
throw new Error("Managed worktree repair requires an embedded PostgreSQL target.");
}
const targetHandle = await ensureEmbeddedPostgres(
input.targetConfig.database.embeddedPostgresDataDir,
input.targetConfig.database.embeddedPostgresPort,
);
try {
const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`;
await ensurePostgresDatabase(adminConnectionString, "paperclip");
const result = await runDatabaseBackup({
connectionString: `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`,
backupDir: path.resolve(input.targetPaths.backupDir, "repair"),
retention: { dailyDays: 30, weeklyWeeks: 12, monthlyMonths: 12 },
filenamePrefix: `${input.targetPaths.instanceId}-pre-repair`,
backupEngine: "auto",
includeMigrationJournal: true,
});
return formatDatabaseBackupResult(result);
} finally {
if (targetHandle.startedByThisProcess) await targetHandle.stop();
}
}
async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
const seedMode = opts.seedMode ?? "full";
if (!isWorktreeSeedMode(seedMode)) {
@@ -3535,8 +4016,23 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
const spinner = p.spinner();
spinner.start(`Reseeding ${targetEndpoint.label} from ${source.label} (${seedMode})...`);
const markers = resolveWorktreeSeedMarkerPaths(targetEndpoint.configPath);
mkdirSync(path.dirname(markers.lock), { recursive: true });
const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock);
try {
const seeded = await seedWorktreeDatabase({
let targetBackupSummary: string | null = null;
if (opts.backupTarget) {
targetBackupSummary = await backupWorktreeReseedTarget({ targetConfig, targetPaths });
p.log.message(pc.dim(`Recoverable pre-repair backup: ${targetBackupSummary}`));
}
markWorktreeSeedPending({
configPath: targetEndpoint.configPath,
sourceConfigPath: source.configPath,
targetInstanceId: targetPaths.instanceId,
seedMode,
});
const seeded = await runVerifiedWorktreeSeed({
configPath: targetEndpoint.configPath,
sourceConfigPath: source.configPath,
sourceConfig,
targetConfig,
@@ -3544,8 +4040,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
instanceId: targetPaths.instanceId,
seedMode,
preserveLiveWork: opts.preserveLiveWork,
seedDatabase: seedWorktreeDatabase,
});
markWorktreeSeedComplete({ configPath: targetEndpoint.configPath, seedMode });
spinner.stop(`Reseeded ${targetEndpoint.label} (${seedMode}).`);
p.log.message(pc.dim(`Source: ${source.configPath}`));
p.log.message(pc.dim(`Target: ${targetEndpoint.configPath}`));
@@ -3567,6 +4063,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
} catch (error) {
spinner.stop(pc.red("Failed to reseed worktree database."));
throw error;
} finally {
await releaseSeedLock();
}
}
@@ -3748,6 +4246,7 @@ export function registerWorktreeCommands(program: Command): void {
.option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
.option("--yes", "Skip the destructive confirmation prompt", false)
.option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false)
.option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false)
.action(worktreeReseedCommand);
worktree