diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index ad0edc4a464..d261b933173 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -8,13 +8,16 @@ import { eq } from "drizzle-orm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { agents, + authAccounts, authUsers, companies, + companyMemberships, createDb, executionWorkspaces, issueComments, issues, projectWorkspaces, + instanceUserRoles, projects, routines, routineTriggers, @@ -27,6 +30,7 @@ import { markWorktreeSeedPending, pauseSeededScheduledRoutines, quarantineSeededWorktreeExecutionState, + readWorktreeSeedManifest, readSourceAttachmentBody, rebindWorkspaceCwd, resolveSourceConfigPath, @@ -64,6 +68,88 @@ const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); const itEmbeddedPostgres = embeddedPostgresSupport.supported ? it : it.skip; const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; +function mockVerifiedSeedResult() { + return { + backupSummary: "snapshot.sql", + snapshotAt: "2026-08-18T00:00:00.000Z", + migrationRevision: "0142_test.sql", + pausedScheduledRoutines: 0, + executionQuarantine: { + disabledTimerHeartbeats: 0, + resetRunningAgents: 0, + quarantinedInProgressIssues: 0, + unassignedTodoIssues: 0, + unassignedReviewIssues: 0, + stoppedProjectWorkspaceRuntimes: 0, + stoppedExecutionWorkspaceRuntimes: 0, + stoppedRuntimeServices: 0, + }, + reboundWorkspaces: [], + validation: { + authUserCount: 1, + credentialAccountCount: 1, + instanceAdminCount: 1, + activeMembershipCount: 1, + companyCount: 1, + issueCount: 1, + representativeCompanyId: "00000000-0000-4000-8000-000000000001", + representativeIssueId: "00000000-0000-4000-8000-000000000002", + migrationRevision: "0142_test.sql", + }, + }; +} + +async function seedValidWorktreeSource(connectionString: string) { + const db = createDb(connectionString); + const companyId = randomUUID(); + const issueId = randomUUID(); + const now = new Date(); + await db.insert(authUsers).values({ + id: "user-existing", + email: "existing@paperclip.ing", + name: "Existing User", + emailVerified: true, + createdAt: now, + updatedAt: now, + }); + await db.insert(authAccounts).values({ + id: "credential-existing", + accountId: "existing@paperclip.ing", + providerId: "credential", + userId: "user-existing", + password: "fixture-password-hash", + createdAt: now, + updatedAt: now, + }); + await db.insert(instanceUserRoles).values({ + userId: "user-existing", + role: "instance_admin", + }); + await db.insert(companies).values({ + id: companyId, + name: "Seed Source", + issuePrefix: "SEED", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(companyMemberships).values({ + companyId, + principalType: "user", + principalId: "user-existing", + status: "active", + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Representative seed issue", + status: "backlog", + priority: "medium", + issueNumber: 1, + identifier: "SEED-1", + }); + await db.$client.end({ timeout: 5 }); + return { companyId, issueId }; +} + if (!embeddedPostgresSupport.supported) { console.warn( `Skipping embedded Postgres worktree CLI tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, @@ -400,7 +486,7 @@ describe("worktree helpers", () => { expect(full.nullifyColumns).toEqual({}); }); - it("ensure-seeded seeds once and fast-exits on the seed-complete marker", async () => { + it("ensure-seeded seeds once and fast-exits on the verified manifest", async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-")); try { const sourceConfigPath = path.join(tempRoot, "source", "config.json"); @@ -429,7 +515,7 @@ describe("worktree helpers", () => { markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath }); const seedDatabase = vi.fn().mockResolvedValue({ - backupSummary: "snapshot.sql", + ...mockVerifiedSeedResult(), pausedScheduledRoutines: 2, executionQuarantine: { disabledTimerHeartbeats: 1, @@ -441,7 +527,6 @@ describe("worktree helpers", () => { stoppedExecutionWorkspaceRuntimes: 0, stoppedRuntimeServices: 0, }, - reboundWorkspaces: [], }); await expect( @@ -449,7 +534,7 @@ describe("worktree helpers", () => { ).resolves.toMatchObject({ seeded: true, reason: "seeded" }); await expect( ensureWorktreeSeeded({ config: targetConfigPath }, { seedDatabase }), - ).resolves.toEqual({ seeded: false, reason: "complete_marker" }); + ).resolves.toEqual({ seeded: false, reason: "verified_manifest" }); expect(seedDatabase).toHaveBeenCalledTimes(1); expect(seedDatabase).toHaveBeenCalledWith(expect.objectContaining({ @@ -458,7 +543,14 @@ describe("worktree helpers", () => { instanceId: "ensure-seeded-test", })); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false); - expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(true); + expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false); + expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({ + version: 2, + state: "verified", + phase: "complete", + migrationRevision: "0142_test.sql", + targetInstanceId: "ensure-seeded-test", + }); } finally { fs.rmSync(tempRoot, { recursive: true, force: true }); } @@ -499,7 +591,11 @@ describe("worktree helpers", () => { ), ).rejects.toThrow("seed failed"); - expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(true); + expect(readWorktreeSeedManifest(targetConfigPath)).toMatchObject({ + state: "failed", + phase: "pending", + }); + expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-pending"))).toBe(false); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed-complete"))).toBe(false); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false); } finally { @@ -537,21 +633,7 @@ describe("worktree helpers", () => { const seedDatabase = vi.fn(async () => { await new Promise((resolve) => setTimeout(resolve, 100)); - return { - backupSummary: "snapshot.sql", - pausedScheduledRoutines: 0, - executionQuarantine: { - disabledTimerHeartbeats: 0, - resetRunningAgents: 0, - quarantinedInProgressIssues: 0, - unassignedTodoIssues: 0, - unassignedReviewIssues: 0, - stoppedProjectWorkspaceRuntimes: 0, - stoppedExecutionWorkspaceRuntimes: 0, - stoppedRuntimeServices: 0, - }, - reboundWorkspaces: [], - }; + return mockVerifiedSeedResult(); }); const results = await Promise.all([ @@ -561,7 +643,7 @@ describe("worktree helpers", () => { expect(results).toEqual(expect.arrayContaining([ expect.objectContaining({ seeded: true, reason: "seeded" }), - { seeded: false, reason: "complete_marker" }, + { seeded: false, reason: "verified_manifest" }, ])); expect(seedDatabase).toHaveBeenCalledTimes(1); expect(fs.existsSync(path.join(targetRoot, ".paperclip", "seed.lock"))).toBe(false); @@ -570,6 +652,58 @@ describe("worktree helpers", () => { } }); + it("records an interrupted phase before retrying to a verified terminal state", async () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-interrupted-seed-")); + try { + const sourceConfigPath = path.join(tempRoot, "source", "config.json"); + const targetRoot = path.join(tempRoot, "worktree"); + const targetConfigPath = path.join(targetRoot, ".paperclip", "config.json"); + const targetPaths = resolveWorktreeLocalPaths({ + cwd: targetRoot, + homeDir: path.join(tempRoot, "worktree-home"), + instanceId: "interrupted-seed", + }); + const sourceConfig = buildSourceConfig(); + const targetConfig = buildWorktreeConfig({ + sourceConfig, + paths: targetPaths, + serverPort: 3196, + databasePort: 54996, + }); + fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true }); + fs.mkdirSync(path.dirname(targetConfigPath), { recursive: true }); + fs.writeFileSync(sourceConfigPath, `${JSON.stringify(sourceConfig)}\n`); + fs.writeFileSync(targetConfigPath, `${JSON.stringify(targetConfig)}\n`); + fs.writeFileSync( + path.join(targetRoot, ".paperclip", ".env"), + `PAPERCLIP_HOME=${targetPaths.homeDir}\nPAPERCLIP_INSTANCE_ID=${targetPaths.instanceId}\n`, + ); + markWorktreeSeedPending({ configPath: targetConfigPath, sourceConfigPath }); + const interrupted = readWorktreeSeedManifest(targetConfigPath)!; + fs.writeFileSync( + path.join(targetRoot, ".paperclip", "seed-manifest.json"), + `${JSON.stringify({ ...interrupted, state: "running", phase: "restore" }, null, 2)}\n`, + ); + + await expect(ensureWorktreeSeeded( + { config: targetConfigPath }, + { seedDatabase: vi.fn().mockResolvedValue(mockVerifiedSeedResult()) }, + )).resolves.toMatchObject({ seeded: true, reason: "seeded" }); + + const verified = readWorktreeSeedManifest(targetConfigPath)!; + expect(verified.state).toBe("verified"); + expect(verified.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ + phase: "restore", + status: "failed", + message: "The previous seed attempt ended without a terminal result.", + }), + ])); + } finally { + fs.rmSync(tempRoot, { recursive: true, force: true }); + } + }); + it("fails closed instead of racing to reclaim a stale seed lock", async () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-ensure-seeded-stale-lock-")); try { @@ -983,15 +1117,7 @@ describe("worktree helpers", () => { const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-auth-source-"); try { - const sourceDbClient = createDb(sourceDb.connectionString); - await sourceDbClient.insert(authUsers).values({ - id: "user-existing", - email: "existing@paperclip.ing", - name: "Existing User", - emailVerified: true, - createdAt: new Date(), - updatedAt: new Date(), - }); + await seedValidWorktreeSource(sourceDb.connectionString); fs.mkdirSync(path.dirname(sourceKeyPath), { recursive: true }); fs.mkdirSync(worktreeRoot, { recursive: true }); @@ -1028,6 +1154,18 @@ describe("worktree helpers", () => { const targetConfig = JSON.parse( fs.readFileSync(path.join(worktreeRoot, ".paperclip", "config.json"), "utf8"), ) as PaperclipConfig; + const manifestText = fs.readFileSync( + path.join(worktreeRoot, ".paperclip", "seed-manifest.json"), + "utf8", + ); + expect(JSON.parse(manifestText)).toMatchObject({ + version: 2, + seedMode: "minimal", + state: "verified", + phase: "complete", + }); + expect(manifestText).not.toContain("fixture-password-hash"); + expect(manifestText).not.toContain("source-master-key"); const { default: EmbeddedPostgres } = await import("embedded-postgres"); const targetPg = new EmbeddedPostgres({ databaseDir: targetConfig.database.embeddedPostgresDataDir, @@ -1284,9 +1422,8 @@ describe("worktree helpers", () => { const originalCwd = process.cwd(); const originalPaperclipConfig = process.env.PAPERCLIP_CONFIG; const currentDatabaseReservation = await reserveTestPort(); - const sourceDatabaseReservation = await reserveTestPort(); const currentDatabasePort = currentDatabaseReservation.port; - const sourceDatabasePort = sourceDatabaseReservation.port; + const sourceDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-reseed-source-"); try { fs.mkdirSync(path.dirname(currentPaths.configPath), { recursive: true }); @@ -1301,15 +1438,28 @@ describe("worktree helpers", () => { serverPort: 3114, databasePort: currentDatabasePort, }); - const sourceConfig = buildWorktreeConfig({ - sourceConfig: buildSourceConfig(), - paths: sourcePaths, - serverPort: 3200, - databasePort: sourceDatabasePort, - }); + const sourceConfig = buildSourceConfig(); + sourceConfig.database = { + mode: "postgres", + embeddedPostgresDataDir: sourcePaths.embeddedPostgresDataDir, + embeddedPostgresPort: 54329, + backup: { + enabled: true, + intervalMinutes: 60, + retentionDays: 30, + dir: sourcePaths.backupDir, + }, + connectionString: sourceDb.connectionString, + }; + sourceConfig.logging.logDir = sourcePaths.logDir; + sourceConfig.storage.localDisk.baseDir = sourcePaths.storageDir; + sourceConfig.secrets.localEncrypted.keyFilePath = sourcePaths.secretsKeyFilePath; + await seedValidWorktreeSource(sourceDb.connectionString); fs.writeFileSync(currentPaths.configPath, JSON.stringify(currentConfig, null, 2), "utf8"); fs.writeFileSync(sourcePaths.configPath, JSON.stringify(sourceConfig, null, 2), "utf8"); fs.writeFileSync(sourcePaths.secretsKeyFilePath, "source-secret", "utf8"); + const worktreeSentinelPath = path.join(repoRoot, "user-worktree-file.txt"); + fs.writeFileSync(worktreeSentinelPath, "preserve me", "utf8"); fs.writeFileSync( currentPaths.envPath, [ @@ -1325,11 +1475,11 @@ describe("worktree helpers", () => { process.chdir(repoRoot); await currentDatabaseReservation.release(); - await sourceDatabaseReservation.release(); await worktreeReseedCommand({ fromConfig: sourcePaths.configPath, yes: true, + backupTarget: true, }); const rewrittenConfig = JSON.parse(fs.readFileSync(currentPaths.configPath, "utf8")); @@ -1341,9 +1491,13 @@ describe("worktree helpers", () => { expect(rewrittenEnv).toContain(`PAPERCLIP_INSTANCE_ID=${currentInstanceId}`); expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_NAME=existing-name"); expect(rewrittenEnv).toContain("PAPERCLIP_WORKTREE_COLOR=\"#112233\""); + expect(fs.readFileSync(worktreeSentinelPath, "utf8")).toBe("preserve me"); + expect( + fs.readdirSync(path.join(currentPaths.backupDir, "repair")).some((name) => name.endsWith(".sql.gz")), + ).toBe(true); } finally { await currentDatabaseReservation.release(); - await sourceDatabaseReservation.release(); + await sourceDb.cleanup(); process.chdir(originalCwd); if (originalPaperclipConfig === undefined) { delete process.env.PAPERCLIP_CONFIG; diff --git a/cli/src/commands/worktree-lib.ts b/cli/src/commands/worktree-lib.ts index 7da2857c5d6..4cdf7ef5364 100644 --- a/cli/src/commands/worktree-lib.ts +++ b/cli/src/commands/worktree-lib.ts @@ -5,12 +5,52 @@ import { expandHomePrefix } from "../config/home.js"; export const DEFAULT_WORKTREE_HOME = "~/.paperclip-worktrees"; export const WORKTREE_SEED_MODES = ["minimal", "full"] as const; +export const WORKTREE_SEED_MANIFEST = "seed-manifest.json"; export const WORKTREE_SEED_PENDING_MARKER = "seed-pending"; export const WORKTREE_SEED_COMPLETE_MARKER = "seed-complete"; export const WORKTREE_SEED_LOCK_MARKER = "seed.lock"; export type WorktreeSeedMode = (typeof WORKTREE_SEED_MODES)[number]; +export const WORKTREE_SEED_PHASES = [ + "pending", + "source_validation", + "snapshot", + "restore", + "migrations", + "execution_quarantine", + "routine_pause", + "workspace_rebind", + "post_restore_validation", + "complete", +] as const; + +export type WorktreeSeedPhase = (typeof WORKTREE_SEED_PHASES)[number]; +export type WorktreeSeedState = "pending" | "running" | "verified" | "failed"; + +export type WorktreeSeedManifest = { + version: 2; + source: { + instanceId: string; + configPath: string; + }; + snapshotAt: string | null; + seedMode: WorktreeSeedMode; + migrationRevision: string | null; + targetInstanceId: string; + phase: WorktreeSeedPhase; + state: WorktreeSeedState; + attemptId: string; + startedAt: string | null; + finishedAt: string | null; + diagnostics: Array<{ + phase: WorktreeSeedPhase; + status: "started" | "succeeded" | "failed"; + at: string; + message?: string; + }>; +}; + export type WorktreeSeedPlan = { mode: WorktreeSeedMode; excludedTables: string[]; @@ -54,6 +94,7 @@ export type WorktreeUiBranding = { }; export type WorktreeSeedMarkerPaths = { + manifest: string; pending: string; complete: string; lock: string; @@ -62,6 +103,7 @@ export type WorktreeSeedMarkerPaths = { export function resolveWorktreeSeedMarkerPaths(configPath: string): WorktreeSeedMarkerPaths { const configDir = path.dirname(path.resolve(configPath)); return { + manifest: path.resolve(configDir, WORKTREE_SEED_MANIFEST), pending: path.resolve(configDir, WORKTREE_SEED_PENDING_MARKER), complete: path.resolve(configDir, WORKTREE_SEED_COMPLETE_MARKER), lock: path.resolve(configDir, WORKTREE_SEED_LOCK_MARKER), diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 7adee70c8ae..7e3a9ece58e 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -7,6 +7,7 @@ import { readdirSync, readFileSync, readlinkSync, + renameSync, rmSync, statSync, symlinkSync, @@ -24,8 +25,11 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { applyPendingMigrations, agents, + authAccounts, + authUsers, assets, companies, + companyMemberships, createDb, documentRevisions, documents, @@ -39,6 +43,7 @@ import { issueComments, issueDocuments, issues, + instanceUserRoles, projectWorkspaces, projects, routines, @@ -65,13 +70,16 @@ import { formatShellExports, generateWorktreeColor, isWorktreeSeedMode, + WORKTREE_SEED_PHASES, resolveSuggestedWorktreeName, resolveWorktreeSeedPlan, resolveWorktreeSeedMarkerPaths, resolveWorktreeLocalPaths, sanitizeWorktreeInstanceId, type WorktreeSeedPlan, + type WorktreeSeedManifest, type WorktreeSeedMode, + type WorktreeSeedPhase, type WorktreeLocalPaths, } from "./worktree-lib.js"; import { @@ -137,6 +145,7 @@ type WorktreeReseedOptions = { preserveLiveWork?: boolean; yes?: boolean; allowLiveTarget?: boolean; + backupTarget?: boolean; }; type WorktreeRepairOptions = { @@ -197,6 +206,8 @@ type CopiedGitHooksResult = { type SeedWorktreeDatabaseResult = { backupSummary: string; + snapshotAt: string; + migrationRevision: string; pausedScheduledRoutines: number; executionQuarantine: SeededWorktreeExecutionQuarantineSummary; reboundWorkspaces: Array<{ @@ -204,28 +215,26 @@ type SeedWorktreeDatabaseResult = { fromCwd: string; toCwd: string; }>; + validation: WorktreeSeedValidationSummary; }; -type WorktreeSeedPendingMarker = { - version: 1; - state: "pending"; - sourceConfigPath: string; - seedMode: "minimal"; - createdAt: string; -}; - -type WorktreeSeedCompleteMarker = { - version: 1; - state: "complete"; - seedMode: WorktreeSeedMode; - completedAt: string; +export type WorktreeSeedValidationSummary = { + authUserCount: number; + credentialAccountCount: number; + instanceAdminCount: number; + activeMembershipCount: number; + companyCount: number; + issueCount: number; + representativeCompanyId: string; + representativeIssueId: string; + migrationRevision: string; }; type SeedWorktreeDatabase = typeof seedWorktreeDatabase; export type EnsureWorktreeSeededResult = { seeded: boolean; - reason: "seeded" | "complete_marker" | "legacy_unmarked"; + reason: "seeded" | "verified_manifest" | "complete_marker" | "legacy_unmarked"; details?: SeedWorktreeDatabaseResult; }; @@ -1393,6 +1402,140 @@ export async function quarantineSeededWorktreeExecutionState( } } +type WorktreeSeedValidationExpectation = { + adminUserId: string; + representativeCompanyId: string; + representativeIssueId: string; +}; + +async function inspectVerifiedSeedDatabase( + connectionString: string, + expected?: WorktreeSeedValidationExpectation, +): Promise<{ summary: WorktreeSeedValidationSummary; expectation: WorktreeSeedValidationExpectation }> { + const migrationState = await inspectMigrations(connectionString); + if (migrationState.status !== "upToDate") { + throw new Error( + `Migration journal is not current (${migrationState.pendingMigrations.length} pending migration(s)).`, + ); + } + const migrationRevision = migrationState.appliedMigrations.at(-1); + if (!migrationRevision) { + throw new Error("Migration journal has no applied revision."); + } + + const db = createDb(connectionString); + try { + const [counts] = await db + .select({ + authUserCount: sql`count(distinct ${authUsers.id})::int`, + credentialAccountCount: sql`count(distinct ${authAccounts.id})::int`, + instanceAdminCount: sql`count(distinct ${instanceUserRoles.userId})::int`, + activeMembershipCount: sql`count(distinct ${companyMemberships.id})::int`, + companyCount: sql`count(distinct ${companies.id})::int`, + issueCount: sql`count(distinct ${issues.id})::int`, + }) + .from(authUsers) + .leftJoin(authAccounts, eq(authAccounts.userId, authUsers.id)) + .leftJoin( + instanceUserRoles, + and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")), + ) + .leftJoin( + companyMemberships, + and( + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, authUsers.id), + eq(companyMemberships.status, "active"), + ), + ) + .leftJoin(companies, eq(companies.id, companyMemberships.companyId)) + .leftJoin(issues, eq(issues.companyId, companies.id)); + + const admin = await db + .select({ userId: authUsers.id }) + .from(authUsers) + .innerJoin( + instanceUserRoles, + and(eq(instanceUserRoles.userId, authUsers.id), eq(instanceUserRoles.role, "instance_admin")), + ) + .innerJoin( + authAccounts, + and( + eq(authAccounts.userId, authUsers.id), + sql`length(trim(${authAccounts.providerId})) > 0`, + sql`length(trim(${authAccounts.accountId})) > 0`, + ), + ) + .innerJoin( + companyMemberships, + and( + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, authUsers.id), + eq(companyMemberships.status, "active"), + ), + ) + .where(expected ? eq(authUsers.id, expected.adminUserId) : undefined) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!admin) { + throw new Error( + "No auth user has a non-empty credential account, instance-admin role, and active company membership.", + ); + } + + const representative = await db + .select({ companyId: companies.id, issueId: issues.id }) + .from(companies) + .innerJoin(issues, eq(issues.companyId, companies.id)) + .where( + expected + ? and( + eq(companies.id, expected.representativeCompanyId), + eq(issues.id, expected.representativeIssueId), + ) + : undefined, + ) + .limit(1) + .then((rows) => rows[0] ?? null); + if (!representative) { + throw new Error("No representative cloned company and issue pair is readable."); + } + + const summary: WorktreeSeedValidationSummary = { + authUserCount: counts?.authUserCount ?? 0, + credentialAccountCount: counts?.credentialAccountCount ?? 0, + instanceAdminCount: counts?.instanceAdminCount ?? 0, + activeMembershipCount: counts?.activeMembershipCount ?? 0, + companyCount: counts?.companyCount ?? 0, + issueCount: counts?.issueCount ?? 0, + representativeCompanyId: representative.companyId, + representativeIssueId: representative.issueId, + migrationRevision, + }; + if ( + summary.authUserCount < 1 + || summary.credentialAccountCount < 1 + || summary.instanceAdminCount < 1 + || summary.activeMembershipCount < 1 + || summary.companyCount < 1 + || summary.issueCount < 1 + ) { + throw new Error("Seed validation found an incomplete auth, membership, company, or issue shape."); + } + + return { + summary, + expectation: { + adminUserId: admin.userId, + representativeCompanyId: representative.companyId, + representativeIssueId: representative.issueId, + }, + }; + } finally { + await db.$client?.end?.({ timeout: 5 }).catch(() => undefined); + } +} + async function seedWorktreeDatabase(input: { sourceConfigPath: string; sourceConfig: PaperclipConfig; @@ -1401,6 +1544,7 @@ async function seedWorktreeDatabase(input: { instanceId: string; seedMode: WorktreeSeedMode; preserveLiveWork?: boolean; + onPhase?: (phase: WorktreeSeedPhase, status: "started" | "succeeded", message?: string) => void; }): Promise { const seedPlan = resolveWorktreeSeedPlan(input.seedMode); const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath); @@ -1428,6 +1572,16 @@ async function seedWorktreeDatabase(input: { sourceEnvEntries, sourceHandle?.port, ); + input.onPhase?.("source_validation", "started"); + const sourceValidation = await inspectVerifiedSeedDatabase(sourceConnectionString); + input.onPhase?.( + "source_validation", + "succeeded", + `Validated ${sourceValidation.summary.companyCount} company record(s) and ${sourceValidation.summary.issueCount} issue record(s).`, + ); + + const snapshotAt = new Date().toISOString(); + input.onPhase?.("snapshot", "started"); const backup = await runDatabaseBackup({ connectionString: sourceConnectionString, backupDir: path.resolve(input.targetPaths.backupDir, "seed"), @@ -1438,6 +1592,7 @@ async function seedWorktreeDatabase(input: { excludeTables: seedPlan.excludedTables, nullifyColumns: seedPlan.nullifyColumns, }); + input.onPhase?.("snapshot", "succeeded", `Created ${path.basename(backup.backupFile)}.`); targetHandle = await ensureEmbeddedPostgres( input.targetConfig.database.embeddedPostgresDataDir, @@ -1445,27 +1600,56 @@ async function seedWorktreeDatabase(input: { ); const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`; + input.onPhase?.("restore", "started"); await resetPostgresDatabase(adminConnectionString, "paperclip"); const targetConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`; await runDatabaseRestore({ connectionString: targetConnectionString, backupFile: backup.backupFile, }); + input.onPhase?.("restore", "succeeded"); + input.onPhase?.("migrations", "started"); await applyPendingMigrations(targetConnectionString); + input.onPhase?.("migrations", "succeeded"); + input.onPhase?.("execution_quarantine", "started"); const executionQuarantine = input.preserveLiveWork ? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY } : await quarantineSeededWorktreeExecutionState(targetConnectionString); + input.onPhase?.( + "execution_quarantine", + "succeeded", + input.preserveLiveWork + ? "Preserved copied live work by explicit request." + : formatSeededWorktreeExecutionQuarantineSummary(executionQuarantine), + ); + input.onPhase?.("routine_pause", "started"); const pausedScheduledRoutines = await pauseSeededScheduledRoutines(targetConnectionString); + input.onPhase?.("routine_pause", "succeeded", `Paused ${pausedScheduledRoutines} scheduled routine(s).`); + input.onPhase?.("workspace_rebind", "started"); const reboundWorkspaces = await rebindSeededProjectWorkspaces({ targetConnectionString, currentCwd: input.targetPaths.cwd, }); + input.onPhase?.("workspace_rebind", "succeeded", `Rebound ${reboundWorkspaces.length} workspace path(s).`); + input.onPhase?.("post_restore_validation", "started"); + const targetValidation = await inspectVerifiedSeedDatabase( + targetConnectionString, + sourceValidation.expectation, + ); + input.onPhase?.( + "post_restore_validation", + "succeeded", + `Validated migration ${targetValidation.summary.migrationRevision}.`, + ); return { backupSummary: formatDatabaseBackupResult(backup), + snapshotAt, + migrationRevision: targetValidation.summary.migrationRevision, pausedScheduledRoutines, executionQuarantine, reboundWorkspaces, + validation: targetValidation.summary, }; } finally { if (targetHandle?.startedByThisProcess) { @@ -1477,46 +1661,184 @@ async function seedWorktreeDatabase(input: { } } -function writeWorktreeSeedMarker( - filePath: string, - marker: WorktreeSeedPendingMarker | WorktreeSeedCompleteMarker, -): void { +const WORKTREE_SEED_DIAGNOSTIC_LIMIT = 32; +const WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT = 512; +const activeSeedInterruptHandlers = new Map void>(); + +function dispatchSeedInterruption(signal: NodeJS.Signals): void { + for (const handler of activeSeedInterruptHandlers.values()) { + try { + handler(signal); + } catch { + // Continue terminalizing the other active manifests before exiting. + } + } + process.exit(signal === "SIGINT" ? 130 : 143); +} + +const dispatchSeedSigint = () => dispatchSeedInterruption("SIGINT"); +const dispatchSeedSigterm = () => dispatchSeedInterruption("SIGTERM"); + +function registerSeedInterruptHandler(handler: (signal: NodeJS.Signals) => void): () => void { + const id = randomUUID(); + if (activeSeedInterruptHandlers.size === 0) { + process.once("SIGINT", dispatchSeedSigint); + process.once("SIGTERM", dispatchSeedSigterm); + } + activeSeedInterruptHandlers.set(id, handler); + return () => { + activeSeedInterruptHandlers.delete(id); + if (activeSeedInterruptHandlers.size === 0) { + process.off("SIGINT", dispatchSeedSigint); + process.off("SIGTERM", dispatchSeedSigterm); + } + }; +} + +type LegacyWorktreeSeedPendingMarker = { + version: 1; + state: "pending"; + sourceConfigPath: string; +}; + +function resolveSeedInstanceId(configPath: string): string { + const envEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(configPath)); + return nonEmpty(envEntries.PAPERCLIP_INSTANCE_ID) + ?? sanitizeWorktreeInstanceId(path.basename(path.dirname(path.resolve(configPath)))); +} + +function writeWorktreeSeedManifest(filePath: string, manifest: WorktreeSeedManifest): void { mkdirSync(path.dirname(filePath), { recursive: true }); - writeFileSync(filePath, `${JSON.stringify(marker, null, 2)}\n`, { mode: 0o600 }); + const temporaryPath = `${filePath}.${process.pid}.${randomUUID()}.tmp`; + writeFileSync(temporaryPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 }); + renameSync(temporaryPath, filePath); +} + +export function readWorktreeSeedManifest(configPath: string): WorktreeSeedManifest | null { + const manifestPath = resolveWorktreeSeedMarkerPaths(configPath).manifest; + if (!existsSync(manifestPath)) return null; + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(manifestPath, "utf8")); + } catch (error) { + throw new Error( + `Invalid worktree seed manifest at ${manifestPath}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const value = parsed as Partial; + const diagnosticsValid = Array.isArray(value.diagnostics) && value.diagnostics.every((diagnostic) => ( + diagnostic + && typeof diagnostic === "object" + && WORKTREE_SEED_PHASES.includes(diagnostic.phase) + && ["started", "succeeded", "failed"].includes(diagnostic.status) + && typeof diagnostic.at === "string" + && (diagnostic.message === undefined || typeof diagnostic.message === "string") + )); + const verifiedTerminalValid = value.state !== "verified" || ( + value.phase === "complete" + && typeof value.snapshotAt === "string" + && value.snapshotAt.length > 0 + && typeof value.migrationRevision === "string" + && value.migrationRevision.length > 0 + && typeof value.startedAt === "string" + && typeof value.finishedAt === "string" + && value.diagnostics?.some((diagnostic) => ( + diagnostic.phase === "complete" && diagnostic.status === "succeeded" + )) === true + ); + if ( + !value + || typeof value !== "object" + || value.version !== 2 + || !value.source + || typeof value.source.instanceId !== "string" + || typeof value.source.configPath !== "string" + || typeof value.targetInstanceId !== "string" + || value.targetInstanceId.length === 0 + || !isWorktreeSeedMode(String(value.seedMode ?? "")) + || !WORKTREE_SEED_PHASES.includes(value.phase as WorktreeSeedPhase) + || !["pending", "running", "verified", "failed"].includes(String(value.state ?? "")) + || typeof value.attemptId !== "string" + || value.attemptId.length === 0 + || !diagnosticsValid + || !verifiedTerminalValid + ) { + throw new Error(`Invalid worktree seed manifest at ${manifestPath}.`); + } + return value as WorktreeSeedManifest; } export function markWorktreeSeedPending(input: { configPath: string; sourceConfigPath: string; - now?: Date; -}): void { - const markers = resolveWorktreeSeedMarkerPaths(input.configPath); - rmSync(markers.complete, { force: true }); - writeWorktreeSeedMarker(markers.pending, { - version: 1, - state: "pending", - sourceConfigPath: path.resolve(input.sourceConfigPath), - seedMode: "minimal", - createdAt: (input.now ?? new Date()).toISOString(), - }); -} - -export function markWorktreeSeedComplete(input: { - configPath: string; + targetInstanceId?: string; seedMode?: WorktreeSeedMode; now?: Date; }): void { const markers = resolveWorktreeSeedMarkerPaths(input.configPath); - writeWorktreeSeedMarker(markers.complete, { - version: 1, - state: "complete", + const at = (input.now ?? new Date()).toISOString(); + writeWorktreeSeedManifest(markers.manifest, { + version: 2, + source: { + instanceId: resolveSeedInstanceId(input.sourceConfigPath), + configPath: path.resolve(input.sourceConfigPath), + }, + snapshotAt: null, seedMode: input.seedMode ?? "minimal", - completedAt: (input.now ?? new Date()).toISOString(), + migrationRevision: null, + targetInstanceId: input.targetInstanceId ?? resolveSeedInstanceId(input.configPath), + phase: "pending", + state: "pending", + attemptId: randomUUID(), + startedAt: null, + finishedAt: null, + diagnostics: [{ phase: "pending", status: "succeeded", at }], }); + // New manifests are authoritative. Legacy files are removed so no caller can + // mistake a stale binary marker for current verified seed state. + rmSync(markers.complete, { force: true }); rmSync(markers.pending, { force: true }); } -function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMarker { +function updateWorktreeSeedManifest(input: { + configPath: string; + phase: WorktreeSeedPhase; + status: "started" | "succeeded" | "failed"; + state?: WorktreeSeedManifest["state"]; + message?: string; + snapshotAt?: string | null; + migrationRevision?: string | null; + now?: Date; +}): WorktreeSeedManifest { + const markers = resolveWorktreeSeedMarkerPaths(input.configPath); + const current = readWorktreeSeedManifest(input.configPath); + if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`); + const at = (input.now ?? new Date()).toISOString(); + const nextState = input.state ?? current.state; + const diagnostic = { + phase: input.phase, + status: input.status, + at, + ...(input.message + ? { message: input.message.slice(0, WORKTREE_SEED_DIAGNOSTIC_MESSAGE_LIMIT) } + : {}), + }; + const next: WorktreeSeedManifest = { + ...current, + phase: input.phase, + state: nextState, + snapshotAt: input.snapshotAt === undefined ? current.snapshotAt : input.snapshotAt, + migrationRevision: + input.migrationRevision === undefined ? current.migrationRevision : input.migrationRevision, + startedAt: current.startedAt ?? (input.status === "started" ? at : null), + finishedAt: nextState === "verified" || nextState === "failed" ? at : null, + diagnostics: [...current.diagnostics, diagnostic].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT), + }; + writeWorktreeSeedManifest(markers.manifest, next); + return next; +} + +function readLegacyWorktreeSeedPendingMarker(filePath: string): LegacyWorktreeSeedPendingMarker { let parsed: unknown; try { parsed = JSON.parse(readFileSync(filePath, "utf8")); @@ -1537,7 +1859,7 @@ function readWorktreeSeedPendingMarker(filePath: string): WorktreeSeedPendingMar throw new Error(`Invalid worktree seed-pending marker at ${filePath}.`); } - return parsed as WorktreeSeedPendingMarker; + return parsed as LegacyWorktreeSeedPendingMarker; } const WORKTREE_SEED_LOCK_POLL_MS = 50; @@ -1631,6 +1953,117 @@ async function acquireWorktreeSeedLock(lockPath: string): Promise<() => Promise< } } +function startWorktreeSeedAttempt(configPath: string, now = new Date()): WorktreeSeedManifest { + const markers = resolveWorktreeSeedMarkerPaths(configPath); + const current = readWorktreeSeedManifest(configPath); + if (!current) throw new Error(`Worktree seed manifest does not exist at ${markers.manifest}.`); + const at = now.toISOString(); + const next: WorktreeSeedManifest = { + ...current, + state: "running", + phase: "pending", + attemptId: randomUUID(), + snapshotAt: null, + migrationRevision: null, + startedAt: at, + finishedAt: null, + diagnostics: [ + ...current.diagnostics, + { phase: "pending" as const, status: "started" as const, at }, + ].slice(-WORKTREE_SEED_DIAGNOSTIC_LIMIT), + }; + writeWorktreeSeedManifest(markers.manifest, next); + return next; +} + +async function runVerifiedWorktreeSeed(input: { + configPath: string; + sourceConfigPath: string; + sourceConfig: PaperclipConfig; + targetConfig: PaperclipConfig; + targetPaths: WorktreeLocalPaths; + instanceId: string; + seedMode: WorktreeSeedMode; + preserveLiveWork?: boolean; + seedDatabase: SeedWorktreeDatabase; +}): Promise { + let activePhase: WorktreeSeedPhase = "pending"; + const previous = readWorktreeSeedManifest(input.configPath); + if (previous?.state === "running") { + updateWorktreeSeedManifest({ + configPath: input.configPath, + phase: previous.phase, + status: "failed", + state: "failed", + message: "The previous seed attempt ended without a terminal result.", + }); + } + startWorktreeSeedAttempt(input.configPath); + + const unregisterInterruption = registerSeedInterruptHandler((signal) => { + updateWorktreeSeedManifest({ + configPath: input.configPath, + phase: activePhase, + status: "failed", + state: "failed", + message: `Seed interrupted by ${signal} during ${activePhase}.`, + }); + }); + + try { + const details = await input.seedDatabase({ + sourceConfigPath: input.sourceConfigPath, + sourceConfig: input.sourceConfig, + targetConfig: input.targetConfig, + targetPaths: input.targetPaths, + instanceId: input.instanceId, + seedMode: input.seedMode, + preserveLiveWork: input.preserveLiveWork, + onPhase: (phase, status, message) => { + activePhase = phase; + updateWorktreeSeedManifest({ + configPath: input.configPath, + phase, + status, + state: "running", + message, + ...(phase === "snapshot" && status === "started" + ? { snapshotAt: new Date().toISOString() } + : {}), + }); + }, + }); + if (!details.snapshotAt || !details.migrationRevision || !details.validation) { + throw new Error("Seed implementation returned without required validation evidence."); + } + updateWorktreeSeedManifest({ + configPath: input.configPath, + phase: "complete", + status: "succeeded", + state: "verified", + snapshotAt: details.snapshotAt, + migrationRevision: details.migrationRevision, + message: + `Verified ${details.validation.companyCount} company record(s), ` + + `${details.validation.issueCount} issue record(s), auth, admin, membership, and migration state.`, + }); + return details; + } catch (error) { + updateWorktreeSeedManifest({ + configPath: input.configPath, + phase: activePhase, + status: "failed", + state: "failed", + // Do not persist the underlying error: database/driver errors may contain + // connection credentials. The CLI still returns the exact error to its caller. + message: `Seed failed during ${activePhase}.`, + }); + throw error; + } finally { + unregisterInterruption(); + } +} + export async function ensureWorktreeSeeded( opts: WorktreeEnsureSeededOptions = {}, dependencies: { seedDatabase?: SeedWorktreeDatabase } = {}, @@ -1642,24 +2075,36 @@ export async function ensureWorktreeSeeded( try { // These checks deliberately happen under the cross-process lock. A second // service process waits for the first seed transaction, then observes the - // complete marker instead of cloning the same database concurrently. - if (existsSync(markers.complete)) { + // verified manifest instead of cloning the same database concurrently. + let manifest = readWorktreeSeedManifest(configPath); + if (manifest?.state === "verified") { + return { seeded: false, reason: "verified_manifest" }; + } + if (!manifest && existsSync(markers.complete)) { return { seeded: false, reason: "complete_marker" }; } - if (!existsSync(markers.pending)) { + if (!manifest && existsSync(markers.pending)) { + const legacyPending = readLegacyWorktreeSeedPendingMarker(markers.pending); + markWorktreeSeedPending({ + configPath, + sourceConfigPath: legacyPending.sourceConfigPath, + seedMode: "minimal", + }); + manifest = readWorktreeSeedManifest(configPath); + } + if (!manifest) { // Worktrees created before lazy seeding shipped were seeded eagerly and // have neither marker. Preserve that compatibility without re-cloning. return { seeded: false, reason: "legacy_unmarked" }; } - const pending = readWorktreeSeedPendingMarker(markers.pending); const sourceConfigPath = opts.fromConfig || opts.fromDataDir || opts.fromInstance ? resolveSourceConfigPath({ fromConfig: opts.fromConfig, fromDataDir: opts.fromDataDir, fromInstance: opts.fromInstance, }) - : path.resolve(pending.sourceConfigPath); + : path.resolve(manifest.source.configPath); if (path.resolve(sourceConfigPath) === path.resolve(configPath)) { throw new Error( @@ -1679,16 +2124,17 @@ export async function ensureWorktreeSeeded( const targetRoot = path.dirname(path.dirname(configPath)); const targetPaths = resolveWorktreeReseedTargetPaths({ configPath, rootPath: targetRoot }); const seedDatabase = dependencies.seedDatabase ?? seedWorktreeDatabase; - const details = await seedDatabase({ + const details = await runVerifiedWorktreeSeed({ + configPath, sourceConfigPath, sourceConfig, targetConfig, targetPaths, instanceId: targetPaths.instanceId, - seedMode: "minimal", + seedMode: manifest.seedMode, preserveLiveWork: opts.preserveLiveWork, + seedDatabase, }); - markWorktreeSeedComplete({ configPath }); return { seeded: true, reason: "seeded", details }; } finally { await releaseLock(); @@ -1762,6 +2208,8 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { markWorktreeSeedPending({ configPath: paths.configPath, sourceConfigPath, + targetInstanceId: instanceId, + seedMode, }); const sourceEnvEntries = readPaperclipEnvEntries(resolvePaperclipEnvFile(sourceConfigPath)); const existingAgentJwtSecret = @@ -1790,8 +2238,11 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { } const spinner = p.spinner(); spinner.start(`Seeding isolated worktree database from source instance (${seedMode})...`); + const markers = resolveWorktreeSeedMarkerPaths(paths.configPath); + const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock); try { - const seeded = await seedWorktreeDatabase({ + const seeded = await runVerifiedWorktreeSeed({ + configPath: paths.configPath, sourceConfigPath, sourceConfig, targetConfig, @@ -1799,16 +2250,18 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { instanceId, seedMode, preserveLiveWork: opts.preserveLiveWork, + seedDatabase: seedWorktreeDatabase, }); seedSummary = seeded.backupSummary; seedExecutionQuarantineSummary = seeded.executionQuarantine; pausedScheduledRoutineCount = seeded.pausedScheduledRoutines; reboundWorkspaceSummary = seeded.reboundWorkspaces; - markWorktreeSeedComplete({ configPath: paths.configPath, seedMode }); spinner.stop(`Seeded isolated worktree database (${seedMode}).`); } catch (error) { spinner.stop(pc.red("Failed to seed worktree database.")); throw error; + } finally { + await releaseSeedLock(); } } @@ -3480,6 +3933,34 @@ export async function worktreeMergeHistoryCommand(sourceArg: string | undefined, } } +async function backupWorktreeReseedTarget(input: { + targetConfig: PaperclipConfig; + targetPaths: WorktreeLocalPaths; +}): Promise { + if (input.targetConfig.database.mode !== "embedded-postgres") { + throw new Error("Managed worktree repair requires an embedded PostgreSQL target."); + } + const targetHandle = await ensureEmbeddedPostgres( + input.targetConfig.database.embeddedPostgresDataDir, + input.targetConfig.database.embeddedPostgresPort, + ); + try { + const adminConnectionString = `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/postgres`; + await ensurePostgresDatabase(adminConnectionString, "paperclip"); + const result = await runDatabaseBackup({ + connectionString: `postgres://paperclip:paperclip@127.0.0.1:${targetHandle.port}/paperclip`, + backupDir: path.resolve(input.targetPaths.backupDir, "repair"), + retention: { dailyDays: 30, weeklyWeeks: 12, monthlyMonths: 12 }, + filenamePrefix: `${input.targetPaths.instanceId}-pre-repair`, + backupEngine: "auto", + includeMigrationJournal: true, + }); + return formatDatabaseBackupResult(result); + } finally { + if (targetHandle.startedByThisProcess) await targetHandle.stop(); + } +} + async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { const seedMode = opts.seedMode ?? "full"; if (!isWorktreeSeedMode(seedMode)) { @@ -3535,8 +4016,23 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { const spinner = p.spinner(); spinner.start(`Reseeding ${targetEndpoint.label} from ${source.label} (${seedMode})...`); + const markers = resolveWorktreeSeedMarkerPaths(targetEndpoint.configPath); + mkdirSync(path.dirname(markers.lock), { recursive: true }); + const releaseSeedLock = await acquireWorktreeSeedLock(markers.lock); try { - const seeded = await seedWorktreeDatabase({ + let targetBackupSummary: string | null = null; + if (opts.backupTarget) { + targetBackupSummary = await backupWorktreeReseedTarget({ targetConfig, targetPaths }); + p.log.message(pc.dim(`Recoverable pre-repair backup: ${targetBackupSummary}`)); + } + markWorktreeSeedPending({ + configPath: targetEndpoint.configPath, + sourceConfigPath: source.configPath, + targetInstanceId: targetPaths.instanceId, + seedMode, + }); + const seeded = await runVerifiedWorktreeSeed({ + configPath: targetEndpoint.configPath, sourceConfigPath: source.configPath, sourceConfig, targetConfig, @@ -3544,8 +4040,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { instanceId: targetPaths.instanceId, seedMode, preserveLiveWork: opts.preserveLiveWork, + seedDatabase: seedWorktreeDatabase, }); - markWorktreeSeedComplete({ configPath: targetEndpoint.configPath, seedMode }); spinner.stop(`Reseeded ${targetEndpoint.label} (${seedMode}).`); p.log.message(pc.dim(`Source: ${source.configPath}`)); p.log.message(pc.dim(`Target: ${targetEndpoint.configPath}`)); @@ -3567,6 +4063,8 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { } catch (error) { spinner.stop(pc.red("Failed to reseed worktree database.")); throw error; + } finally { + await releaseSeedLock(); } } @@ -3748,6 +4246,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false) .option("--yes", "Skip the destructive confirmation prompt", false) .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false) + .option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false) .action(worktreeReseedCommand); worktree diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 4c448aaa387..21c3b97a4bb 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -479,25 +479,24 @@ After `worktree init`, both the server and the CLI auto-load the repo-local `.pa Seeding a worktree database is the heaviest part of `worktree init`. That work can be deferred so a worktree is cheap to create and only pays the seed cost the first time it is actually used — the CLI/dev-time analog of the server's lazy runtime provisioning (see the board-operator guide's "Lazy runtime provisioning" section). -Seeding state is tracked with two marker files under the worktree's `.paperclip/` directory: +Seeding state is tracked in `.paperclip/seed-manifest.json`. The versioned manifest records only non-secret evidence: source instance id/config path, target instance id, seed mode, snapshot time, migration revision, attempt timestamps, current phase, terminal state, and a bounded phase diagnostic history. It never stores database credentials or auth credential material. The legacy `seed-pending` and `seed-complete` files remain read-only compatibility signals for worktrees created before the manifest shipped. -- `seed-pending` — the isolated database has not been seeded yet (a **lean** worktree). Written by `worktree init` before any seed runs. -- `seed-complete` — the database was seeded; the pending marker is removed. +The default `worktree init` still seeds eagerly. A lean worktree (created without an eager seed) has a `pending` manifest until something seeds it on demand: -The default `worktree init` still seeds eagerly and writes `seed-complete` immediately. A lean worktree (created without an eager seed) keeps its `seed-pending` marker until something seeds it on demand: - -- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: a present `seed-complete` marker or a missing `seed-pending` marker short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source instance from the `seed-pending` marker unless you pass `--from-config`. +- `pnpm paperclipai worktree ensure-seeded` performs the deferred seed **exactly once**. It is lock-guarded and idempotent: only a complete `verified` manifest short-circuits it, so it is safe to call repeatedly and from concurrent processes. It reads the source identity from the manifest unless you pass `--from-config`. - `paperclipai run` calls `ensureWorktreeSeeded` automatically before doctor/boot, so `run` transparently seeds a lean worktree on first launch. -- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the worktree is still `seed-pending`. An explicitly configured runtime provision command always takes precedence. +- Managed git-worktree runtime startup also runs `scripts/provision-worktree-runtime.sh` automatically when a legacy workspace policy has no explicit runtime provision command and the manifest is not verified. An explicitly configured runtime provision command always takes precedence. - Worktrees created before lazy seeding shipped have neither marker; they are treated as already-seeded for backward compatibility (never re-cloned). -**Seed-pending guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose database is still `seed-pending` and points you at the fix: +Both `minimal` and `full` modes use the same terminal validation contract. Before a manifest can become `verified`, Paperclip proves the migration journal is current and reads a credential-backed auth user, instance administrator role, active company membership, and representative cloned company/issue pair in both source and target. Restore, migrations, execution quarantine, routine pausing, workspace rebinding, and post-restore validation all run under the seed lock. An interruption leaves the exact active phase in terminal `failed` state; it cannot produce readiness evidence. + +**Unverified-seed guard.** `pnpm dev` (the dev-runner) refuses to boot a worktree whose manifest is pending, running, failed, malformed, or missing required verification evidence and points you at the fix: ``` [paperclip] this worktree database is seed-pending. Run `pnpm paperclipai worktree ensure-seeded` before `pnpm dev`. ``` -This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty, unseeded database — run `worktree ensure-seeded` once and re-run `pnpm dev`. +This guard (`isWorktreeSeedPending` in `server/src/dev-runner-worktree.ts`) prevents `pnpm dev` from starting the app against an empty or partially restored database — run `worktree ensure-seeded` once and re-run `pnpm dev`. Provisioned git worktrees also pause seeded routines that still have enabled schedule triggers in the isolated worktree database by default. This prevents copied daily/cron routines from firing unexpectedly inside the new workspace instance during development without disabling webhook/API-only routines. @@ -603,6 +602,7 @@ For an already-created worktree where you want to keep the existing repo-local c | `--seed-mode ` | Seed profile: `minimal` or `full` (default: `full`) | | `--yes` | Skip the destructive confirmation prompt | | `--allow-live-target` | Override the guard that requires the target worktree DB to be stopped first | +| `--backup-target` | Retain a recoverable full target-DB backup before reseeding | Examples: @@ -622,6 +622,8 @@ npx paperclipai worktree reseed \ --seed-mode full ``` +Managed workspace repair uses this same verified full-reseed contract through `POST /api/execution-workspaces/:id/runtime-commands/repair`. The exclusive, audited operation stops managed services, writes a recoverable pre-repair database backup under the isolated instance's backup directory, performs the full seed/migration/quarantine/rebinding sequence, and restarts only after terminal manifest and service-health validation. It preserves the worktree filesystem. On failure, services remain stopped while the database backup, seed manifest, bounded phase diagnostics, and operation log are retained for inspection; repair never retries itself in a loop. + **`npx paperclipai worktree:make [options]`** — Create `~/NAME` as a git worktree, then initialize an isolated Paperclip instance inside it. This combines `git worktree add` with `worktree init` in a single step. | Option | Description | diff --git a/packages/shared/src/types/workspace-operation.ts b/packages/shared/src/types/workspace-operation.ts index d1ccf6622d5..345e306ca7a 100644 --- a/packages/shared/src/types/workspace-operation.ts +++ b/packages/shared/src/types/workspace-operation.ts @@ -3,6 +3,7 @@ export type WorkspaceOperationPhase = | "workspace_config_freshness" | "workspace_provision" | "workspace_runtime_provision" + | "workspace_repair" | "workspace_teardown" | "worktree_cleanup" | "workspace_finalize"; diff --git a/scripts/__tests__/provision-worktree-self-heal.test.mjs b/scripts/__tests__/provision-worktree-self-heal.test.mjs index 606a24edc2d..b81ff42942f 100644 --- a/scripts/__tests__/provision-worktree-self-heal.test.mjs +++ b/scripts/__tests__/provision-worktree-self-heal.test.mjs @@ -137,7 +137,10 @@ test("uses the base CLI when its import graph boots", () => { assert.equal(result.status, 0, result.stderr); const config = readWorktreeConfig(worktreeCwd); assert.equal(config.$meta.source, "fake-cli"); - assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending"))); + assert.equal( + JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state, + "pending", + ); const initInvocation = readCliInvocations(baseCwd).find( (args) => args[0] === "worktree" && args[1] === "init", ); @@ -166,7 +169,10 @@ test("falls back to an isolated config when the base CLI cannot boot", () => { ); const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8"); assert.match(env, /PAPERCLIP_IN_WORKTREE=true/); - assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending"))); + assert.equal( + JSON.parse(fs.readFileSync(path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), "utf8")).state, + "pending", + ); }); test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => { @@ -271,7 +277,7 @@ test("runtime provisioning invokes ensure-seeded once and fast-exits after succe const second = runRuntimeProvision(baseCwd, worktreeCwd); assert.equal(second.status, 0, second.stderr); - assert.match(second.stderr, /already seeded; skipping/); + assert.match(second.stderr, /already seeded.*skipping/); const ensureCallsAfterSecond = readCliInvocations(baseCwd) .filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded"); assert.equal(ensureCallsAfterSecond.length, 1); @@ -290,3 +296,22 @@ test("runtime provisioning leaves seed-pending in place when ensure-seeded fails assert.ok(fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-pending"))); assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "seed-complete"))); }); + +test("runtime provisioning does not trust a truncated verified manifest", () => { + const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0, ensureExit: 4 }); + const worktreeCwd = makeTempDir("paperclip-provision-runtime-truncated-"); + fs.mkdirSync(path.join(worktreeCwd, ".paperclip"), { recursive: true }); + fs.writeFileSync(path.join(worktreeCwd, ".paperclip", "config.json"), "{}\n"); + fs.writeFileSync( + path.join(worktreeCwd, ".paperclip", "seed-manifest.json"), + JSON.stringify({ version: 2, state: "verified" }), + ); + + const result = runRuntimeProvision(baseCwd, worktreeCwd); + assert.equal(result.status, 4, result.stderr); + assert.equal( + readCliInvocations(baseCwd) + .filter((args) => args[0] === "worktree" && args[1] === "ensure-seeded").length, + 1, + ); +}); diff --git a/scripts/provision-worktree-runtime.sh b/scripts/provision-worktree-runtime.sh index 539713fa096..7fd9fc8213c 100755 --- a/scripts/provision-worktree-runtime.sh +++ b/scripts/provision-worktree-runtime.sh @@ -7,6 +7,7 @@ paperclip_home="${PAPERCLIP_HOME:-$HOME/.paperclip}" paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}" paperclip_dir="$worktree_cwd/.paperclip" worktree_config_path="$paperclip_dir/config.json" +seed_manifest_path="$paperclip_dir/seed-manifest.json" seed_pending_marker_path="$paperclip_dir/seed-pending" seed_complete_marker_path="$paperclip_dir/seed-complete" @@ -20,8 +21,37 @@ if [[ ! -d "$worktree_cwd" ]]; then exit 1 fi -if [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then - echo "Worktree database is already seeded; skipping runtime provisioning." >&2 +if [[ -e "$seed_manifest_path" ]]; then + seed_manifest_state="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF' +const fs = require("node:fs"); +try { + const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8")); + const complete = value?.version === 2 + && value?.state === "verified" + && value?.phase === "complete" + && typeof value?.source?.instanceId === "string" && value.source.instanceId.length > 0 + && typeof value?.source?.configPath === "string" && value.source.configPath.length > 0 + && (value?.seedMode === "minimal" || value?.seedMode === "full") + && typeof value?.snapshotAt === "string" && value.snapshotAt.length > 0 + && typeof value?.migrationRevision === "string" && value.migrationRevision.length > 0 + && typeof value?.targetInstanceId === "string" && value.targetInstanceId.length > 0 + && typeof value?.attemptId === "string" && value.attemptId.length > 0 + && typeof value?.startedAt === "string" + && typeof value?.finishedAt === "string" + && Array.isArray(value?.diagnostics) + && value.diagnostics.some((entry) => entry?.phase === "complete" && entry?.status === "succeeded" && typeof entry?.at === "string"); + process.stdout.write(complete ? "verified" : "incomplete"); +} catch { + process.stdout.write("invalid"); +} +EOF +)" + if [[ "$seed_manifest_state" == "verified" ]]; then + echo "Worktree database has a verified seed manifest; skipping runtime provisioning." >&2 + exit 0 + fi +elif [[ -e "$seed_complete_marker_path" || ! -e "$seed_pending_marker_path" ]]; then + echo "Worktree database is already seeded by a legacy marker; skipping runtime provisioning." >&2 exit 0 fi @@ -31,6 +61,17 @@ if [[ ! -f "$worktree_config_path" ]]; then fi source_config_path="${PAPERCLIP_CONFIG:-}" +if [[ -e "$seed_manifest_path" ]]; then + manifest_source_config_path="$(SEED_MANIFEST_PATH="$seed_manifest_path" node <<'EOF' +const fs = require("node:fs"); +const value = JSON.parse(fs.readFileSync(process.env.SEED_MANIFEST_PATH, "utf8")); +process.stdout.write(typeof value?.source?.configPath === "string" ? value.source.configPath : ""); +EOF +)" + if [[ -n "$manifest_source_config_path" ]]; then + source_config_path="$manifest_source_config_path" + fi +fi if [[ -z "$source_config_path" && ( -e "$base_cwd/.paperclip/config.json" || -L "$base_cwd/.paperclip/config.json" ) ]]; then source_config_path="$base_cwd/.paperclip/config.json" fi @@ -41,7 +82,7 @@ source_config_args=(--from-config "$source_config_path") if [[ "$source_config_path" == "$worktree_config_path" ]]; then # A human may invoke this after sourcing `worktree env`, which points # PAPERCLIP_CONFIG at the target. In that case the CLI reads the original - # source config from the seed-pending marker instead. + # source config from the seed manifest or legacy pending marker instead. source_config_args=() fi diff --git a/scripts/provision-worktree.sh b/scripts/provision-worktree.sh index b5b283a1343..55b0c2e1c30 100644 --- a/scripts/provision-worktree.sh +++ b/scripts/provision-worktree.sh @@ -8,6 +8,7 @@ paperclip_instance_id="${PAPERCLIP_INSTANCE_ID:-default}" paperclip_dir="$worktree_cwd/.paperclip" worktree_config_path="$paperclip_dir/config.json" worktree_env_path="$paperclip_dir/.env" +seed_manifest_path="$paperclip_dir/seed-manifest.json" seed_pending_marker_path="$paperclip_dir/seed-pending" seed_complete_marker_path="$paperclip_dir/seed-complete" worktree_name="${PAPERCLIP_WORKSPACE_BRANCH:-$(basename "$worktree_cwd")}" @@ -237,25 +238,48 @@ for (const rawValue of runtimePaths) { EOF } -write_seed_pending_marker() { +write_seed_pending_manifest() { + SEED_MANIFEST_PATH="$seed_manifest_path" \ SEED_PENDING_MARKER_PATH="$seed_pending_marker_path" \ SEED_COMPLETE_MARKER_PATH="$seed_complete_marker_path" \ SOURCE_CONFIG_PATH="$source_config_path" \ + TARGET_INSTANCE_ID="$worktree_instance_id" \ node <<'EOF' +const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); +const manifestPath = process.env.SEED_MANIFEST_PATH; const pendingPath = process.env.SEED_PENDING_MARKER_PATH; const completePath = process.env.SEED_COMPLETE_MARKER_PATH; +const sourceConfigPath = path.resolve(process.env.SOURCE_CONFIG_PATH); +const sourceEnvPath = path.join(path.dirname(sourceConfigPath), ".env"); +let sourceInstanceId = path.basename(path.dirname(sourceConfigPath)); +if (fs.existsSync(sourceEnvPath)) { + const match = fs.readFileSync(sourceEnvPath, "utf8").match(/^\s*(?:export\s+)?PAPERCLIP_INSTANCE_ID\s*=\s*["']?([^\s"'#]+)["']?/m); + if (match?.[1]) sourceInstanceId = match[1]; +} fs.rmSync(completePath, { force: true }); +fs.rmSync(pendingPath, { force: true }); +const at = new Date().toISOString(); fs.writeFileSync( - pendingPath, + manifestPath, `${JSON.stringify({ - version: 1, - state: "pending", - sourceConfigPath: path.resolve(process.env.SOURCE_CONFIG_PATH), + version: 2, + source: { + instanceId: sourceInstanceId, + configPath: sourceConfigPath, + }, + snapshotAt: null, seedMode: "minimal", - createdAt: new Date().toISOString(), + migrationRevision: null, + targetInstanceId: process.env.TARGET_INSTANCE_ID, + phase: "pending", + state: "pending", + attemptId: crypto.randomUUID(), + startedAt: null, + finishedAt: null, + diagnostics: [{ phase: "pending", status: "succeeded", at }], }, null, 2)}\n`, { mode: 0o600 }, ); @@ -551,8 +575,8 @@ else created_worktree_config=1 fi -if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then - write_seed_pending_marker +if [[ "$created_worktree_config" -eq 1 && ! -e "$seed_manifest_path" && ! -e "$seed_pending_marker_path" && ! -e "$seed_complete_marker_path" ]]; then + write_seed_pending_manifest fi list_base_node_modules_paths() { diff --git a/server/src/__tests__/dev-runner-worktree.test.ts b/server/src/__tests__/dev-runner-worktree.test.ts index f3a29440ebd..5dd179edb58 100644 --- a/server/src/__tests__/dev-runner-worktree.test.ts +++ b/server/src/__tests__/dev-runner-worktree.test.ts @@ -36,6 +36,36 @@ describe("dev-runner worktree env bootstrap", () => { expect(isWorktreeSeedPending(root)).toBe(false); }); + it("guards every manifest state except a complete verified manifest", () => { + const root = createTempRoot("paperclip-dev-runner-seed-manifest-"); + const manifestPath = path.join(root, ".paperclip", "seed-manifest.json"); + fs.mkdirSync(path.dirname(manifestPath), { recursive: true }); + fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "failed" }), "utf8"); + expect(isWorktreeSeedPending(root)).toBe(true); + + fs.writeFileSync(manifestPath, JSON.stringify({ version: 2, state: "verified" }), "utf8"); + expect(isWorktreeSeedPending(root)).toBe(true); + + fs.writeFileSync(manifestPath, JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: "/source/config.json" }, + snapshotAt: "2026-08-18T00:00:00.000Z", + seedMode: "minimal", + migrationRevision: "0001", + targetInstanceId: "target", + phase: "complete", + state: "verified", + attemptId: "attempt", + startedAt: "2026-08-18T00:00:00.000Z", + finishedAt: "2026-08-18T00:01:00.000Z", + diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }], + }), "utf8"); + expect(isWorktreeSeedPending(root)).toBe(false); + + fs.writeFileSync(manifestPath, "not-json", "utf8"); + expect(isWorktreeSeedPending(root)).toBe(true); + }); + it("detects linked git worktrees from .git files", () => { const root = createTempRoot("paperclip-dev-runner-worktree-"); fs.writeFileSync(path.join(root, ".git"), "gitdir: /tmp/paperclip/.git/worktrees/feature\n", "utf8"); diff --git a/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts b/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts index 3999299f586..8290e4eddfa 100644 --- a/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts +++ b/server/src/__tests__/execution-workspace-runtime-control-conflict.test.ts @@ -1,3 +1,8 @@ +import { EventEmitter } from "node:events"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; import express from "express"; import request from "supertest"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -29,6 +34,12 @@ const mockBuildWorkspaceRuntimeDesiredStatePatch = vi.hoisted(() => vi.fn()); // `execution-workspace-runtime-lease-route.test.ts` exercises the lease itself against a real db. const mockClaimRuntimeLease = vi.hoisted(() => vi.fn(async () => null)); const mockReleaseRuntimeLease = vi.hoisted(() => vi.fn(async () => undefined)); +const mockSpawn = vi.hoisted(() => vi.fn()); + +vi.mock("node:child_process", async () => ({ + ...(await vi.importActual("node:child_process")), + spawn: mockSpawn, +})); vi.mock("../telemetry.js", () => ({ getTelemetryClient: mockGetTelemetryClient })); @@ -45,7 +56,7 @@ vi.mock("../services/index.js", () => ({ claim: mockClaimRuntimeLease, release: mockReleaseRuntimeLease, }), - LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"], + LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"], })); vi.mock("../services/workspace-runtime.js", () => ({ @@ -135,6 +146,7 @@ async function createApp() { describe.sequential("execution workspace runtime control conflict and failure reconciliation", () => { beforeEach(() => { vi.clearAllMocks(); + mockSpawn.mockReset(); mockAccessService.decide.mockResolvedValue({ allowed: true, action: "runtime:manage", @@ -165,7 +177,7 @@ describe.sequential("execution workspace runtime control conflict and failure re // Mirror the real recorder: a throwing `run` becomes a terminal failed operation and // the error still propagates to the caller. try { - await input.run(); + await input.run(async () => undefined); } catch (error) { (error as any).recordedOperationStatus = "failed"; throw error; @@ -247,6 +259,156 @@ describe.sequential("execution workspace runtime control conflict and failure re ); }); + it("runs an audited full repair through terminal seed and service readiness validation", async () => { + const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-success-")); + try { + const configDir = path.join(workspaceCwd, ".paperclip"); + const sourceConfigPath = path.join(workspaceCwd, "source", "config.json"); + const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs"); + const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts"); + fs.mkdirSync(configDir, { recursive: true }); + fs.mkdirSync(path.dirname(sourceConfigPath), { recursive: true }); + fs.mkdirSync(path.dirname(cliRunner), { recursive: true }); + fs.mkdirSync(path.dirname(cliEntry), { recursive: true }); + fs.writeFileSync(path.join(configDir, "config.json"), "{}\n"); + fs.writeFileSync(sourceConfigPath, "{}\n"); + fs.writeFileSync(cliRunner, "// test runner\n"); + fs.writeFileSync(cliEntry, "// test entry\n"); + fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: sourceConfigPath }, + state: "failed", + attemptId: "previous", + })); + + const { deriveWorktreeInstanceId } = await import("../services/workspace-instance-cleanup.js"); + mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd })); + mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd }); + mockStartRuntimeServices.mockResolvedValue([{ + id: "service-1", + status: "running", + healthStatus: "healthy", + }]); + mockSpawn.mockImplementation((_command: string, args: string[]) => { + expect(args).toEqual(expect.arrayContaining([ + "worktree", "reseed", "--seed-mode", "full", "--yes", "--backup-target", + ])); + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = vi.fn(); + queueMicrotask(() => { + fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: sourceConfigPath }, + snapshotAt: "2026-08-18T00:00:00.000Z", + seedMode: "full", + migrationRevision: "0142_test.sql", + targetInstanceId: deriveWorktreeInstanceId(workspaceCwd), + phase: "complete", + state: "verified", + attemptId: "repair-attempt", + startedAt: "2026-08-18T00:00:00.000Z", + finishedAt: "2026-08-18T00:01:00.000Z", + diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }], + })); + child.emit("exit", 0); + }); + return child; + }); + + const app = await createApp(); + const res = await request(app) + .post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`) + .send({}); + + expect(res.status).toBe(200); + expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledWith( + expect.objectContaining({ executionWorkspaceId, workspaceCwd }), + ); + expect(mockStartRuntimeServices).toHaveBeenCalledTimes(1); + expect(mockLogActivity).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ action: "execution_workspace.runtime_repair" }), + ); + } finally { + fs.rmSync(workspaceCwd, { recursive: true, force: true }); + } + }); + + it("fails repair at the exact seed phase and leaves managed services stopped", async () => { + const workspaceCwd = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-route-repair-failure-")); + try { + const configDir = path.join(workspaceCwd, ".paperclip"); + const sourceConfigPath = path.join(workspaceCwd, "source", "config.json"); + const cliRunner = path.join(workspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs"); + const cliEntry = path.join(workspaceCwd, "cli", "src", "index.ts"); + for (const dir of [configDir, path.dirname(sourceConfigPath), path.dirname(cliRunner), path.dirname(cliEntry)]) { + fs.mkdirSync(dir, { recursive: true }); + } + fs.writeFileSync(path.join(configDir, "config.json"), "{}\n"); + fs.writeFileSync(sourceConfigPath, "{}\n"); + fs.writeFileSync(cliRunner, "// test runner\n"); + fs.writeFileSync(cliEntry, "// test entry\n"); + fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: sourceConfigPath }, + state: "failed", + attemptId: "previous", + })); + mockExecutionWorkspaceService.getById.mockResolvedValue(buildExecutionWorkspace({ cwd: workspaceCwd })); + mockEnsurePersistedExecutionWorkspaceAvailable.mockResolvedValue({ cwd: workspaceCwd }); + mockSpawn.mockImplementation(() => { + const child = new EventEmitter() as EventEmitter & { + stdout: PassThrough; + stderr: PassThrough; + kill: ReturnType; + }; + child.stdout = new PassThrough(); + child.stderr = new PassThrough(); + child.kill = vi.fn(); + queueMicrotask(() => { + fs.writeFileSync(path.join(configDir, "seed-manifest.json"), JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: sourceConfigPath }, + state: "failed", + phase: "restore", + attemptId: "repair-attempt", + })); + child.emit("exit", 4); + }); + return child; + }); + + const progress: Array> = []; + mockWorkspaceOperationService.createRecorder.mockReturnValue({ + attachExecutionWorkspaceId: vi.fn(), + recordOperation: async (input: any) => { + await input.run(async (entry: Record) => { + progress.push(entry); + }); + }, + }); + const app = await createApp(); + const res = await request(app) + .post(`/api/execution-workspaces/${executionWorkspaceId}/runtime-commands/repair`) + .send({}); + + expect(res.status).toBeGreaterThanOrEqual(400); + expect(progress).toEqual(expect.arrayContaining([ + expect.objectContaining({ metadata: { seedFailurePhase: "restore" } }), + ])); + expect(mockStartRuntimeServices).not.toHaveBeenCalled(); + expect(mockStopRuntimeServicesForExecutionWorkspace).toHaveBeenCalledTimes(2); + } finally { + fs.rmSync(workspaceCwd, { recursive: true, force: true }); + } + }); + it("reconciles once when the operation's own time budget fails a start that never settles", async () => { const { WorkspaceOperationTimeoutError } = await import("../services/workspace-operations.js"); // The recorder's ceiling fires outside `run`, so only the outer handler can reconcile. diff --git a/server/src/__tests__/execution-workspaces-routes.test.ts b/server/src/__tests__/execution-workspaces-routes.test.ts index 1d113f46bfe..89111b9b0aa 100644 --- a/server/src/__tests__/execution-workspaces-routes.test.ts +++ b/server/src/__tests__/execution-workspaces-routes.test.ts @@ -19,6 +19,7 @@ const mockExecutionWorkspaceService = vi.hoisted(() => ({ const mockWorkspaceOperationService = vi.hoisted(() => ({ listForExecutionWorkspace: vi.fn(), createRecorder: vi.fn(), + assertRuntimeControlAvailable: vi.fn(async () => undefined), })); const mockWorkspaceRuntimeLeaseService = vi.hoisted(() => ({ @@ -47,7 +48,7 @@ vi.mock("../services/index.js", () => ({ logActivity: mockLogActivity, workspaceOperationService: () => mockWorkspaceOperationService, workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService, - LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"], + LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"], })); vi.mock("../services/environment-runtime.js", () => ({ diff --git a/server/src/__tests__/workspace-operations-reconciliation.test.ts b/server/src/__tests__/workspace-operations-reconciliation.test.ts index 9f2808a9a1c..9133fbae0e6 100644 --- a/server/src/__tests__/workspace-operations-reconciliation.test.ts +++ b/server/src/__tests__/workspace-operations-reconciliation.test.ts @@ -44,14 +44,14 @@ describe("workspace runtime control serialization", () => { await expect(runExclusiveWorkspaceRuntimeControl({ executionWorkspaceId: "workspace-1", - action: "restart", - run: async () => "restarted", + action: "repair", + run: async () => "repaired", })).rejects.toMatchObject({ status: 409, details: { code: "workspace_runtime_control_in_progress", activeAction: "start", - requestedAction: "restart", + requestedAction: "repair", }, }); @@ -204,4 +204,71 @@ describeEmbeddedPostgres("workspace operation reconciliation", () => { .then((rows) => rows[0]); expect(unrelated?.status).toBe("running"); }); + + it("persists bounded repair phase progress on the operation row and log", async () => { + const companyId = randomUUID(); + const projectId = randomUUID(); + const executionWorkspaceId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Workspace repair diagnostics", + issuePrefix: `R${companyId.replace(/-/g, "").slice(0, 7).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(projects).values({ + id: projectId, + companyId, + name: "Repair diagnostics", + status: "in_progress", + }); + await db.insert(executionWorkspaces).values({ + id: executionWorkspaceId, + companyId, + projectId, + mode: "isolated_workspace", + strategyType: "git_worktree", + name: "Repair diagnostics workspace", + status: "active", + cwd: "/tmp/repair-diagnostics-workspace", + }); + + const operation = await workspaceOperationService(db) + .createRecorder({ companyId, executionWorkspaceId }) + .recordOperation({ + phase: "workspace_repair", + command: "workspace command repair", + cwd: "/tmp/repair-diagnostics-workspace", + metadata: { action: "repair" }, + run: async (reportProgress) => { + await reportProgress({ + metadata: { + repairPhase: "target_backup", + repairDiagnostics: [{ + phase: "target_backup", + status: "succeeded", + at: "2026-08-18T00:00:00.000Z", + }], + databaseOnly: true, + worktreePreserved: true, + }, + system: "Workspace repair target_backup: succeeded.\n", + }); + return { status: "succeeded", metadata: { backupRetained: true } }; + }, + }); + + expect(operation).toMatchObject({ + status: "succeeded", + metadata: { + action: "repair", + repairPhase: "target_backup", + databaseOnly: true, + worktreePreserved: true, + backupRetained: true, + }, + }); + expect((await workspaceOperationService(db).readLog(operation.id)).content).toContain( + "Workspace repair target_backup: succeeded.", + ); + }); }); diff --git a/server/src/__tests__/workspace-runtime-routes-authz.test.ts b/server/src/__tests__/workspace-runtime-routes-authz.test.ts index 3b7906cd867..64bfce7f9c2 100644 --- a/server/src/__tests__/workspace-runtime-routes-authz.test.ts +++ b/server/src/__tests__/workspace-runtime-routes-authz.test.ts @@ -54,7 +54,7 @@ vi.mock("../services/index.js", () => ({ secretService: () => mockSecretService, workspaceOperationService: () => mockWorkspaceOperationService, workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService, - LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"], + LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"], })); vi.mock("../services/workspace-runtime.js", () => ({ @@ -84,7 +84,7 @@ function registerWorkspaceRouteMocks() { secretService: () => mockSecretService, workspaceOperationService: () => mockWorkspaceOperationService, workspaceRuntimeLeaseService: () => mockWorkspaceRuntimeLeaseService, - LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart"], + LEASED_WORKSPACE_RUNTIME_ACTIONS: ["start", "stop", "restart", "repair"], })); vi.doMock("../services/workspace-runtime.js", () => ({ diff --git a/server/src/__tests__/workspace-runtime.test.ts b/server/src/__tests__/workspace-runtime.test.ts index b96d455a05e..a2adbeab014 100644 --- a/server/src/__tests__/workspace-runtime.test.ts +++ b/server/src/__tests__/workspace-runtime.test.ts @@ -456,6 +456,39 @@ describe("resolveRuntimeProvisionCommand", () => { await fs.writeFile(path.join(cwd, ".paperclip", "seed-complete"), "{}\n"); expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(""); + + await fs.writeFile( + path.join(cwd, ".paperclip", "seed-manifest.json"), + JSON.stringify({ version: 2, state: "failed" }), + ); + expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe( + "bash ./scripts/provision-worktree-runtime.sh", + ); + await fs.writeFile( + path.join(cwd, ".paperclip", "seed-manifest.json"), + JSON.stringify({ version: 2, state: "verified" }), + ); + expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe( + "bash ./scripts/provision-worktree-runtime.sh", + ); + await fs.writeFile( + path.join(cwd, ".paperclip", "seed-manifest.json"), + JSON.stringify({ + version: 2, + source: { instanceId: "source", configPath: "/source/config.json" }, + snapshotAt: "2026-08-18T00:00:00.000Z", + seedMode: "full", + migrationRevision: "0001", + targetInstanceId: "target", + phase: "complete", + state: "verified", + attemptId: "attempt", + startedAt: "2026-08-18T00:00:00.000Z", + finishedAt: "2026-08-18T00:01:00.000Z", + diagnostics: [{ phase: "complete", status: "succeeded", at: "2026-08-18T00:01:00.000Z" }], + }), + ); + expect(resolveRuntimeProvisionCommand({ config: {}, workspace })).toBe(""); } finally { await fs.rm(baseCwd, { recursive: true, force: true }); } diff --git a/server/src/dev-runner-worktree.ts b/server/src/dev-runner-worktree.ts index 800aa1e3158..b78e80fe70b 100644 --- a/server/src/dev-runner-worktree.ts +++ b/server/src/dev-runner-worktree.ts @@ -1,6 +1,7 @@ import { existsSync, lstatSync, readFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; +import { hasVerifiedWorktreeSeedManifest } from "./worktree-seed-manifest.js"; function parseEnvFile(contents: string): Record { const entries: Record = {}; @@ -58,6 +59,10 @@ export function resolveWorktreeEnvFilePath(rootDir: string): string { export function isWorktreeSeedPending(rootDir: string): boolean { const markerDir = path.resolve(rootDir, ".paperclip"); + const manifestPath = path.resolve(markerDir, "seed-manifest.json"); + if (existsSync(manifestPath)) { + return !hasVerifiedWorktreeSeedManifest(manifestPath); + } return existsSync(path.resolve(markerDir, "seed-pending")) && !existsSync(path.resolve(markerDir, "seed-complete")); } diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index 9477b2d0ac5..e195ad12433 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -1,3 +1,6 @@ +import { spawn } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; import { and, eq } from "drizzle-orm"; import { Router, type Request, type Response } from "express"; import type { Db } from "@paperclipai/db"; @@ -45,6 +48,8 @@ import { appendWithCap } from "../adapters/utils.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { runExclusiveWorkspaceRuntimeControl } from "../services/workspace-operations.js"; +import { deriveWorktreeInstanceId } from "../services/workspace-instance-cleanup.js"; +import { isVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; const WORKSPACE_CONTROL_OUTPUT_MAX_CHARS = 256 * 1024; @@ -151,7 +156,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P async function handleExecutionWorkspaceRuntimeCommand(req: Request, res: Response) { const id = req.params.id as string; const action = String(req.params.action ?? "").trim().toLowerCase(); - if (action !== "start" && action !== "stop" && action !== "restart" && action !== "run") { + if (action !== "start" && action !== "stop" && action !== "restart" && action !== "repair" && action !== "run") { res.status(404).json({ error: "Workspace command action not found" }); return; } @@ -262,7 +267,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P return; } - if ((action === "start" || action === "restart") && !effectiveRuntimeConfig) { + if ((action === "start" || action === "restart" || action === "repair") && !effectiveRuntimeConfig) { res.status(422).json({ error: "Execution workspace has no workspace command configuration or inherited project workspace default" }); return; } @@ -327,7 +332,11 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P } const recordRuntimeControlOperation = () => recorder.recordOperation({ - phase: action === "stop" ? "workspace_teardown" : "workspace_provision", + phase: action === "stop" + ? "workspace_teardown" + : action === "repair" + ? "workspace_repair" + : "workspace_provision", command: workspaceCommand?.command ?? `workspace command ${action}`, cwd: existing.cwd, metadata: { @@ -339,7 +348,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P runtimeServiceId: selectedRuntimeServiceId, serviceIndex: selectedServiceIndex, }, - run: async () => { + run: async (reportProgress) => { const ensureWorkspaceAvailable = async () => await ensurePersistedExecutionWorkspaceAvailable({ base: { @@ -429,7 +438,234 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P else stderr = appendWithCap(stderr, chunk, WORKSPACE_CONTROL_OUTPUT_MAX_CHARS); }; - if (action === "stop" || action === "restart") { + if (action === "repair") { + type RepairPhase = + | "managed_stop" + | "target_backup" + | "full_reseed" + | "managed_restart" + | "readiness_validation"; + const repairDiagnostics: Array<{ + phase: RepairPhase; + status: "started" | "succeeded" | "failed"; + at: string; + }> = []; + let repairPhase: RepairPhase = "managed_stop"; + const reportRepairPhase = async ( + phase: RepairPhase, + status: "started" | "succeeded" | "failed", + ) => { + repairPhase = phase; + repairDiagnostics.push({ phase, status, at: new Date().toISOString() }); + if (repairDiagnostics.length > 32) repairDiagnostics.splice(0, repairDiagnostics.length - 32); + await reportProgress({ + metadata: { + repairPhase, + repairDiagnostics: [...repairDiagnostics], + databaseOnly: true, + worktreePreserved: true, + }, + system: `Workspace repair ${phase}: ${status}.\n`, + }); + }; + + try { + await reportRepairPhase("managed_stop", "started"); + await stopRuntimeServicesForExecutionWorkspace({ + db, + executionWorkspaceId: existing.id, + workspaceCwd, + }); + await reportRepairPhase("managed_stop", "succeeded"); + + const manifestPath = path.join(workspaceCwd, ".paperclip", "seed-manifest.json"); + const targetConfigPath = path.join(workspaceCwd, ".paperclip", "config.json"); + let sourceConfigPath: string | null = null; + let previousAttemptId: string | null = null; + if (existsSync(manifestPath)) { + try { + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + attemptId?: unknown; + source?: { configPath?: unknown }; + }; + previousAttemptId = typeof manifest.attemptId === "string" ? manifest.attemptId : null; + sourceConfigPath = typeof manifest.source?.configPath === "string" + ? path.resolve(manifest.source.configPath) + : null; + } catch { + throw new Error("Workspace seed manifest is malformed; repair source identity cannot be trusted."); + } + } + const baseWorkspaceCwd = projectWorkspace?.cwd ?? workspaceCwd; + const baseConfigPath = path.join(baseWorkspaceCwd, ".paperclip", "config.json"); + if (!sourceConfigPath && existsSync(baseConfigPath) && path.resolve(baseConfigPath) !== path.resolve(targetConfigPath)) { + sourceConfigPath = baseConfigPath; + } + if (!sourceConfigPath || !existsSync(sourceConfigPath)) { + throw new Error("Workspace repair has no readable configured source instance."); + } + + const cliRunner = path.join(baseWorkspaceCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs"); + const cliEntry = path.join(baseWorkspaceCwd, "cli", "src", "index.ts"); + const cliDist = path.join(baseWorkspaceCwd, "cli", "dist", "index.js"); + const cliArgs = existsSync(cliRunner) && existsSync(cliEntry) + ? [cliRunner, cliEntry] + : existsSync(cliDist) + ? [cliDist] + : null; + if (!cliArgs) { + throw new Error("Workspace repair cannot find a runnable Paperclip CLI in the base workspace."); + } + + await reportRepairPhase("target_backup", "started"); + const child = spawn(process.execPath, [ + ...cliArgs, + "worktree", + "reseed", + "--from-config", + sourceConfigPath, + "--to", + workspaceCwd, + "--seed-mode", + "full", + "--yes", + "--backup-target", + ], { + cwd: baseWorkspaceCwd, + env: process.env, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout?.on("data", (chunk: Buffer) => { + void onLog("stdout", chunk.toString("utf8")); + }); + child.stderr?.on("data", (chunk: Buffer) => { + void onLog("stderr", chunk.toString("utf8")); + }); + let repairCommandTimedOut = false; + let repairCommandForceTimeout: NodeJS.Timeout | null = null; + const repairCommandTimeout = setTimeout(() => { + repairCommandTimedOut = true; + child.kill("SIGTERM"); + repairCommandForceTimeout = setTimeout(() => child.kill("SIGKILL"), 5_000); + repairCommandForceTimeout.unref?.(); + }, 20 * 60_000); + repairCommandTimeout.unref?.(); + + let reseedObserved = false; + const manifestPoll = setInterval(() => { + if (reseedObserved || !existsSync(manifestPath)) return; + try { + const current = JSON.parse(readFileSync(manifestPath, "utf8")) as { + attemptId?: unknown; + state?: unknown; + }; + if ( + typeof current.attemptId === "string" + && current.attemptId !== previousAttemptId + && (current.state === "pending" || current.state === "running" || current.state === "failed") + ) { + reseedObserved = true; + } + } catch { + // The atomic manifest writer makes this unlikely; the terminal + // read below remains authoritative. + } + }, 100); + manifestPoll.unref?.(); + const exitCode = await new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code ?? 1)); + }).finally(() => { + clearInterval(manifestPoll); + clearTimeout(repairCommandTimeout); + if (repairCommandForceTimeout) clearTimeout(repairCommandForceTimeout); + }); + if (reseedObserved) { + await reportRepairPhase("target_backup", "succeeded"); + await reportRepairPhase("full_reseed", "started"); + } + if (exitCode !== 0) { + let seedFailurePhase: string | null = null; + try { + const failedManifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + phase?: unknown; + state?: unknown; + }; + seedFailurePhase = failedManifest.state === "failed" && typeof failedManifest.phase === "string" + ? failedManifest.phase + : null; + } catch { + // The outer repair phase remains exact when no seed phase was persisted. + } + await reportProgress({ + metadata: { seedFailurePhase }, + system: seedFailurePhase ? `Workspace seed failed during ${seedFailurePhase}.\n` : null, + }); + throw new Error( + repairCommandTimedOut + ? `Workspace database repair command timed out during ${repairPhase}.` + : `Workspace database repair command failed during ${repairPhase}.`, + ); + } + if (!reseedObserved) { + await reportRepairPhase("target_backup", "succeeded"); + await reportRepairPhase("full_reseed", "started"); + } + + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record; + if (!isVerifiedWorktreeSeedManifest(manifest)) { + throw new Error("Workspace reseed returned without a verified terminal manifest."); + } + const expectedInstanceId = deriveWorktreeInstanceId(workspaceCwd); + if (manifest.targetInstanceId !== expectedInstanceId) { + throw new Error("Verified seed manifest belongs to a different workspace instance."); + } + await reportRepairPhase("full_reseed", "succeeded"); + + await reportRepairPhase("managed_restart", "started"); + const availableWorkspace = await ensureWorkspaceAvailable(); + if (!availableWorkspace) { + throw new Error("Execution workspace needs a local path before Paperclip can restart it."); + } + const startedServices = await startRuntimeServicesForWorkspaceControl({ + db, + actor: { + id: actor.agentId ?? null, + name: actor.actorType === "user" ? "Board" : "Agent", + companyId: existing.companyId, + }, + issue: existing.sourceIssueId + ? { id: existing.sourceIssueId, identifier: null, title: existing.name } + : null, + workspace: availableWorkspace, + executionWorkspaceId: existing.id, + config: { + workspaceRuntime: effectiveRuntimeConfig, + runtimeProvisionCommand: + existing.config?.runtimeProvisionCommand + ?? projectPolicy?.workspaceStrategy?.runtimeProvisionCommand + ?? null, + }, + adapterEnv: {}, + onLog, + recorder, + }); + runtimeServiceCount = startedServices.length; + await reportRepairPhase("managed_restart", "succeeded"); + + await reportRepairPhase("readiness_validation", "started"); + if ( + startedServices.length === 0 + || startedServices.some((service) => service.status !== "running" || service.healthStatus !== "healthy") + ) { + throw new Error("Managed restart did not produce healthy runtime services."); + } + await reportRepairPhase("readiness_validation", "succeeded"); + } catch (error) { + await reportRepairPhase(repairPhase, "failed"); + throw error; + } + } else if (action === "stop" || action === "restart") { await stopRuntimeServicesForExecutionWorkspace({ db, executionWorkspaceId: existing.id, @@ -481,7 +717,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P throw error; } runtimeServiceCount = startedServices.length; - } else { + } else if (action !== "repair") { runtimeServiceCount = selectedRuntimeServiceId ? Math.max(0, (existing.runtimeServices?.length ?? 1) - 1) : 0; } @@ -504,7 +740,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P config: { workspaceRuntime: effectiveRuntimeConfig }, currentDesiredState, currentServiceStates: existing.config?.serviceStates ?? null, - action, + action: action === "repair" ? "start" : action, serviceIndex: selectedServiceIndex, }); const metadata = mergeExecutionWorkspaceConfig(existing.metadata as Record | null, { @@ -522,6 +758,8 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P ? "Stopped execution workspace runtime services.\n" : action === "restart" ? "Restarted execution workspace runtime services.\n" + : action === "repair" + ? "Repaired the isolated workspace database and restarted healthy runtime services.\n" : "Started execution workspace runtime services.\n", metadata: { runtimeServiceCount, @@ -565,7 +803,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P // The operation is already terminal here. This also catches the recorder's own time // budget expiring on a start that never settles, which is the one failure the inner // handler cannot see — reconcile there too so no listener or desired state is left over. - if (action === "start" || action === "restart") await reconcileFailedRuntimeStart(error); + if (action === "start" || action === "restart" || action === "repair") { + await reconcileFailedRuntimeStart(error); + } throw error; } }, diff --git a/server/src/services/workspace-operations.ts b/server/src/services/workspace-operations.ts index 384f5785ee8..773744a40d7 100644 --- a/server/src/services/workspace-operations.ts +++ b/server/src/services/workspace-operations.ts @@ -14,7 +14,7 @@ type WorkspaceOperationRow = typeof workspaceOperations.$inferSelect; * Managed runtime control actions. Every one of these mutates the runtime rows and * local listeners of a single execution workspace, so only one may be live at a time. */ -const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "run"]); +const RUNTIME_CONTROL_ACTIONS = new Set(["start", "stop", "restart", "repair", "run"]); /** * Identity of this server process. A `running` runtime-control operation stamped with a @@ -234,7 +234,10 @@ export interface WorkspaceOperationRecorder { * timeout error is rethrown, so a hung provider can never leave an active operation. */ timeoutMs?: number | null; - run: () => Promise<{ + run: (reportProgress: (input: { + metadata?: Record | null; + system?: string | null; + }) => Promise) => Promise<{ status?: WorkspaceOperationStatus; exitCode?: number | null; stdout?: string | null; @@ -484,7 +487,7 @@ export function workspaceOperationService(db: Db) { // Managed runtime controls get an ownership stamp so bounded recovery can tell a // slow-but-live operation from one abandoned by a dead request or server process. const runtimeControlAction = readRuntimeControlAction(recordInput.metadata); - const insertedMetadata = runtimeControlAction + let currentMetadata = runtimeControlAction ? { ...(recordInput.metadata ?? {}), runtimeControlOwner: { @@ -515,7 +518,7 @@ export function workspaceOperationService(db: Db) { logStore: handle.store, logRef: handle.logRef, metadata: redactCurrentUserValue( - insertedMetadata, + currentMetadata, currentUserRedactionOptions, ) as Record | null, startedAt, @@ -532,17 +535,21 @@ export function workspaceOperationService(db: Db) { if (runtimeControlAction) { heartbeatTimer = setInterval(() => { const heartbeatAt = new Date(); + currentMetadata = combineMetadata(currentMetadata, { + runtimeControlOwner: { + ownerId: RUNTIME_CONTROL_OWNER_ID, + pid: process.pid, + action: runtimeControlAction, + heartbeatAt: heartbeatAt.toISOString(), + } satisfies RuntimeControlOwnerStamp, + }); void db .update(workspaceOperations) .set({ - metadata: combineMetadata(insertedMetadata, { - runtimeControlOwner: { - ownerId: RUNTIME_CONTROL_OWNER_ID, - pid: process.pid, - action: runtimeControlAction, - heartbeatAt: heartbeatAt.toISOString(), - } satisfies RuntimeControlOwnerStamp, - }), + metadata: redactCurrentUserValue( + currentMetadata, + currentUserRedactionOptions, + ) as Record | null, updatedAt: heartbeatAt, }) .where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running"))) @@ -551,9 +558,29 @@ export function workspaceOperationService(db: Db) { heartbeatTimer.unref?.(); } + const reportProgress = async (progress: { + metadata?: Record | null; + system?: string | null; + }) => { + await append("system", progress.system ?? null); + currentMetadata = combineMetadata(currentMetadata, progress.metadata); + await db + .update(workspaceOperations) + .set({ + metadata: redactCurrentUserValue( + currentMetadata, + currentUserRedactionOptions, + ) as Record | null, + stdoutExcerpt: stdoutExcerpt || null, + stderrExcerpt: stderrExcerpt || null, + updatedAt: new Date(), + }) + .where(and(eq(workspaceOperations.id, id), eq(workspaceOperations.status, "running"))); + }; + const timeoutMs = recordInput.timeoutMs ?? defaultRuntimeControlTimeoutMs(runtimeControlAction); const settle = async () => { - if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(); + if (!timeoutMs || timeoutMs <= 0) return await recordInput.run(reportProgress); const timeout = new Promise((_resolve, reject) => { timeoutTimer = setTimeout( () => reject(new WorkspaceOperationTimeoutError(timeoutMs, runtimeControlAction)), @@ -561,7 +588,7 @@ export function workspaceOperationService(db: Db) { ); timeoutTimer.unref?.(); }); - return await Promise.race([recordInput.run(), timeout]); + return await Promise.race([recordInput.run(reportProgress), timeout]); }; try { @@ -583,7 +610,7 @@ export function workspaceOperationService(db: Db) { logSha256: finalized.sha256, logCompressed: finalized.compressed, metadata: redactCurrentUserValue( - combineMetadata(insertedMetadata, result.metadata), + combineMetadata(currentMetadata, result.metadata), currentUserRedactionOptions, ) as Record | null, finishedAt, @@ -613,7 +640,7 @@ export function workspaceOperationService(db: Db) { ...(runtimeControlAction ? { metadata: redactCurrentUserValue( - combineMetadata(insertedMetadata, { + combineMetadata(currentMetadata, { failureReason: error instanceof WorkspaceOperationTimeoutError ? "runtime_control_timeout" : "runtime_control_error", diff --git a/server/src/services/workspace-runtime-leases.ts b/server/src/services/workspace-runtime-leases.ts index 79e8fe9c6c8..eba79d341d6 100644 --- a/server/src/services/workspace-runtime-leases.ts +++ b/server/src/services/workspace-runtime-leases.ts @@ -28,7 +28,7 @@ const TERMINAL_HEARTBEAT_RUN_STATUSES = new Set([ ]); /** Runtime control actions that mutate the workspace and therefore need the lease. */ -export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart"]; +export const LEASED_WORKSPACE_RUNTIME_ACTIONS: readonly string[] = ["start", "stop", "restart", "repair"]; /** * Upper bound on how long a lease survives without the owner touching it. Recovery diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 3753fe60eb1..d024758dea5 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -36,6 +36,7 @@ import { and, desc, eq, gte, inArray, isNull, lte, ne, or } from "drizzle-orm"; import { asNumber, asString, parseObject, renderTemplate } from "../adapters/utils.js"; import { conflict } from "../errors.js"; import { resolveHomeAwarePath } from "../home-paths.js"; +import { hasVerifiedWorktreeSeedManifest } from "../worktree-seed-manifest.js"; import { createLocalServiceKey, findLocalServiceRegistryRecordByRuntimeServiceId, @@ -5015,6 +5016,7 @@ export function resolveRuntimeProvisionCommand(input: { if (input.workspace.strategy !== "git_worktree") return ""; const stateDir = path.join(input.workspace.cwd, ".paperclip"); + const manifestPath = path.join(stateDir, "seed-manifest.json"); const pendingMarker = path.join(stateDir, "seed-pending"); const completeMarker = path.join(stateDir, "seed-complete"); const provisionScript = path.join( @@ -5022,11 +5024,11 @@ export function resolveRuntimeProvisionCommand(input: { "scripts", "provision-worktree-runtime.sh", ); - if ( - !existsSync(pendingMarker) - || existsSync(completeMarker) - || !existsSync(provisionScript) - ) { + let needsSeed = existsSync(pendingMarker) && !existsSync(completeMarker); + if (existsSync(manifestPath)) { + needsSeed = !hasVerifiedWorktreeSeedManifest(manifestPath); + } + if (!needsSeed || !existsSync(provisionScript)) { return ""; } diff --git a/server/src/worktree-seed-manifest.ts b/server/src/worktree-seed-manifest.ts new file mode 100644 index 00000000000..d1b281ed894 --- /dev/null +++ b/server/src/worktree-seed-manifest.ts @@ -0,0 +1,54 @@ +import { existsSync, readFileSync } from "node:fs"; + +const WORKTREE_SEED_MODES = new Set(["minimal", "full"]); + +type SeedDiagnostic = { + phase?: unknown; + status?: unknown; + at?: unknown; +}; + +/** + * A manifest is readiness evidence only when the complete validation phase was + * durably recorded. A bare `state: verified` is deliberately insufficient so + * truncated or hand-edited files cannot start a partially restored workspace. + */ +export function isVerifiedWorktreeSeedManifest(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + const manifest = value as Record; + const source = manifest.source as Record | null; + const diagnostics = manifest.diagnostics as SeedDiagnostic[] | null; + return manifest.version === 2 + && manifest.state === "verified" + && manifest.phase === "complete" + && typeof source?.instanceId === "string" + && source.instanceId.length > 0 + && typeof source.configPath === "string" + && source.configPath.length > 0 + && WORKTREE_SEED_MODES.has(String(manifest.seedMode ?? "")) + && typeof manifest.snapshotAt === "string" + && manifest.snapshotAt.length > 0 + && typeof manifest.migrationRevision === "string" + && manifest.migrationRevision.length > 0 + && typeof manifest.targetInstanceId === "string" + && manifest.targetInstanceId.length > 0 + && typeof manifest.attemptId === "string" + && manifest.attemptId.length > 0 + && typeof manifest.startedAt === "string" + && typeof manifest.finishedAt === "string" + && Array.isArray(diagnostics) + && diagnostics.some((diagnostic) => ( + diagnostic?.phase === "complete" + && diagnostic.status === "succeeded" + && typeof diagnostic.at === "string" + )); +} + +export function hasVerifiedWorktreeSeedManifest(manifestPath: string): boolean { + if (!existsSync(manifestPath)) return false; + try { + return isVerifiedWorktreeSeedManifest(JSON.parse(readFileSync(manifestPath, "utf8"))); + } catch { + return false; + } +} diff --git a/ui/src/api/execution-workspaces.ts b/ui/src/api/execution-workspaces.ts index e6c075a86d1..b3de8342297 100644 --- a/ui/src/api/execution-workspaces.ts +++ b/ui/src/api/execution-workspaces.ts @@ -114,6 +114,11 @@ export const executionWorkspacesApi = { `/execution-workspaces/${id}/runtime-commands/${action}`, sanitizeWorkspaceRuntimeControlTarget(target), ), + repair: (id: string) => + api.post<{ workspace: ExecutionWorkspace; operation: WorkspaceOperation }>( + `/execution-workspaces/${id}/runtime-commands/repair`, + {}, + ), update: (id: string, data: Record) => api.patch(`/execution-workspaces/${id}`, data), /** * Reconcile a git-worktree branch divergence via the S4 (`PAP-1586`) op.