diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index ce0f85ad530..98e2da7769c 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -3,12 +3,15 @@ import os from "node:os"; import path from "node:path"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, createDb, + issueComments, + issues, projects, routines, routineTriggers, @@ -17,6 +20,7 @@ import { copyGitHooksToWorktreeGitDir, copySeededSecretsKey, pauseSeededScheduledRoutines, + quarantineSeededWorktreeExecutionState, readSourceAttachmentBody, rebindWorkspaceCwd, resolveSourceConfigPath, @@ -282,6 +286,138 @@ describe("worktree helpers", () => { expect(full.nullifyColumns).toEqual({}); }); + itEmbeddedPostgres("quarantines copied live execution state in seeded worktree databases", async () => { + const tempDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-quarantine-"); + const db = createDb(tempDb.connectionString); + const companyId = randomUUID(); + const agentId = randomUUID(); + const idleAgentId = randomUUID(); + const inProgressIssueId = randomUUID(); + const todoIssueId = randomUUID(); + const reviewIssueId = randomUUID(); + const userIssueId = randomUUID(); + + try { + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: "WTQ", + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values([ + { + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "running", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { + heartbeat: { enabled: true, intervalSec: 60 }, + wakeOnDemand: true, + }, + permissions: {}, + }, + { + id: idleAgentId, + companyId, + name: "Reviewer", + role: "reviewer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: { heartbeat: { enabled: false, intervalSec: 300 } }, + permissions: {}, + }, + ]); + await db.insert(issues).values([ + { + id: inProgressIssueId, + companyId, + title: "Copied in-flight issue", + status: "in_progress", + priority: "medium", + assigneeAgentId: agentId, + issueNumber: 1, + identifier: "WTQ-1", + executionAgentNameKey: "codexcoder", + executionLockedAt: new Date("2026-04-18T00:00:00.000Z"), + }, + { + id: todoIssueId, + companyId, + title: "Copied assigned todo issue", + status: "todo", + priority: "medium", + assigneeAgentId: agentId, + issueNumber: 2, + identifier: "WTQ-2", + }, + { + id: reviewIssueId, + companyId, + title: "Copied assigned review issue", + status: "in_review", + priority: "medium", + assigneeAgentId: idleAgentId, + issueNumber: 3, + identifier: "WTQ-3", + }, + { + id: userIssueId, + companyId, + title: "Copied user issue", + status: "todo", + priority: "medium", + assigneeUserId: "user-1", + issueNumber: 4, + identifier: "WTQ-4", + }, + ]); + + await expect(quarantineSeededWorktreeExecutionState(tempDb.connectionString)).resolves.toEqual({ + disabledTimerHeartbeats: 1, + resetRunningAgents: 1, + quarantinedInProgressIssues: 1, + unassignedTodoIssues: 1, + unassignedReviewIssues: 1, + }); + + const [quarantinedAgent] = await db.select().from(agents).where(eq(agents.id, agentId)); + expect(quarantinedAgent?.status).toBe("idle"); + expect(quarantinedAgent?.runtimeConfig).toMatchObject({ + heartbeat: { enabled: false, intervalSec: 60 }, + wakeOnDemand: true, + }); + + const [inProgressIssue] = await db.select().from(issues).where(eq(issues.id, inProgressIssueId)); + expect(inProgressIssue?.status).toBe("blocked"); + expect(inProgressIssue?.assigneeAgentId).toBeNull(); + expect(inProgressIssue?.executionAgentNameKey).toBeNull(); + expect(inProgressIssue?.executionLockedAt).toBeNull(); + + const [todoIssue] = await db.select().from(issues).where(eq(issues.id, todoIssueId)); + expect(todoIssue?.status).toBe("todo"); + expect(todoIssue?.assigneeAgentId).toBeNull(); + + const [reviewIssue] = await db.select().from(issues).where(eq(issues.id, reviewIssueId)); + expect(reviewIssue?.status).toBe("in_review"); + expect(reviewIssue?.assigneeAgentId).toBeNull(); + + const [userIssue] = await db.select().from(issues).where(eq(issues.id, userIssueId)); + expect(userIssue?.status).toBe("todo"); + expect(userIssue?.assigneeUserId).toBe("user-1"); + + const comments = await db.select().from(issueComments).where(eq(issueComments.issueId, inProgressIssueId)); + expect(comments).toHaveLength(1); + expect(comments[0]?.body).toContain("Quarantined during worktree seed"); + } finally { + await db.$client?.end?.({ timeout: 5 }).catch(() => undefined); + await tempDb.cleanup(); + } + }, 20_000); + it("copies the source local_encrypted secrets key into the seeded worktree instance", () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-secrets-")); const originalInlineMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index d42cd9f3dbb..bff59f50d9f 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -93,6 +93,7 @@ type WorktreeInitOptions = { dbPort?: number; seed?: boolean; seedMode?: string; + preserveLiveWork?: boolean; force?: boolean; }; @@ -126,6 +127,7 @@ type WorktreeReseedOptions = { fromDataDir?: string; fromInstance?: string; seedMode?: string; + preserveLiveWork?: boolean; yes?: boolean; allowLiveTarget?: boolean; }; @@ -137,6 +139,7 @@ type WorktreeRepairOptions = { fromDataDir?: string; fromInstance?: string; seedMode?: string; + preserveLiveWork?: boolean; noSeed?: boolean; allowLiveTarget?: boolean; }; @@ -179,6 +182,8 @@ type CopiedGitHooksResult = { type SeedWorktreeDatabaseResult = { backupSummary: string; + pausedScheduledRoutines: number; + executionQuarantine: SeededWorktreeExecutionQuarantineSummary; reboundWorkspaces: Array<{ name: string; fromCwd: string; @@ -186,6 +191,14 @@ type SeedWorktreeDatabaseResult = { }>; }; +export type SeededWorktreeExecutionQuarantineSummary = { + disabledTimerHeartbeats: number; + resetRunningAgents: number; + quarantinedInProgressIssues: number; + unassignedTodoIssues: number; + unassignedReviewIssues: number; +}; + function nonEmpty(value: string | null | undefined): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } @@ -198,6 +211,18 @@ function isCurrentSourceConfigPath(sourceConfigPath: string): boolean { return path.resolve(currentConfigPath) === path.resolve(sourceConfigPath); } +function formatSeededWorktreeExecutionQuarantineSummary( + summary: SeededWorktreeExecutionQuarantineSummary, +): string { + return [ + `disabled timer heartbeats: ${summary.disabledTimerHeartbeats}`, + `reset running agents: ${summary.resetRunningAgents}`, + `quarantined in-progress issues: ${summary.quarantinedInProgressIssues}`, + `unassigned todo issues: ${summary.unassignedTodoIssues}`, + `unassigned review issues: ${summary.unassignedReviewIssues}`, + ].join(", "); +} + const WORKTREE_NAME_PREFIX = "paperclip-"; function resolveWorktreeMakeName(name: string): string { @@ -1119,6 +1144,133 @@ export async function pauseSeededScheduledRoutines(connectionString: string): Pr } } +const EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY: SeededWorktreeExecutionQuarantineSummary = { + disabledTimerHeartbeats: 0, + resetRunningAgents: 0, + quarantinedInProgressIssues: 0, + unassignedTodoIssues: 0, + unassignedReviewIssues: 0, +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isEnabledValue(value: unknown): boolean { + return value === true || value === "true" || value === 1 || value === "1"; +} + +function normalizeWorktreeRuntimeConfig(runtimeConfig: unknown): { + runtimeConfig: Record; + disabledTimerHeartbeat: boolean; + changed: boolean; +} { + const nextRuntimeConfig = isRecord(runtimeConfig) ? { ...runtimeConfig } : {}; + const heartbeat = isRecord(nextRuntimeConfig.heartbeat) ? { ...nextRuntimeConfig.heartbeat } : null; + if (!heartbeat) { + return { runtimeConfig: nextRuntimeConfig, disabledTimerHeartbeat: false, changed: false }; + } + + const disabledTimerHeartbeat = isEnabledValue(heartbeat.enabled); + if (heartbeat.enabled !== false) { + heartbeat.enabled = false; + nextRuntimeConfig.heartbeat = heartbeat; + return { runtimeConfig: nextRuntimeConfig, disabledTimerHeartbeat, changed: true }; + } + + return { runtimeConfig: nextRuntimeConfig, disabledTimerHeartbeat: false, changed: false }; +} + +export async function quarantineSeededWorktreeExecutionState( + connectionString: string, +): Promise { + const db = createDb(connectionString); + const summary = { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }; + try { + await db.transaction(async (tx) => { + const seededAgents = await tx + .select({ + id: agents.id, + status: agents.status, + runtimeConfig: agents.runtimeConfig, + }) + .from(agents); + + for (const agent of seededAgents) { + const normalized = normalizeWorktreeRuntimeConfig(agent.runtimeConfig); + const nextStatus = agent.status === "running" ? "idle" : agent.status; + if (normalized.disabledTimerHeartbeat) { + summary.disabledTimerHeartbeats += 1; + } + if (agent.status === "running") { + summary.resetRunningAgents += 1; + } + if (normalized.changed || nextStatus !== agent.status) { + await tx + .update(agents) + .set({ + runtimeConfig: normalized.runtimeConfig, + status: nextStatus, + updatedAt: new Date(), + }) + .where(eq(agents.id, agent.id)); + } + } + + const affectedIssues = await tx + .select({ + id: issues.id, + companyId: issues.companyId, + status: issues.status, + }) + .from(issues) + .where( + and( + sql`${issues.assigneeAgentId} is not null`, + sql`${issues.assigneeUserId} is null`, + inArray(issues.status, ["todo", "in_progress", "in_review"]), + ), + ); + + for (const issue of affectedIssues) { + const nextStatus = issue.status === "in_progress" ? "blocked" : issue.status; + await tx + .update(issues) + .set({ + status: nextStatus, + assigneeAgentId: null, + checkoutRunId: null, + executionRunId: null, + executionAgentNameKey: null, + executionLockedAt: null, + executionWorkspaceId: null, + updatedAt: new Date(), + }) + .where(eq(issues.id, issue.id)); + + if (issue.status === "in_progress") { + summary.quarantinedInProgressIssues += 1; + await tx.insert(issueComments).values({ + companyId: issue.companyId, + issueId: issue.id, + body: + "Quarantined during worktree seed so copied in-flight work does not auto-run in this isolated instance. " + + "Reassign or unblock here only if you intentionally want the worktree instance to own this task.", + }); + } else if (issue.status === "todo") { + summary.unassignedTodoIssues += 1; + } else if (issue.status === "in_review") { + summary.unassignedReviewIssues += 1; + } + } + }); + + return summary; + } finally { + await db.$client?.end?.({ timeout: 5 }).catch(() => undefined); + } +} + async function seedWorktreeDatabase(input: { sourceConfigPath: string; sourceConfig: PaperclipConfig; @@ -1126,6 +1278,7 @@ async function seedWorktreeDatabase(input: { targetPaths: WorktreeLocalPaths; instanceId: string; seedMode: WorktreeSeedMode; + preserveLiveWork?: boolean; }): Promise { const seedPlan = resolveWorktreeSeedPlan(input.seedMode); const sourceEnvFile = resolvePaperclipEnvFile(input.sourceConfigPath); @@ -1176,7 +1329,10 @@ async function seedWorktreeDatabase(input: { backupFile: backup.backupFile, }); await applyPendingMigrations(targetConnectionString); - await pauseSeededScheduledRoutines(targetConnectionString); + const executionQuarantine = input.preserveLiveWork + ? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY } + : await quarantineSeededWorktreeExecutionState(targetConnectionString); + const pausedScheduledRoutines = await pauseSeededScheduledRoutines(targetConnectionString); const reboundWorkspaces = await rebindSeededProjectWorkspaces({ targetConnectionString, currentCwd: input.targetPaths.cwd, @@ -1184,6 +1340,8 @@ async function seedWorktreeDatabase(input: { return { backupSummary: formatDatabaseBackupResult(backup), + pausedScheduledRoutines, + executionQuarantine, reboundWorkspaces, }; } finally { @@ -1262,6 +1420,8 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { const copiedGitHooks = copyGitHooksToWorktreeGitDir(cwd); let seedSummary: string | null = null; + let seedExecutionQuarantineSummary: SeededWorktreeExecutionQuarantineSummary | null = null; + let pausedScheduledRoutineCount: number | null = null; let reboundWorkspaceSummary: SeedWorktreeDatabaseResult["reboundWorkspaces"] = []; if (opts.seed !== false) { if (!sourceConfig) { @@ -1279,8 +1439,11 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { targetPaths: paths, instanceId, seedMode, + preserveLiveWork: opts.preserveLiveWork, }); seedSummary = seeded.backupSummary; + seedExecutionQuarantineSummary = seeded.executionQuarantine; + pausedScheduledRoutineCount = seeded.pausedScheduledRoutines; reboundWorkspaceSummary = seeded.reboundWorkspaces; spinner.stop(`Seeded isolated worktree database (${seedMode}).`); } catch (error) { @@ -1303,6 +1466,16 @@ async function runWorktreeInit(opts: WorktreeInitOptions): Promise { if (seedSummary) { p.log.message(pc.dim(`Seed mode: ${seedMode}`)); p.log.message(pc.dim(`Seed snapshot: ${seedSummary}`)); + if (opts.preserveLiveWork) { + p.log.warning("Preserved copied live work; this worktree instance may auto-run source-instance assignments."); + } else if (seedExecutionQuarantineSummary) { + p.log.message( + pc.dim(`Seed execution quarantine: ${formatSeededWorktreeExecutionQuarantineSummary(seedExecutionQuarantineSummary)}`), + ); + } + if (pausedScheduledRoutineCount != null) { + p.log.message(pc.dim(`Paused scheduled routines: ${pausedScheduledRoutineCount}`)); + } for (const rebound of reboundWorkspaceSummary) { p.log.message( pc.dim(`Rebound workspace ${rebound.name}: ${rebound.fromCwd} -> ${rebound.toCwd}`), @@ -2947,11 +3120,20 @@ async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { targetPaths, instanceId: targetPaths.instanceId, seedMode, + preserveLiveWork: opts.preserveLiveWork, }); spinner.stop(`Reseeded ${targetEndpoint.label} (${seedMode}).`); p.log.message(pc.dim(`Source: ${source.configPath}`)); p.log.message(pc.dim(`Target: ${targetEndpoint.configPath}`)); p.log.message(pc.dim(`Seed snapshot: ${seeded.backupSummary}`)); + if (opts.preserveLiveWork) { + p.log.warning("Preserved copied live work; this worktree instance may auto-run source-instance assignments."); + } else { + p.log.message( + pc.dim(`Seed execution quarantine: ${formatSeededWorktreeExecutionQuarantineSummary(seeded.executionQuarantine)}`), + ); + } + p.log.message(pc.dim(`Paused scheduled routines: ${seeded.pausedScheduledRoutines}`)); for (const rebound of seeded.reboundWorkspaces) { p.log.message( pc.dim(`Rebound workspace ${rebound.name}: ${rebound.fromCwd} -> ${rebound.toCwd}`), @@ -3015,6 +3197,7 @@ export async function worktreeRepairCommand(opts: WorktreeRepairOptions): Promis fromConfig: source.configPath, to: target.rootPath, seedMode, + preserveLiveWork: opts.preserveLiveWork, yes: true, allowLiveTarget: opts.allowLiveTarget, }); @@ -3047,6 +3230,7 @@ export async function worktreeRepairCommand(opts: WorktreeRepairOptions): Promis fromInstance: opts.fromInstance, seed: opts.noSeed ? false : true, seedMode, + preserveLiveWork: opts.preserveLiveWork, force: true, }); } finally { @@ -3070,6 +3254,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--server-port ", "Preferred server port", (value) => Number(value)) .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") + .option("--preserve-live-work", "Do not quarantine copied agent timers or assigned open issues in the seeded worktree", false) .option("--no-seed", "Skip database seeding from the source instance") .option("--force", "Replace existing repo-local config and isolated instance data", false) .action(worktreeMakeCommand); @@ -3086,6 +3271,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--server-port ", "Preferred server port", (value) => Number(value)) .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") + .option("--preserve-live-work", "Do not quarantine copied agent timers or assigned open issues in the seeded worktree", false) .option("--no-seed", "Skip database seeding from the source instance") .option("--force", "Replace existing repo-local config and isolated instance data", false) .action(worktreeInitCommand); @@ -3125,6 +3311,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--from-data-dir ", "Source PAPERCLIP_HOME used when deriving the source config") .option("--from-instance ", "Source instance id when deriving the source config") .option("--seed-mode ", "Seed profile: minimal or full (default: full)", "full") + .option("--preserve-live-work", "Do not quarantine copied agent timers or assigned open issues in the seeded worktree", false) .option("--yes", "Skip the destructive confirmation prompt", false) .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false) .action(worktreeReseedCommand); @@ -3138,6 +3325,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--from-data-dir ", "Source PAPERCLIP_HOME used when deriving the source config") .option("--from-instance ", "Source instance id when deriving the source config (default: default)") .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") + .option("--preserve-live-work", "Do not quarantine copied agent timers or assigned open issues in the seeded worktree", false) .option("--no-seed", "Repair metadata only and skip reseeding when bootstrapping a missing worktree config", false) .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false) .action(worktreeRepairCommand); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7958ffd19cc..62c42ed661b 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -209,6 +209,8 @@ Seed modes: - `full` makes a full logical clone of the source instance - `--no-seed` creates an empty isolated instance +Seeded worktree instances quarantine copied live execution by default for both `minimal` and `full` seeds. During restore, Paperclip disables copied agent timer heartbeats, resets copied `running` agents to `idle`, blocks and unassigns copied agent-owned `in_progress` issues, and unassigns copied agent-owned `todo`/`in_review` issues. This keeps a freshly booted worktree from starting agents for work already owned by the source instance. Pass `--preserve-live-work` only when you intentionally want the isolated worktree to resume copied assignments. + After `worktree init`, both the server and the CLI auto-load the repo-local `.paperclip/.env` when run inside that worktree, so normal commands like `pnpm dev`, `paperclipai doctor`, and `paperclipai db:backup` stay scoped to the worktree instance. `pnpm dev` now fails fast in a linked git worktree when `.paperclip/.env` is missing, instead of silently booting against the default instance/port. If that happens, run `paperclipai worktree init` in the worktree first. diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 0bd61ba0422..da1427b7089 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -629,6 +629,56 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { } }); + it("does not continue seeded in-progress work that has no run linkage", async () => { + const companyId = randomUUID(); + const agentId = randomUUID(); + const issueId = randomUUID(); + const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`; + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix, + requireBoardApprovalForNewAgents: false, + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "CodexCoder", + role: "engineer", + status: "idle", + adapterType: "codex_local", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + await db.insert(issues).values({ + id: issueId, + companyId, + title: "Seeded in-flight work", + status: "in_progress", + priority: "medium", + assigneeAgentId: agentId, + checkoutRunId: null, + executionRunId: null, + issueNumber: 1, + identifier: `${issuePrefix}-1`, + startedAt: new Date("2026-03-19T00:00:00.000Z"), + }); + const heartbeat = heartbeatService(db); + + const result = await heartbeat.reconcileStrandedAssignedIssues(); + expect(result.dispatchRequeued).toBe(0); + expect(result.continuationRequeued).toBe(0); + expect(result.escalated).toBe(0); + expect(result.skipped).toBe(1); + + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, agentId)); + expect(runs).toHaveLength(0); + const [issue] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(issue?.status).toBe("in_progress"); + expect(issue?.executionRunId).toBeNull(); + }); + it("blocks stranded in-progress work after the continuation retry was already used", async () => { const { issueId } = await seedStrandedIssueFixture({ status: "in_progress", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 1dfdf7bb108..2a39ad5c509 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -3178,6 +3178,11 @@ export function heartbeatService(db: Db) { continue; } + if (!latestRun && !issue.checkoutRunId && !issue.executionRunId) { + result.skipped += 1; + continue; + } + if (didAutomaticRecoveryFail(latestRun, "issue_continuation_needed")) { const failureSummary = summarizeRunFailureForIssueComment(latestRun); const updated = await escalateStrandedAssignedIssue({