diff --git a/cli/src/__tests__/worktree.test.ts b/cli/src/__tests__/worktree.test.ts index 42112836f2f..52e1381d00d 100644 --- a/cli/src/__tests__/worktree.test.ts +++ b/cli/src/__tests__/worktree.test.ts @@ -4,9 +4,10 @@ import path from "node:path"; import { execFileSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { createServer } from "node:net"; -import { eq } from "drizzle-orm"; +import { eq, inArray } from "drizzle-orm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + agentWakeupRequests, agents, authAccounts, authUsers, @@ -14,7 +15,9 @@ import { companyMemberships, createDb, executionWorkspaces, + heartbeatRuns, inspectMigrations, + instanceSettings, issueComments, issues, projectWorkspaces, @@ -35,6 +38,7 @@ import { pauseSeededScheduledRoutines, quarantineSeededWorktreeExecutionState, readWorktreeSeedManifest, + resetSeededWorktreeRunExecutionActivation, readSourceAttachmentBody, rebindWorkspaceCwd, requiresWorktreeSeedCredentialAccount, @@ -1170,6 +1174,11 @@ describe("worktree helpers", () => { const projectWorkspaceId = randomUUID(); const executionWorkspaceId = randomUUID(); const runtimeServiceId = randomUUID(); + const wakeupId = randomUUID(); + const completedWakeupId = randomUUID(); + const queuedRunId = randomUUID(); + const runningRunId = randomUUID(); + const succeededRunId = randomUUID(); try { await db.insert(companies).values({ @@ -1275,6 +1284,11 @@ describe("worktree helpers", () => { identifier: "WTQ-1", executionAgentNameKey: "codexcoder", executionLockedAt: new Date("2026-04-18T00:00:00.000Z"), + monitorNextCheckAt: new Date("2026-04-18T00:01:00.000Z"), + monitorWakeRequestedAt: new Date("2026-04-18T00:00:30.000Z"), + monitorLastTriggeredAt: new Date("2026-04-18T00:00:00.000Z"), + monitorAttemptCount: 2, + monitorNotes: "copied monitor state", }, { id: todoIssueId, @@ -1307,8 +1321,53 @@ describe("worktree helpers", () => { identifier: "WTQ-4", }, ]); + await db.insert(agentWakeupRequests).values([ + { + id: wakeupId, + companyId, + agentId, + source: "system", + status: "queued", + }, + { + id: completedWakeupId, + companyId, + agentId, + source: "system", + status: "completed", + }, + ]); + await db.insert(heartbeatRuns).values([ + { + id: queuedRunId, + companyId, + agentId, + status: "queued", + wakeupRequestId: wakeupId, + processPid: 1234, + processGroupId: 1234, + }, + { + id: runningRunId, + companyId, + agentId, + status: "running", + processPid: 5678, + processGroupId: 5678, + }, + { + id: succeededRunId, + companyId, + agentId, + status: "succeeded", + wakeupRequestId: completedWakeupId, + }, + ]); await expect(quarantineSeededWorktreeExecutionState(tempDb.connectionString)).resolves.toEqual({ + cancelledHeartbeatRuns: 2, + deletedPendingWakeups: 1, + clearedIssueMonitors: 1, disabledTimerHeartbeats: 1, resetRunningAgents: 1, quarantinedInProgressIssues: 1, @@ -1331,6 +1390,32 @@ describe("worktree helpers", () => { expect(inProgressIssue?.assigneeAgentId).toBeNull(); expect(inProgressIssue?.executionAgentNameKey).toBeNull(); expect(inProgressIssue?.executionLockedAt).toBeNull(); + expect(inProgressIssue?.monitorNextCheckAt).toBeNull(); + expect(inProgressIssue?.monitorWakeRequestedAt).toBeNull(); + expect(inProgressIssue?.monitorLastTriggeredAt).toBeNull(); + expect(inProgressIssue?.monitorAttemptCount).toBe(0); + expect(inProgressIssue?.monitorNotes).toBeNull(); + + const quarantinedRuns = await db + .select() + .from(heartbeatRuns) + .where(inArray(heartbeatRuns.id, [queuedRunId, runningRunId])); + expect(quarantinedRuns).toHaveLength(2); + for (const run of quarantinedRuns) { + expect(run.status).toBe("cancelled"); + expect(run.errorCode).toBe("worktree_seed_quarantine"); + expect(run.processPid).toBeNull(); + expect(run.processGroupId).toBeNull(); + expect(run.finishedAt).toBeInstanceOf(Date); + } + + const [succeededRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, succeededRunId)); + expect(succeededRun?.status).toBe("succeeded"); + expect(succeededRun?.wakeupRequestId).toBe(completedWakeupId); + + const wakeups = await db.select().from(agentWakeupRequests); + expect(wakeups).toHaveLength(1); + expect(wakeups[0]?.id).toBe(completedWakeupId); const [todoIssue] = await db.select().from(issues).where(eq(issues.id, todoIssueId)); expect(todoIssue?.status).toBe("todo"); @@ -1394,6 +1479,38 @@ describe("worktree helpers", () => { } }, 20_000); + itEmbeddedPostgres("always disarms copied worktree run execution settings", async () => { + const tempDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-activation-reset-"); + const db = createDb(tempDb.connectionString); + + try { + await db + .update(instanceSettings) + .set({ + experimental: { + enableWorktreeRunExecution: true, + worktreeRunExecutionActivatedAt: "2026-07-16T12:00:00.000Z", + worktreeRunExecutionActivationInstanceId: "source-instance", + enableSmokeLab: true, + }, + }) + .where(eq(instanceSettings.singletonKey, "default")); + + await expect(resetSeededWorktreeRunExecutionActivation(tempDb.connectionString)).resolves.toBe(1); + + const [settings] = await db.select().from(instanceSettings); + expect(settings?.experimental).toMatchObject({ + enableWorktreeRunExecution: false, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + enableSmokeLab: true, + }); + } finally { + await db.$client?.end?.({ timeout: 5 }).catch(() => undefined); + await tempDb.cleanup(); + } + }, 20_000); + it("copies the source local_encrypted secrets key into the seeded worktree instance", () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-secrets-")); const originalInlineMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY; diff --git a/cli/src/commands/worktree.ts b/cli/src/commands/worktree.ts index 6b7c76ed70f..7249fa07f7b 100644 --- a/cli/src/commands/worktree.ts +++ b/cli/src/commands/worktree.ts @@ -33,6 +33,7 @@ import { } from "@paperclipai/shared/worktree-port-registry"; import { applyPendingMigrations, + agentWakeupRequests, agents, authAccounts, authUsers, @@ -47,6 +48,7 @@ import { formatDatabaseBackupResult, goals, heartbeatRuns, + instanceSettings, inspectMigrations, issueAttachments, issueComments, @@ -261,6 +263,9 @@ export type LegacyWorktreeDatabaseEvidence = { }; export type SeededWorktreeExecutionQuarantineSummary = { + cancelledHeartbeatRuns: number; + deletedPendingWakeups: number; + clearedIssueMonitors: number; disabledTimerHeartbeats: number; resetRunningAgents: number; quarantinedInProgressIssues: number; @@ -287,6 +292,9 @@ function formatSeededWorktreeExecutionQuarantineSummary( summary: SeededWorktreeExecutionQuarantineSummary, ): string { return [ + `cancelled heartbeat runs: ${summary.cancelledHeartbeatRuns}`, + `deleted pending wakeups: ${summary.deletedPendingWakeups}`, + `cleared issue monitors: ${summary.clearedIssueMonitors}`, `disabled timer heartbeats: ${summary.disabledTimerHeartbeats}`, `reset running agents: ${summary.resetRunningAgents}`, `quarantined in-progress issues: ${summary.quarantinedInProgressIssues}`, @@ -1247,6 +1255,9 @@ export async function pauseSeededScheduledRoutines(connectionString: string): Pr } const EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY: SeededWorktreeExecutionQuarantineSummary = { + cancelledHeartbeatRuns: 0, + deletedPendingWakeups: 0, + clearedIssueMonitors: 0, disabledTimerHeartbeats: 0, resetRunningAgents: 0, quarantinedInProgressIssues: 0, @@ -1323,6 +1334,63 @@ export async function quarantineSeededWorktreeExecutionState( const summary = { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }; try { await db.transaction(async (tx) => { + const now = new Date(); + const pendingWakeups = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where(inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed"])); + const pendingWakeupIds = pendingWakeups.map((row) => row.id); + + if (pendingWakeupIds.length > 0) { + await tx + .update(heartbeatRuns) + .set({ wakeupRequestId: null, updatedAt: now }) + .where(inArray(heartbeatRuns.wakeupRequestId, pendingWakeupIds)); + } + + const cancelledRuns = await tx + .update(heartbeatRuns) + .set({ + status: "cancelled", + errorCode: "worktree_seed_quarantine", + processPid: null, + processGroupId: null, + scheduledRetryAt: null, + finishedAt: now, + updatedAt: now, + }) + .where(inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"])) + .returning({ id: heartbeatRuns.id }); + summary.cancelledHeartbeatRuns = cancelledRuns.length; + + if (pendingWakeupIds.length > 0) { + const deletedWakeups = await tx + .delete(agentWakeupRequests) + .where(inArray(agentWakeupRequests.id, pendingWakeupIds)) + .returning({ id: agentWakeupRequests.id }); + summary.deletedPendingWakeups = deletedWakeups.length; + } + + const clearedMonitors = await tx + .update(issues) + .set({ + monitorNextCheckAt: null, + monitorWakeRequestedAt: null, + monitorLastTriggeredAt: null, + monitorAttemptCount: 0, + monitorNotes: null, + updatedAt: now, + }) + .where(sql` + ${issues.monitorNextCheckAt} is not null + or ${issues.monitorWakeRequestedAt} is not null + or ${issues.monitorLastTriggeredAt} is not null + or ${issues.monitorAttemptCount} <> 0 + or ${issues.monitorNotes} is not null + `) + .returning({ id: issues.id }); + summary.clearedIssueMonitors = clearedMonitors.length; + const seededAgents = await tx .select({ id: agents.id, @@ -1346,7 +1414,7 @@ export async function quarantineSeededWorktreeExecutionState( .set({ runtimeConfig: normalized.runtimeConfig, status: nextStatus, - updatedAt: new Date(), + updatedAt: now, }) .where(eq(agents.id, agent.id)); } @@ -1379,7 +1447,7 @@ export async function quarantineSeededWorktreeExecutionState( executionAgentNameKey: null, executionLockedAt: null, executionWorkspaceId: null, - updatedAt: new Date(), + updatedAt: now, }) .where(eq(issues.id, issue.id)); @@ -1677,6 +1745,31 @@ async function inspectVerifiedSeedDatabase( } } +export async function resetSeededWorktreeRunExecutionActivation(connectionString: string): Promise { + const db = createDb(connectionString); + try { + const rows = await db.select({ id: instanceSettings.id, experimental: instanceSettings.experimental }).from(instanceSettings); + let resetCount = 0; + for (const row of rows) { + const experimental = isRecord(row.experimental) ? row.experimental : {}; + const nextExperimental = { + ...experimental, + enableWorktreeRunExecution: false, + worktreeRunExecutionActivatedAt: null, + worktreeRunExecutionActivationInstanceId: null, + }; + await db + .update(instanceSettings) + .set({ experimental: nextExperimental, updatedAt: new Date() }) + .where(eq(instanceSettings.id, row.id)); + resetCount += 1; + } + return resetCount; + } finally { + await db.$client?.end?.({ timeout: 5 }).catch(() => undefined); + } +} + async function seedWorktreeDatabase(input: { sourceConfigPath: string; sourceConfig: PaperclipConfig; @@ -1762,6 +1855,7 @@ async function seedWorktreeDatabase(input: { await applyPendingMigrations(targetConnectionString); input.onPhase?.("migrations", "succeeded"); input.onPhase?.("execution_quarantine", "started"); + await resetSeededWorktreeRunExecutionActivation(targetConnectionString); const executionQuarantine = input.preserveLiveWork ? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY } : await quarantineSeededWorktreeExecutionState(targetConnectionString); @@ -4273,7 +4367,7 @@ async function backupWorktreeReseedTarget(input: { } async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise { - const seedMode = opts.seedMode ?? "full"; + const seedMode = opts.seedMode ?? "minimal"; if (!isWorktreeSeedMode(seedMode)) { throw new Error(`Unsupported seed mode "${seedMode}". Expected one of: minimal, full.`); } @@ -4488,7 +4582,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--server-port ", "Preferred server port", (value) => Number(value)) .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") - .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false) + .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false) .option("--no-seed", "Skip database seeding from the source instance") .option("--force", "Replace existing repo-local config and isolated instance data", false) .action(worktreeMakeCommand); @@ -4505,7 +4599,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--server-port ", "Preferred server port", (value) => Number(value)) .option("--db-port ", "Preferred embedded Postgres port", (value) => Number(value)) .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") - .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false) + .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false) .option("--no-seed", "Skip database seeding from the source instance") .option("--force", "Replace existing repo-local config and isolated instance data", false) .action(worktreeInitCommand); @@ -4555,7 +4649,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--from-data-dir ", "Source PAPERCLIP_HOME used when deriving the source config") .option("--from-instance ", "Source instance id when deriving the source config") .option("--seed-mode ", "Seed profile: minimal or full (default: full)", "full") - .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false) + .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false) .option("--yes", "Skip the destructive confirmation prompt", false) .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false) .option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false) @@ -4570,7 +4664,7 @@ export function registerWorktreeCommands(program: Command): void { .option("--from-data-dir ", "Source PAPERCLIP_HOME used when deriving the source config") .option("--from-instance ", "Source instance id when deriving the source config (default: default)") .option("--seed-mode ", "Seed profile: minimal or full (default: minimal)", "minimal") - .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false) + .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false) .option("--no-seed", "Repair metadata only and skip reseeding when bootstrapping a missing worktree config", false) .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false) .action(worktreeRepairCommand); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 20747f26818..9dd6f66adad 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -564,6 +564,7 @@ The workspace UI surfaces `Provisioning database`, `Validating clone`, `Ready`, | `--server-port ` | Preferred server port | | `--db-port ` | Preferred embedded Postgres port | | `--seed-mode ` | Seed profile: `minimal` or `full` (default: `minimal`) | +| `--preserve-live-work` | Explicitly bypass seed quarantine for copied runs, wakeups, monitors, agent timers, and assigned open issues. The worktree run-execution toggle is still reset. | | `--no-seed` | Skip database seeding from the source instance | | `--force` | Replace existing repo-local config and isolated instance data | @@ -600,6 +601,7 @@ For an already-created worktree where you want the CLI to decide whether to rebu | `--from-data-dir ` | Source `PAPERCLIP_HOME` used when deriving the source config | | `--from-instance ` | Source instance id when deriving the source config (default: `default`) | | `--seed-mode ` | Seed profile: `minimal` or `full` (default: `minimal`) | +| `--preserve-live-work` | Explicitly bypass seed quarantine for copied runs, wakeups, monitors, agent timers, and assigned open issues. The worktree run-execution toggle is still reset. | | `--no-seed` | Repair metadata only when bootstrapping a missing worktree config | | `--allow-live-target` | Override the guard that requires the target worktree DB to be stopped first | @@ -627,7 +629,8 @@ For an already-created worktree where you want to keep the existing repo-local c | `--from-config ` | Source config.json to seed from | | `--from-data-dir ` | Source `PAPERCLIP_HOME` used when deriving the source config | | `--from-instance ` | Source instance id when deriving the source config | -| `--seed-mode ` | Seed profile: `minimal` or `full` (default: `full`) | +| `--seed-mode ` | Seed profile: `minimal` or `full` (default: `minimal`) | +| `--preserve-live-work` | Explicitly bypass seed quarantine for copied runs, wakeups, monitors, agent timers, and assigned open issues. The worktree run-execution toggle is still reset. | | `--yes` | Skip the destructive confirmation prompt | | `--allow-live-target` | Override the guard that requires the target worktree DB to be stopped first | | `--backup-target` | Retain a recoverable full target-DB backup before reseeding | @@ -665,6 +668,7 @@ Managed workspace repair uses this same verified full-reseed contract through `P | `--server-port ` | Preferred server port | | `--db-port ` | Preferred embedded Postgres port | | `--seed-mode ` | Seed profile: `minimal` or `full` (default: `minimal`) | +| `--preserve-live-work` | Explicitly bypass seed quarantine for copied runs, wakeups, monitors, agent timers, and assigned open issues. The worktree run-execution toggle is still reset. | | `--no-seed` | Skip database seeding from the source instance | | `--force` | Replace existing repo-local config and isolated instance data |