fix(cli): quarantine worktree seed execution state

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cli/src/__tests__/worktree.test.ts

@@ -4,9 +4,10 @@ import path from "node:path";
 import { execFileSync } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import { createServer } from "node:net";
-import { eq } from "drizzle-orm";
+import { eq, inArray } from "drizzle-orm";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import {
+  agentWakeupRequests,
   agents,
   authAccounts,
   authUsers,
@@ -14,7 +15,9 @@ import {
   companyMemberships,
   createDb,
   executionWorkspaces,
+  heartbeatRuns,
   inspectMigrations,
+  instanceSettings,
   issueComments,
   issues,
   projectWorkspaces,
@@ -35,6 +38,7 @@ import {
   pauseSeededScheduledRoutines,
   quarantineSeededWorktreeExecutionState,
   readWorktreeSeedManifest,
+  resetSeededWorktreeRunExecutionActivation,
   readSourceAttachmentBody,
   rebindWorkspaceCwd,
   requiresWorktreeSeedCredentialAccount,
@@ -1170,6 +1174,11 @@ describe("worktree helpers", () => {
     const projectWorkspaceId = randomUUID();
     const executionWorkspaceId = randomUUID();
     const runtimeServiceId = randomUUID();
+    const wakeupId = randomUUID();
+    const completedWakeupId = randomUUID();
+    const queuedRunId = randomUUID();
+    const runningRunId = randomUUID();
+    const succeededRunId = randomUUID();
 
     try {
       await db.insert(companies).values({
@@ -1275,6 +1284,11 @@ describe("worktree helpers", () => {
           identifier: "WTQ-1",
           executionAgentNameKey: "codexcoder",
           executionLockedAt: new Date("2026-04-18T00:00:00.000Z"),
+          monitorNextCheckAt: new Date("2026-04-18T00:01:00.000Z"),
+          monitorWakeRequestedAt: new Date("2026-04-18T00:00:30.000Z"),
+          monitorLastTriggeredAt: new Date("2026-04-18T00:00:00.000Z"),
+          monitorAttemptCount: 2,
+          monitorNotes: "copied monitor state",
         },
         {
           id: todoIssueId,
@@ -1307,8 +1321,53 @@ describe("worktree helpers", () => {
           identifier: "WTQ-4",
         },
       ]);
+      await db.insert(agentWakeupRequests).values([
+        {
+          id: wakeupId,
+          companyId,
+          agentId,
+          source: "system",
+          status: "queued",
+        },
+        {
+          id: completedWakeupId,
+          companyId,
+          agentId,
+          source: "system",
+          status: "completed",
+        },
+      ]);
+      await db.insert(heartbeatRuns).values([
+        {
+          id: queuedRunId,
+          companyId,
+          agentId,
+          status: "queued",
+          wakeupRequestId: wakeupId,
+          processPid: 1234,
+          processGroupId: 1234,
+        },
+        {
+          id: runningRunId,
+          companyId,
+          agentId,
+          status: "running",
+          processPid: 5678,
+          processGroupId: 5678,
+        },
+        {
+          id: succeededRunId,
+          companyId,
+          agentId,
+          status: "succeeded",
+          wakeupRequestId: completedWakeupId,
+        },
+      ]);
 
       await expect(quarantineSeededWorktreeExecutionState(tempDb.connectionString)).resolves.toEqual({
+        cancelledHeartbeatRuns: 2,
+        deletedPendingWakeups: 1,
+        clearedIssueMonitors: 1,
         disabledTimerHeartbeats: 1,
         resetRunningAgents: 1,
         quarantinedInProgressIssues: 1,
@@ -1331,6 +1390,32 @@ describe("worktree helpers", () => {
       expect(inProgressIssue?.assigneeAgentId).toBeNull();
       expect(inProgressIssue?.executionAgentNameKey).toBeNull();
       expect(inProgressIssue?.executionLockedAt).toBeNull();
+      expect(inProgressIssue?.monitorNextCheckAt).toBeNull();
+      expect(inProgressIssue?.monitorWakeRequestedAt).toBeNull();
+      expect(inProgressIssue?.monitorLastTriggeredAt).toBeNull();
+      expect(inProgressIssue?.monitorAttemptCount).toBe(0);
+      expect(inProgressIssue?.monitorNotes).toBeNull();
+
+      const quarantinedRuns = await db
+        .select()
+        .from(heartbeatRuns)
+        .where(inArray(heartbeatRuns.id, [queuedRunId, runningRunId]));
+      expect(quarantinedRuns).toHaveLength(2);
+      for (const run of quarantinedRuns) {
+        expect(run.status).toBe("cancelled");
+        expect(run.errorCode).toBe("worktree_seed_quarantine");
+        expect(run.processPid).toBeNull();
+        expect(run.processGroupId).toBeNull();
+        expect(run.finishedAt).toBeInstanceOf(Date);
+      }
+
+      const [succeededRun] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, succeededRunId));
+      expect(succeededRun?.status).toBe("succeeded");
+      expect(succeededRun?.wakeupRequestId).toBe(completedWakeupId);
+
+      const wakeups = await db.select().from(agentWakeupRequests);
+      expect(wakeups).toHaveLength(1);
+      expect(wakeups[0]?.id).toBe(completedWakeupId);
 
       const [todoIssue] = await db.select().from(issues).where(eq(issues.id, todoIssueId));
       expect(todoIssue?.status).toBe("todo");
@@ -1394,6 +1479,38 @@ describe("worktree helpers", () => {
     }
   }, 20_000);
 
+  itEmbeddedPostgres("always disarms copied worktree run execution settings", async () => {
+    const tempDb = await startEmbeddedPostgresTestDatabase("paperclip-worktree-activation-reset-");
+    const db = createDb(tempDb.connectionString);
+
+    try {
+      await db
+        .update(instanceSettings)
+        .set({
+          experimental: {
+          enableWorktreeRunExecution: true,
+          worktreeRunExecutionActivatedAt: "2026-07-16T12:00:00.000Z",
+          worktreeRunExecutionActivationInstanceId: "source-instance",
+          enableSmokeLab: true,
+          },
+        })
+        .where(eq(instanceSettings.singletonKey, "default"));
+
+      await expect(resetSeededWorktreeRunExecutionActivation(tempDb.connectionString)).resolves.toBe(1);
+
+      const [settings] = await db.select().from(instanceSettings);
+      expect(settings?.experimental).toMatchObject({
+        enableWorktreeRunExecution: false,
+        worktreeRunExecutionActivatedAt: null,
+        worktreeRunExecutionActivationInstanceId: null,
+        enableSmokeLab: true,
+      });
+    } finally {
+      await db.$client?.end?.({ timeout: 5 }).catch(() => undefined);
+      await tempDb.cleanup();
+    }
+  }, 20_000);
+
   it("copies the source local_encrypted secrets key into the seeded worktree instance", () => {
     const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-worktree-secrets-"));
     const originalInlineMasterKey = process.env.PAPERCLIP_SECRETS_MASTER_KEY;

cli/src/commands/worktree.ts

@@ -33,6 +33,7 @@ import {
 } from "@paperclipai/shared/worktree-port-registry";
 import {
   applyPendingMigrations,
+  agentWakeupRequests,
   agents,
   authAccounts,
   authUsers,
@@ -47,6 +48,7 @@ import {
   formatDatabaseBackupResult,
   goals,
   heartbeatRuns,
+  instanceSettings,
   inspectMigrations,
   issueAttachments,
   issueComments,
@@ -261,6 +263,9 @@ export type LegacyWorktreeDatabaseEvidence = {
 };
 
 export type SeededWorktreeExecutionQuarantineSummary = {
+  cancelledHeartbeatRuns: number;
+  deletedPendingWakeups: number;
+  clearedIssueMonitors: number;
   disabledTimerHeartbeats: number;
   resetRunningAgents: number;
   quarantinedInProgressIssues: number;
@@ -287,6 +292,9 @@ function formatSeededWorktreeExecutionQuarantineSummary(
   summary: SeededWorktreeExecutionQuarantineSummary,
 ): string {
   return [
+    `cancelled heartbeat runs: ${summary.cancelledHeartbeatRuns}`,
+    `deleted pending wakeups: ${summary.deletedPendingWakeups}`,
+    `cleared issue monitors: ${summary.clearedIssueMonitors}`,
     `disabled timer heartbeats: ${summary.disabledTimerHeartbeats}`,
     `reset running agents: ${summary.resetRunningAgents}`,
     `quarantined in-progress issues: ${summary.quarantinedInProgressIssues}`,
@@ -1247,6 +1255,9 @@ export async function pauseSeededScheduledRoutines(connectionString: string): Pr
 }
 
 const EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY: SeededWorktreeExecutionQuarantineSummary = {
+  cancelledHeartbeatRuns: 0,
+  deletedPendingWakeups: 0,
+  clearedIssueMonitors: 0,
   disabledTimerHeartbeats: 0,
   resetRunningAgents: 0,
   quarantinedInProgressIssues: 0,
@@ -1323,6 +1334,63 @@ export async function quarantineSeededWorktreeExecutionState(
   const summary = { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY };
   try {
     await db.transaction(async (tx) => {
+      const now = new Date();
+      const pendingWakeups = await tx
+        .select({ id: agentWakeupRequests.id })
+        .from(agentWakeupRequests)
+        .where(inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution", "claimed"]));
+      const pendingWakeupIds = pendingWakeups.map((row) => row.id);
+
+      if (pendingWakeupIds.length > 0) {
+        await tx
+          .update(heartbeatRuns)
+          .set({ wakeupRequestId: null, updatedAt: now })
+          .where(inArray(heartbeatRuns.wakeupRequestId, pendingWakeupIds));
+      }
+
+      const cancelledRuns = await tx
+        .update(heartbeatRuns)
+        .set({
+          status: "cancelled",
+          errorCode: "worktree_seed_quarantine",
+          processPid: null,
+          processGroupId: null,
+          scheduledRetryAt: null,
+          finishedAt: now,
+          updatedAt: now,
+        })
+        .where(inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]))
+        .returning({ id: heartbeatRuns.id });
+      summary.cancelledHeartbeatRuns = cancelledRuns.length;
+
+      if (pendingWakeupIds.length > 0) {
+        const deletedWakeups = await tx
+          .delete(agentWakeupRequests)
+          .where(inArray(agentWakeupRequests.id, pendingWakeupIds))
+          .returning({ id: agentWakeupRequests.id });
+        summary.deletedPendingWakeups = deletedWakeups.length;
+      }
+
+      const clearedMonitors = await tx
+        .update(issues)
+        .set({
+          monitorNextCheckAt: null,
+          monitorWakeRequestedAt: null,
+          monitorLastTriggeredAt: null,
+          monitorAttemptCount: 0,
+          monitorNotes: null,
+          updatedAt: now,
+        })
+        .where(sql`
+          ${issues.monitorNextCheckAt} is not null
+          or ${issues.monitorWakeRequestedAt} is not null
+          or ${issues.monitorLastTriggeredAt} is not null
+          or ${issues.monitorAttemptCount} <> 0
+          or ${issues.monitorNotes} is not null
+        `)
+        .returning({ id: issues.id });
+      summary.clearedIssueMonitors = clearedMonitors.length;
+
       const seededAgents = await tx
         .select({
           id: agents.id,
@@ -1346,7 +1414,7 @@ export async function quarantineSeededWorktreeExecutionState(
             .set({
               runtimeConfig: normalized.runtimeConfig,
               status: nextStatus,
-              updatedAt: new Date(),
+              updatedAt: now,
             })
             .where(eq(agents.id, agent.id));
         }
@@ -1379,7 +1447,7 @@ export async function quarantineSeededWorktreeExecutionState(
             executionAgentNameKey: null,
             executionLockedAt: null,
             executionWorkspaceId: null,
-            updatedAt: new Date(),
+            updatedAt: now,
           })
           .where(eq(issues.id, issue.id));
 
@@ -1677,6 +1745,31 @@ async function inspectVerifiedSeedDatabase(
   }
 }
 
+export async function resetSeededWorktreeRunExecutionActivation(connectionString: string): Promise<number> {
+  const db = createDb(connectionString);
+  try {
+    const rows = await db.select({ id: instanceSettings.id, experimental: instanceSettings.experimental }).from(instanceSettings);
+    let resetCount = 0;
+    for (const row of rows) {
+      const experimental = isRecord(row.experimental) ? row.experimental : {};
+      const nextExperimental = {
+        ...experimental,
+        enableWorktreeRunExecution: false,
+        worktreeRunExecutionActivatedAt: null,
+        worktreeRunExecutionActivationInstanceId: null,
+      };
+      await db
+        .update(instanceSettings)
+        .set({ experimental: nextExperimental, updatedAt: new Date() })
+        .where(eq(instanceSettings.id, row.id));
+      resetCount += 1;
+    }
+    return resetCount;
+  } finally {
+    await db.$client?.end?.({ timeout: 5 }).catch(() => undefined);
+  }
+}
+
 async function seedWorktreeDatabase(input: {
   sourceConfigPath: string;
   sourceConfig: PaperclipConfig;
@@ -1762,6 +1855,7 @@ async function seedWorktreeDatabase(input: {
     await applyPendingMigrations(targetConnectionString);
     input.onPhase?.("migrations", "succeeded");
     input.onPhase?.("execution_quarantine", "started");
+    await resetSeededWorktreeRunExecutionActivation(targetConnectionString);
     const executionQuarantine = input.preserveLiveWork
       ? { ...EMPTY_SEEDED_WORKTREE_EXECUTION_QUARANTINE_SUMMARY }
       : await quarantineSeededWorktreeExecutionState(targetConnectionString);
@@ -4273,7 +4367,7 @@ async function backupWorktreeReseedTarget(input: {
 }
 
 async function runWorktreeReseed(opts: WorktreeReseedOptions): Promise<void> {
-  const seedMode = opts.seedMode ?? "full";
+  const seedMode = opts.seedMode ?? "minimal";
   if (!isWorktreeSeedMode(seedMode)) {
     throw new Error(`Unsupported seed mode "${seedMode}". Expected one of: minimal, full.`);
   }
@@ -4488,7 +4582,7 @@ export function registerWorktreeCommands(program: Command): void {
     .option("--server-port <port>", "Preferred server port", (value) => Number(value))
     .option("--db-port <port>", "Preferred embedded Postgres port", (value) => Number(value))
     .option("--seed-mode <mode>", "Seed profile: minimal or full (default: minimal)", "minimal")
-    .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
+    .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false)
     .option("--no-seed", "Skip database seeding from the source instance")
     .option("--force", "Replace existing repo-local config and isolated instance data", false)
     .action(worktreeMakeCommand);
@@ -4505,7 +4599,7 @@ export function registerWorktreeCommands(program: Command): void {
     .option("--server-port <port>", "Preferred server port", (value) => Number(value))
     .option("--db-port <port>", "Preferred embedded Postgres port", (value) => Number(value))
     .option("--seed-mode <mode>", "Seed profile: minimal or full (default: minimal)", "minimal")
-    .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
+    .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false)
     .option("--no-seed", "Skip database seeding from the source instance")
     .option("--force", "Replace existing repo-local config and isolated instance data", false)
     .action(worktreeInitCommand);
@@ -4555,7 +4649,7 @@ export function registerWorktreeCommands(program: Command): void {
     .option("--from-data-dir <path>", "Source PAPERCLIP_HOME used when deriving the source config")
     .option("--from-instance <id>", "Source instance id when deriving the source config")
     .option("--seed-mode <mode>", "Seed profile: minimal or full (default: full)", "full")
-    .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
+    .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false)
     .option("--yes", "Skip the destructive confirmation prompt", false)
     .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false)
     .option("--backup-target", "Retain a recoverable full backup of the isolated target DB before reseeding", false)
@@ -4570,7 +4664,7 @@ export function registerWorktreeCommands(program: Command): void {
     .option("--from-data-dir <path>", "Source PAPERCLIP_HOME used when deriving the source config")
     .option("--from-instance <id>", "Source instance id when deriving the source config (default: default)")
     .option("--seed-mode <mode>", "Seed profile: minimal or full (default: minimal)", "minimal")
-    .option("--preserve-live-work", "Do not quarantine copied agent work or workspace runtime services in the seeded worktree", false)
+    .option("--preserve-live-work", "Preserve copied runs, wakeups, monitors, agent work, and workspace runtime services", false)
     .option("--no-seed", "Repair metadata only and skip reseeding when bootstrapping a missing worktree config", false)
     .option("--allow-live-target", "Override the guard that requires the target worktree DB to be stopped first", false)
     .action(worktreeRepairCommand);