Drive remaining live run and resource recovery from durable work (#18419)

Reviewed and verified at 6a4c193906.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/chat-channels.integration.test.ts

@@ -76892,7 +76892,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
           resolveSessionBeforeForWakeup: async () => null,
         });
         return adapter.withIssueExecutionLock({ companyId: context.fixture.companyId, runId: releaseRunId, now: new Date() }, async (_locked, ports) => {
-          const target = { companyId: context.fixture.companyId, wakeId: context.action.id, now: new Date() };
+          const target = { companyId: context.fixture.companyId, wakeId: context.action.id, agentId: context.fixture.assignedAgentId, now: new Date() };
           if (change === "queue rejection") await ports.transaction.cancelDeferredWake({ ...target, reason: "PRIVATE rejection" });
           else await ports.transaction.failDeferredWake(target);
           return { outcome: { kind: "released" }, postCommitEffects: [] };

server/src/__tests__/heartbeat-active-run-output-watchdog.test.ts

@@ -1,5 +1,6 @@
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
 import { randomUUID } from "node:crypto";
-import { and, eq } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
 import {
   activityLog,
@@ -39,7 +40,8 @@ vi.mock("../services/heartbeat-run-events.js", async (importOriginal) => {
 const mockedAppendHeartbeatRunEvent = vi.mocked(appendHeartbeatRunEvent);
 
 const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
-const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_OUTPUT_WATCHDOG_TEST_DATABASE_URL");
+const describeEmbeddedPostgres = external || embeddedPostgresSupport.supported ? describe : describe.skip;
 
 if (!embeddedPostgresSupport.supported) {
   console.warn(
@@ -52,8 +54,9 @@ describeEmbeddedPostgres("active-run output watchdog", () => {
   let db: ReturnType<typeof createDb>;
 
   beforeAll(async () => {
-    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-active-run-output-watchdog-");
-    db = createDb(tempDb.connectionString);
+    if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-active-run-output-watchdog-");
+    db = createDb(external ?? tempDb!.connectionString);
+    if (external) await db.execute(sql`truncate companies cascade`);
   }, 30_000);
 
   afterEach(async () => {
@@ -65,6 +68,7 @@ describeEmbeddedPostgres("active-run output watchdog", () => {
   });
 
   afterAll(async () => {
+    await db?.$client.end({ timeout: 1 });
     await tempDb?.cleanup();
   });
 

server/src/__tests__/live-recovery-work.test.ts

@@ -0,0 +1,875 @@
+import { randomUUID } from "node:crypto";
+import os from "node:os";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import { setImmediate as immediate, setTimeout as realDelay } from "node:timers/promises";
+import { and, eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { agents, authUsers, companies, companyMemberships, createDb, closeLivenessDb, closeCheckpointDb,
+  heartbeatRuns, heartbeatRunEvents, nativeRunFinalizations, environmentLeases, issueThreadInteractions, applyPendingMigrations,
+  issueRecoveryActions, issues, activityLog, agentWakeupRequests, reconcileDatabaseWork } from "@paperclipai/db";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
+import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { heartbeatService } from "../services/heartbeat.js";
+import { notifyLiveRunRecovery, notifyLiveResourceRecovery, notifyStrandedAgentStatus } from "../services/live-run-recovery-notifications.js";
+import { notifyExecutionRunWork } from "../services/execution-control-notifications.js";
+import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
+import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js";
+import { prepareSandboxStopAndRetain, settleStopOnlyCleanup, hasConfirmedSandboxStopAndRetain } from "../services/sandbox-stop-and-retain.js";
+import { beginLegacyShutdownWorkspaceSettlement } from "../services/legacy-shutdown-workspace-settlement.js";
+import { renewLegacyControllerLease, legacyControllerBootId } from "../services/legacy-controller-lease.js";
+import { idleWorkSnapshot } from "../services/task-admission.js";
+import { createHeartbeatRunOutput } from "../services/heartbeat/run-output.js";
+import { recoveryService } from "../services/recovery/service.js";
+import { instanceSettingsService } from "../services/instance-settings.js";
+import { persistActivity } from "../services/activity-log.js";
+import { accountRunCost } from "../services/run-cost-accounting.js";
+import { persistHeartbeatRunProcessMetadata, createHeartbeatLifecycle } from "../services/heartbeat/run-lifecycle.js";
+import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js";
+import { issueTreeControlService } from "../services/issue-tree-control.js";
+import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
+import { issueService } from "../services/issues.js";
+import { nativeRetryCancellationEligible, pendingFailedNativeRetryCancellationCondition } from "../services/native-runtime/native-cancellation-request.js";
+import { requestNativeQuestionRunCancellation } from "../services/native-runtime/native-question-bridge.js";
+import { readNativeLocalProcessStop } from "../services/native-local-process-stop.js";
+import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
+import { logger } from "../middleware/logger.js";
+
+vi.setConfig({ testTimeout: 90_000 });
+describe("residual recovery production matrix", () => {
+  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
+  let db: ReturnType<typeof createDb>;
+  let heartbeat: ReturnType<typeof heartbeatService>;
+  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
+  let draining = false, standby = false;
+  const errors: unknown[] = [], passes = new Set<Promise<unknown>>();
+  const deadlines = new Map<string, () => number | null>();
+  const timeline: unknown[] = [];
+  const extraStops: Array<() => Promise<void>> = [];
+  beforeAll(async () => {
+    const external = resolveExternalTestDatabaseUrl("PAPERCLIP_LIVE_RECOVERY_TEST_DATABASE_URL");
+    if (!external) temporary = await startEmbeddedPostgresTestDatabase("live-recovery-");
+    else await applyPendingMigrations(external);
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+  });
+  beforeEach(async () => {
+    await db.execute(sql`truncate companies cascade`); errors.length = 0; deadlines.clear(); timeline.length = 0; draining = false; standby = false;
+    heartbeat = heartbeatService(db);
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !standby, canReconcile: () => !standby,
+      onError: error => errors.push(error) });
+    const register = coordinator.register.bind(coordinator);
+    coordinator.register = (queue, task) => { deadlines.set(queue, () => task.nextRunAt?.() ?? null); return register(queue, { ...task, onSettled: hints => { timeline.push(["settled", queue, Date.now(), [...hints ?? []]]); task.onSettled?.(hints); }, run: signal => {
+      timeline.push(["pass", queue, Date.now()]);
+      const pass = Promise.resolve(task.run(signal)); passes.add(pass);
+      void pass.then(() => { timeline.push(["finished", queue, Date.now()]); passes.delete(pass); }, () => passes.delete(pass)); return pass;
+    } }); };
+  });
+  afterEach(async () => {
+    for (const stop of extraStops.splice(0)) await stop();
+    await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
+    vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
+  });
+  afterAll(async () => {
+    await closeLivenessDb(db, 1); await closeCheckpointDb(db, 1);
+    await db?.$client.end({ timeout: 1 }); await temporary?.cleanup();
+  });
+  async function fixture() {
+    const companyId = randomUUID(), actorId = "live-recovery-user";
+    await db.insert(authUsers).values({ id: actorId, name: "Recovery owner", email: `${actorId}@example.test`,
+      createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
+    await db.insert(companies).values({ id: companyId, name: "Live recovery", issuePrefix: `LR${companyId.slice(0, 6)}`,
+      requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: actorId });
+    await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: actorId,
+      status: "active", membershipRole: "owner" });
+    const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", status: "idle", adapterType: "process",
+      adapterConfig: { command: process.execPath, args: ["-e", "console.log('live-recovery-proof')"], cwd: process.cwd() },
+      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
+    const [issue] = await db.insert(issues).values({ companyId, title: "Recovered work", status: "done", assigneeAgentId: agent.id }).returning();
+    return { companyId, agent, issue, actorId };
+  }
+  async function start(realClock = false) {
+    if (!realClock && !vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const admitted = () => !draining && !standby;
+    await heartbeat.registerResidualRecoveryWork(coordinator, admitted).ready;
+    await heartbeat.registerActiveRunSilenceWork(coordinator, admitted).ready;
+    await heartbeat.registerIssueStateRecoveryWork(coordinator, admitted).ready;
+    await heartbeat.registerDeferredReceiptWork(coordinator, admitted).ready;
+    await heartbeat.registerRunDispatchWork(coordinator, admitted).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, admitted).ready;
+  }
+  async function advance(ms: number) {
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const until = Date.now() + ms;
+    while (Date.now() < until) {
+      await vi.advanceTimersByTimeAsync(Math.min(500, until - Date.now()));
+      await Promise.allSettled([...passes]); await immediate();
+    }
+  }
+  async function quiet() {
+    await heartbeat.drainActiveRunExecutions();
+    expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
+    const quietAt = Date.now();
+    if (vi.isFakeTimers()) await advance(31_000);
+    else { await realDelay(31_000); await Promise.allSettled([...passes]); }
+    console.info("quiet-drain", JSON.stringify({ quietAt, now: Date.now(), active: idleWorkSnapshot().active, next: [...deadlines].map(([q, at]) => [q, at()]), passes: Object.fromEntries([...deadlines.keys()].map(queue => [queue, timeline.filter(event => Array.isArray(event) && event[0] === "pass" && event[1] === queue).length])), settled: timeline.filter(event => Array.isArray(event) && event[0] === "settled") }));
+    await vi.waitFor(() => { expect(coordinator.nextWakeAt(), JSON.stringify([...deadlines].map(([name, at]) => [name, at()]))).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); });
+    const queries: string[] = [], prior = db.$client.options.debug;
+    db.$client.options.debug = (_connection, query) => { queries.push(query); };
+    try { if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await vi.advanceTimersByTimeAsync(20 * 60_000); await immediate(); expect(queries).toEqual([]); }
+    finally { db.$client.options.debug = prior; }
+  }
+  async function getRun(id: string) { return (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]; }
+  async function getLease(id: string) { return (await db.select().from(environmentLeases).where(eq(environmentLeases.id, id)))[0]; }
+  it("leaves configured idle and completed native history without a timer or SQL", async () => {
+    const f = await fixture();
+    const runs = await db.insert(heartbeatRuns).values(Array.from({ length: 150 }, () => ({ companyId: f.companyId,
+      agentId: f.agent.id, status: "failed", runtimeMode: "native", nativeIssueId: f.issue.id, finishedAt: new Date() }))).returning();
+    await db.insert(nativeRunFinalizations).values(runs.map(run => ({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, phase: "committed" })));
+    await start(); await quiet();
+  });
+  it("catches up native resultless retry debt through its real claim owner without replaying missing state", async () => {
+    const f = await fixture(); await db.update(issues).set({ status: "todo" }).where(eq(issues.id, f.issue.id));
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "failed", runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id },
+      finishedAt: new Date(), errorCode: "original_native_failure" }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id,
+      phase: "retryable_failure", nextAttemptAt: new Date(Date.now() - 1_000) });
+    await start();
+    await vi.waitFor(async () => {
+      expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0])
+        .toMatchObject({ phase: "terminal_failure", nextAttemptAt: null, leaseOwner: null });
+    });
+    expect((await getRun(run.id)).errorCode).toBe("original_native_failure");
+    expect(await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, f.issue.id)))
+      .toEqual([expect.objectContaining({ ownerType: "board", status: "active" })]);
+    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
+    await quiet();
+  });
+  it.each(["scoped remote retry", "startup"] as const)("reconstructs only the retry deadlines owed by %s", async mode => {
+    const selected = await fixture(), other = await fixture();
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const now = Date.now(), otherAt = now + 30_000, selectedAt = now + 60_000;
+    const seed = async (f: Awaited<ReturnType<typeof fixture>>, nextAttemptAt: number, remote = false) => {
+      await db.update(issues).set({ status: "todo" }).where(eq(issues.id, f.issue.id));
+      const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+        status: "failed", runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id },
+        finishedAt: new Date(), errorCode: "saved_native_failure",
+        runnerProfileJson: remote ? { nativeWorkspaceSync: { schema: "paperclip.native-workspace-sync/v1",
+          state: "prepared", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64),
+          workspaceId: randomUUID(), leaseId: randomUUID(), providerLeaseId: "remote-retry-fixture", remoteCwd: "/workspace" } } : null,
+      }).returning();
+      await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id,
+        phase: "retryable_failure", nextAttemptAt: new Date(nextAttemptAt) });
+      return run;
+    };
+    const selectedRun = await seed(selected, selectedAt, true), otherRun = await seed(other, otherAt);
+    await heartbeat.recoverNativeRunsAfterRestart(new Date(now), mode === "startup" ? undefined : [selectedRun.id]);
+    const queries: string[] = [], prior = db.$client.options.debug;
+    if (mode === "scoped remote retry") {
+      db.$client.options.debug = (_connection, query) => { queries.push(query); };
+      try { await advance(30_001); await immediate(); expect(queries).toEqual([]); }
+      finally { db.$client.options.debug = prior; }
+      expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, otherRun.id)))[0])
+        .toMatchObject({ phase: "retryable_failure", nextAttemptAt: new Date(otherAt) });
+    } else {
+      await advance(30_001);
+      await vi.waitFor(async () => expect((await db.select().from(nativeRunFinalizations)
+        .where(eq(nativeRunFinalizations.runId, otherRun.id)))[0].phase).toBe("terminal_failure"));
+    }
+    await advance(selectedAt - Date.now() + 1);
+    await vi.waitFor(async () => expect((await db.select().from(nativeRunFinalizations)
+      .where(eq(nativeRunFinalizations.runId, selectedRun.id)))[0].phase).toBe("terminal_failure"));
+    expect((await getRun(selectedRun.id)).errorCode).toBe("saved_native_failure");
+    expect(await db.select().from(heartbeatRuns)).toHaveLength(2);
+    await heartbeat.drainActiveRunExecutions();
+    expect(coordinator.nextWakeAt()).toBeNull();
+  });
+  it("consumes the durable native question cancellation from ordinary task completion", async () => {
+    const f = await fixture(); await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
+      runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id }, nativePhase: "awaiting_input" }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id,
+      phase: "observed", recoveryState: "awaiting_evidence", leaseExpiresAt: new Date(Date.now() + 60_000) });
+    await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issue.id, kind: "ask_user_questions",
+      sourceRunId: run.id, status: "pending", continuationPolicy: "none",
+      idempotencyKey: `paperclip-runner-question:${run.id}:question-1`, payload: {
+        version: 1, questions: [{ id: "continue", prompt: "Continue?", selectionMode: "single", required: true, allowOther: false, options: [{ id: "yes", label: "Yes" }] }],
+        questionSet: { schema: "paperclip.question_set.v1", questions: [{ id: "continue", prompt: "Continue?", required: true, answerMode: "single_select", options: [{ id: "yes", label: "Yes" }] }] }, runtimeRequestId: "question-1",
+      } });
+    await start(); await issueService(db).update(f.issue.id, { status: "done" });
+    await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("cancelled"), { timeout: 15_000 });
+    expect((await getRun(run.id)).resultJson).toMatchObject({ cancelledByIssueStatus: "done", cancelledIssueId: f.issue.id });
+    await quiet();
+  });
+  it.each([false, true])("finishes a question cancellation whose run fails into a pending native retry before consumption (partial Stop fault: %s)", async partialStopFault => {
+    const f = await fixture();
+    let stopReceipt: Record<string, unknown> | undefined;
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", runtimeMode: "native", nativeIssueId: f.issue.id, nativePhase: "awaiting_input",
+      contextSnapshot: { issueId: f.issue.id } }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id,
+      runId: run.id, phase: "observed", recoveryState: "awaiting_evidence" });
+    if (partialStopFault) {
+      await db.execute(sql`create sequence question_cancel_failure_once`);
+      await db.execute(sql.raw(`create function question_cancel_failure_once() returns trigger language plpgsql as $$
+        begin if new.id = '${run.id}'::uuid and new.status = 'cancelled' and nextval('question_cancel_failure_once') = 1
+          then raise exception 'fixture question cancellation commit unavailable'; end if; return new; end $$`));
+      await db.execute(sql`create trigger question_cancel_failure_once before update on heartbeat_runs for each row execute function question_cancel_failure_once()`);
+      extraStops.push(async () => {
+        await db.execute(sql`drop trigger if exists question_cancel_failure_once on heartbeat_runs`);
+        await db.execute(sql`drop function if exists question_cancel_failure_once()`);
+        await db.execute(sql`drop sequence if exists question_cancel_failure_once`);
+      });
+    }
+    const marker = { version: 1, issueId: f.issue.id, kind: "issue_terminal", issueStatus: "done" };
+    const history = await db.insert(heartbeatRuns).values(Array.from({ length: 25 }, () => ({
+      companyId: f.companyId, agentId: f.agent.id, runtimeMode: "native", nativeIssueId: f.issue.id,
+      status: "failed", finishedAt: new Date(), nativePhase: "terminal_failure",
+      contextSnapshot: { nativeQuestionCancellation: marker }, resultJson: { nativeCancellation: {
+        schema: "paperclip.native-cancellation.v1", runId: randomUUID(), companyId: f.companyId, issueId: f.issue.id,
+        scope: "run", intentId: randomUUID(), intentAuditId: randomUUID(), dispatched: false, dispatchState: "pending",
+      } },
+    }))).returning();
+    await db.insert(nativeRunFinalizations).values(history.map(row => ({ companyId: f.companyId, issueId: f.issue.id,
+      runId: row.id, phase: "terminal_failure", failureCode: "native_retry_cancelled", recoveryState: "blocked" })));
+    draining = true; await start();
+    expect(await db.transaction(tx => requestNativeQuestionRunCancellation(tx, { companyId: f.companyId,
+      issueId: f.issue.id, sourceRunId: run.id, idempotencyKey: `paperclip-runner-question:${run.id}:raced-question`,
+      payload: { questionSet: {}, runtimeRequestId: "raced-question" },
+    }, { kind: "issue_terminal", issueStatus: "done" }))).toBe(run.id);
+    expect((await getRun(run.id)).contextSnapshot?.nativeQuestionCancellation).toMatchObject(marker);
+    await db.transaction(async tx => {
+      await notifyExecutionRunWork(tx, { companyId: f.companyId, runId: run.id, agentId: f.agent.id });
+      await tx.update(heartbeatRuns).set({ status: "failed", nativePhase: "retryable_failure", finishedAt: new Date() })
+        .where(eq(heartbeatRuns.id, run.id));
+      await tx.update(nativeRunFinalizations).set({ phase: "retryable_failure", recoveryState: "resuming_session",
+        nextAttemptAt: new Date(), leaseOwner: null, leaseExpiresAt: null }).where(eq(nativeRunFinalizations.runId, run.id));
+    });
+    draining = false; await advance(1_000);
+    if (partialStopFault) {
+      await vi.waitFor(() => expect(errors).toHaveLength(1));
+      expect(errors.shift()).toBeInstanceOf(AggregateError);
+      expect((await getRun(run.id)).status).toBe("failed");
+      const saved = await getRun(run.id);
+      stopReceipt = saved.resultJson?.nativeCancellation as Record<string, unknown>;
+      const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id));
+      expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", resultId: null });
+      expect(nativeRetryCancellationEligible({ runId: run.id, companyId: f.companyId, issueId: f.issue.id,
+        resultJson: saved.resultJson }, coordinator)).toBe(true);
+      const [debt] = await db.select({ pending: pendingFailedNativeRetryCancellationCondition() }).from(heartbeatRuns)
+        .where(eq(heartbeatRuns.id, run.id));
+      expect(debt.pending).toBe(true);
+    }
+    await advance(30_000);
+    await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("cancelled"));
+    expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0])
+      .toMatchObject({ phase: "terminal_failure", failureCode: "native_retry_cancelled", nextAttemptAt: null });
+    if (stopReceipt) expect((await getRun(run.id)).resultJson?.nativeCancellation).toMatchObject({
+      intentId: stopReceipt.intentId, intentAuditId: stopReceipt.intentAuditId,
+      acknowledgementAuditId: stopReceipt.acknowledgementAuditId,
+    });
+    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(26);
+    expect(await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.eventType, "native.process_start_requested"))).toEqual([]);
+    expect((await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.id} <> ${run.id}`)).every(row => row.status === "failed")).toBe(true);
+    await quiet();
+  });
+  it("preserves a fresh foreign controller until exact expiry and admits one process retry across two owners", async () => {
+    const f = await fixture(), expiry = new Date(Date.now() + 3_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "running",
+      runtimeMode: "legacy", responsibleUserId: f.actorId, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry,
+      executionStage: "preparing", updatedAt: new Date(Date.now() - 6 * 60_000), startedAt: new Date(Date.now() - 6 * 60_000) }).returning();
+    await start(true);
+    const other = heartbeatService(db), competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
+    await other.registerResidualRecoveryWork(competing, () => true).ready; extraStops.push(() => competing.stop());
+    expect((await getRun(run.id)).status).toBe("running"); expect(coordinator.nextWakeAt()).toBeLessThanOrEqual(expiry.getTime());
+    await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("failed"), { timeout: 10_000 });
+    let retryId: string;
+    await vi.waitFor(async () => {
+      const retries = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, run.id));
+      expect(retries).toHaveLength(1); retryId = retries[0].id;
+      await heartbeat.drainActiveRunExecutions(); await other.drainActiveRunExecutions();
+      expect((await getRun(retryId)).status).toBe("succeeded");
+      expect((await getRun(retryId)).stdoutExcerpt).toContain("live-recovery-proof");
+    }, { timeout: 45_000 });
+    await competing.stop(); extraStops.pop(); await quiet();
+  });
+  it("releases a settled native process only after matching host and birth evidence arrives", async () => {
+    const f = await fixture(), startedAt = new Date();
+    const child = spawn(process.execPath, ["-e", "console.log('identity-proof')"], { stdio: "ignore" });
+    const pid = child.pid!; await once(child, "exit");
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, runtimeMode: "native",
+      nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(), processPid: pid, processStartedAt: startedAt,
+      runnerProfileJson: { sessionCheckpoint: { sessionId: "preserved-native-session" } } }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, phase: "committed" });
+    await appendHeartbeatRunEvent(db, { companyId: f.companyId, agentId: f.agent.id, runId: run.id,
+      eventType: "native.process_identity_recorded", payload: { targetKind: "local", hostname: os.hostname(),
+        processPid: pid, processGroupId: null, processStartedAt: new Date(startedAt.getTime() - 1).toISOString() } });
+    await start(); expect((await getRun(run.id)).processPid).toBe(pid);
+    await appendHeartbeatRunEvent(db, { companyId: f.companyId, agentId: f.agent.id, runId: run.id,
+      eventType: "native.process_identity_recorded", payload: { targetKind: "local", hostname: os.hostname(),
+        processPid: pid, processGroupId: null, processStartedAt: startedAt.toISOString() } });
+    await vi.waitFor(async () => expect((await getRun(run.id)).processPid).toBeNull(), { timeout: 40_000 });
+    expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, run.id),
+      eq(heartbeatRunEvents.eventType, "native.local_process_stopped")))).toHaveLength(1);
+    await quiet();
+  });
+  it("excludes unverifiable local birth history until actual process identity arrives", async () => {
+    const f = await fixture(), startedAt = new Date();
+    const child = spawn(process.execPath, ["-e", "console.log('legacy-identity-proof')"], { stdio: "ignore" });
+    const pid = child.pid!; await once(child, "exit");
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(), processPid: pid,
+      runnerProfileJson: { sessionCheckpoint: { sessionId: "legacy-unverifiable-session" } } }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, phase: "committed" });
+    const checks: string[] = [], prior = db.$client.options.debug;
+    db.$client.options.debug = (_connection, query) => {
+      if (query.includes("sessionCheckpoint") && query.includes("for update")) checks.push(query);
+    };
+    try {
+      await start();
+      expect(checks).toEqual([]);
+      await db.transaction(tx => notifyLiveRunRecovery(tx, { companyId: f.companyId }));
+      await advance(31_000); expect(checks).toEqual([]);
+    } finally { db.$client.options.debug = prior; }
+    expect(await getRun(run.id)).toMatchObject({ processPid: pid, processStartedAt: null });
+    await quiet();
+    await persistHeartbeatRunProcessMetadata(db, run.id, { pid, processGroupId: null, startedAt: startedAt.toISOString() });
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await getRun(run.id)).processPid).toBeNull());
+    expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, run.id),
+      eq(heartbeatRunEvents.eventType, "native.local_process_stopped")))).toHaveLength(1);
+    await quiet();
+  });
+  it("preserves legacy null-birth recovery with exact server identity and an absent local PID group", async () => {
+    const f = await fixture();
+    const child = spawn(process.execPath, ["-e", "console.log('legacy-group-proof')"], { detached: true, stdio: "ignore" });
+    const pid = child.pid!; await once(child, "exit");
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(),
+      processPid: pid, processGroupId: pid, runnerProfileJson: { sessionCheckpoint: { sessionId: "legacy-exact-session" } } }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, phase: "committed" });
+    await appendHeartbeatRunEvent(db, { companyId: f.companyId, agentId: f.agent.id, runId: run.id,
+      eventType: "native.process_identity_recorded", payload: { targetKind: "local", hostname: os.hostname(),
+        processPid: pid, processGroupId: pid } });
+    await start();
+    expect(await getRun(run.id)).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
+    expect(await readNativeLocalProcessStop(db, f.companyId, run.id)).toEqual({ processPid: pid, processGroupId: pid });
+    await quiet();
+  });
+  it("releases remote process identity without local birth after the exact provider stop receipt", async () => {
+    const f = await fixture();
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(), processPid: 12345,
+      runnerProfileJson: { sessionCheckpoint: { sessionId: "remote-no-local-birth" } } }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id, phase: "committed" });
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run.id,
+      provider: "daytona", providerLeaseId: randomUUID(), status: "released", releasedAt: new Date() }).returning();
+    await start(); expect((await getRun(run.id)).processPid).toBe(12345);
+    const receipt = remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "stopped" });
+    expect(receipt).toBeDefined();
+    await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ cleanupStatus: "success",
+      metadata: { remoteExecutionTermination: receipt } }).where(eq(environmentLeases.id, lease.id)),
+    { companyId: f.companyId, runId: run.id, leaseId: lease.id });
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await getRun(run.id)).processPid).toBeNull());
+    expect((await getRun(run.id)).processStartedAt).toBeNull();
+    expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, run.id),
+      eq(heartbeatRunEvents.eventType, "native.local_process_stopped")))).toEqual([]);
+    await quiet();
+  });
+  it("renews only the active recovery owner and re-reads the moved controller deadline", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000), expiry = new Date(Date.now() + 20_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      runtimeMode: "legacy", status: "running", controllerBootId: legacyControllerBootId,
+      controllerLeaseExpiresAt: expiry, executionStage: "preparing", startedAt: old, updatedAt: old,
+      lastOutputAt: new Date() }).returning();
+    await start(); expect(deadlines.get("active-execution-recovery")!()).toBe(expiry.getTime());
+    const before = timeline.length;
+    expect(await renewLegacyControllerLease(db, run)).toBe(true);
+    const moved = (await getRun(run.id)).controllerLeaseExpiresAt!;
+    expect(moved.getTime()).toBeGreaterThan(expiry.getTime());
+    await advance(1_000);
+    expect(timeline.slice(before).filter(event => Array.isArray(event) && event[0] === "pass")
+      .map(event => (event as unknown[])[1])).toEqual(["active-execution-recovery", "active-execution-recovery"]);
+    await advance(expiry.getTime() - Date.now());
+    expect((await getRun(run.id)).status).toBe("running");
+    expect(deadlines.get("active-execution-recovery")!()).toBe(moved.getTime());
+    await heartbeat.cancelRun(run.id, "Operator cancelled the controller renewal fixture");
+    await quiet();
+  });
+  it("recovers orphan active leases after restart, including a deleted heartbeat reference, without starving behind 100 guarded rows", async () => {
+    const f = await fixture(), foreign = await fixture(), resource = randomUUID(), old = new Date(Date.now() - 6 * 60_000);
+    await db.insert(environmentLeases).values({ companyId: foreign.companyId, status: "retained", provider: "fixture", providerLeaseId: resource, updatedAt: old });
+    const guarded = await db.insert(environmentLeases).values(Array.from({ length: 105 }, () => ({ companyId: f.companyId,
+      status: "active", provider: "fixture", providerLeaseId: resource, updatedAt: old }))).returning();
+    const [deleted] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "succeeded", finishedAt: old }).returning();
+    const [orphan] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: deleted.id,
+      status: "active", provider: "local", updatedAt: old }).returning();
+    await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, deleted.id));
+    expect((await getLease(orphan.id)).heartbeatRunId).toBeNull(); await start();
+    await vi.waitFor(async () => expect((await getLease(orphan.id)).status).toBe("pending_cleanup"), { timeout: 15_000 });
+    expect(await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, f.companyId),
+      eq(environmentLeases.status, "pending_cleanup")))).toHaveLength(1);
+    for (const row of guarded) expect((await getLease(row.id)).status).toBe("active");
+    await quiet();
+  });
+  it("reactivates an orphan when a different company's physical resource owner releases it", async () => {
+    const f = await fixture(), other = await fixture(), old = new Date(Date.now() - 6 * 60_000), resource = randomUUID();
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: other.companyId, agentId: other.agent.id, status: "succeeded", finishedAt: old }).returning();
+    const [owner] = await db.insert(environmentLeases).values({ companyId: other.companyId, heartbeatRunId: run.id,
+      status: "retained", provider: "fixture", providerLeaseId: resource, updatedAt: old }).returning();
+    const [orphan] = await db.insert(environmentLeases).values({ companyId: f.companyId, status: "active",
+      provider: "fixture", providerLeaseId: resource, updatedAt: old }).returning();
+    await start(); expect((await getLease(orphan.id)).status).toBe("active");
+    await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "released", releasedAt: new Date() })
+      .where(eq(environmentLeases.id, owner.id)), { companyId: other.companyId, runId: run.id, leaseId: owner.id });
+    await vi.waitFor(async () => expect((await getLease(orphan.id)).status).toBe("pending_cleanup"), { timeout: 40_000 });
+    await quiet();
+  });
+  async function shutdownSource(released: boolean, cleanupExpiry?: number) {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, issueId: f.issue.id,
+      status: "failed", runtimeMode: "legacy", contextSnapshot: { issueId: f.issue.id }, finishedAt: old }).returning();
+    let [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run.id,
+      issueId: f.issue.id, status: "active", provider: "fixture", providerLeaseId: randomUUID(), acquiredAt: old, updatedAt: old,
+      metadata: { driver: "sandbox", sandboxProviderPlugin: true, pluginId: "shutdown-fixture" } }).returning();
+    await beginLegacyShutdownWorkspaceSettlement(db, run, new Date(Date.now() - 1000));
+    if (released) {
+      const prepared = await prepareSandboxStopAndRetain(db, await getLease(lease.id));
+      expect(prepared).toBeTruthy();
+      const settled = await settleStopOnlyCleanup(db, prepared!, { attemptId: String(prepared!.metadata?.pendingCleanupAttemptId),
+        receipt: { providerLeaseId: lease.providerLeaseId, state: "stopped" } });
+      expect(settled).toBeTruthy(); lease = settled!;
+      expect(hasConfirmedSandboxStopAndRetain(lease)).toBe(true);
+    } else {
+      [lease] = await db.update(environmentLeases).set({ status: "pending_cleanup", cleanupStatus: "failed",
+        metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ pendingCleanupInFlight: true,
+          ...(cleanupExpiry === undefined ? {} : { pendingCleanupLeaseExpiresAtMs: cleanupExpiry }) })}::jsonb`,
+      }).where(eq(environmentLeases.id, lease.id)).returning();
+    }
+    return { ...f, run, lease };
+  }
+  it.each(["competing allocation lease", "competing allocation run", "cleanup in flight"] as const)("parks unchanged shutdown refusal for %s until its owning commit", async gate => {
+    const competingGate = gate !== "cleanup in flight";
+    const f = await shutdownSource(competingGate);
+    const [otherRun] = gate === "competing allocation run" ? await db.insert(heartbeatRuns).values({
+      companyId: f.companyId, agentId: f.agent.id, status: "failed", runtimeMode: "legacy", finishedAt: new Date(),
+    }).returning() : [];
+    const [competing] = competingGate ? await db.insert(environmentLeases).values({ companyId: f.companyId,
+      heartbeatRunId: otherRun?.id,
+      status: "pending_cleanup", provider: f.lease.provider, providerLeaseId: f.lease.providerLeaseId,
+      acquiredAt: new Date(f.lease.acquiredAt.getTime() - 1000) }).returning() : [];
+    await start();
+    expect((await getLease(f.lease.id)).metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "pending" });
+    await quiet();
+    const releaseGate = async (database: typeof db) => writeEnvironmentCleanup(database, tx => tx.update(environmentLeases).set(
+      competing ? { status: "released", releasedAt: new Date() }
+        : { metadata: sql`${environmentLeases.metadata} || '{"pendingCleanupInFlight":false}'::jsonb` },
+    ).where(eq(environmentLeases.id, competing?.id ?? f.lease.id)), { companyId: f.companyId,
+      ...(otherRun ? { runId: otherRun.id } : { leaseId: competing?.id ?? f.lease.id }) });
+    await expect(db.transaction(async tx => {
+      await releaseGate(tx as unknown as typeof db);
+      throw new Error("roll back shutdown gate release");
+    })).rejects.toThrow("roll back shutdown gate release");
+    await reconcileDatabaseWork(db, "legacy-shutdown-settlement"); await Promise.allSettled([...passes]); await immediate();
+    expect((await getLease(f.lease.id)).metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "pending" });
+    await quiet();
+    await releaseGate(db);
+    await vi.waitFor(async () => {
+      await Promise.allSettled([...passes]);
+      const current = await getLease(f.lease.id);
+      expect(current.metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "expired" });
+      expect(current.leasePolicy).toBe("retain_on_failure");
+      expect(current.metadata?.workspaceRestoreRecovery).toMatchObject({ runId: f.run.id });
+      expect(current.providerLeaseId).toBe(f.lease.providerLeaseId);
+    }, { timeout: 15_000 });
+    expect(await db.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, f.companyId),
+      eq(issueRecoveryActions.fingerprint, `legacy-execution:${f.run.id}`)))).toHaveLength(1);
+    await reconcileDatabaseWork(db, "legacy-shutdown-settlement"); await Promise.allSettled([...passes]); await immediate();
+    await quiet();
+  });
+  it("retains the actual cleanup claim deadline after shutdown expiry and honors a moved expiry", async () => {
+    const initial = Date.now() + 1500, f = await shutdownSource(false, initial);
+    await start();
+    expect(deadlines.get("legacy-shutdown-settlement")!()).toBe(initial);
+    const moved = Date.now() + 4000;
+    await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({
+      metadata: sql`jsonb_set(${environmentLeases.metadata}, '{pendingCleanupLeaseExpiresAtMs}', ${JSON.stringify(moved)}::jsonb)`,
+    }).where(eq(environmentLeases.id, f.lease.id)), { companyId: f.companyId, leaseId: f.lease.id });
+    await advance(initial - Date.now());
+    expect((await getLease(f.lease.id)).metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "pending" });
+    expect(deadlines.get("legacy-shutdown-settlement")!()).toBe(moved);
+    await advance(moved - Date.now() - 1);
+    expect((await getLease(f.lease.id)).metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "pending" });
+    await advance(1);
+    await vi.waitFor(async () => expect((await getLease(f.lease.id)).metadata?.legacyShutdownWorkspaceSettlement)
+      .toMatchObject({ state: "expired" }));
+    expect((await getLease(f.lease.id)).metadata?.workspaceRestoreRecovery).toMatchObject({ runId: f.run.id });
+    await quiet();
+  });
+  it("expires the exact shutdown settlement marker into preserved recovery instead of inferring file export", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, issueId: f.issue.id,
+      status: "failed", runtimeMode: "legacy", contextSnapshot: { issueId: f.issue.id }, finishedAt: old }).returning();
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run.id,
+      issueId: f.issue.id, status: "active", provider: "fixture", providerLeaseId: randomUUID(), updatedAt: old,
+      metadata: { driver: "sandbox" } }).returning();
+    await beginLegacyShutdownWorkspaceSettlement(db, run, new Date(Date.now() + 1_500)); await start();
+    expect((await getLease(lease.id)).metadata?.legacyShutdownWorkspaceSettlement).toMatchObject({ state: "pending" });
+    await vi.waitFor(async () => expect((await getLease(lease.id)).metadata?.legacyShutdownWorkspaceSettlement)
+      .toMatchObject({ state: "expired" }), { timeout: 10_000 });
+    expect((await getLease(lease.id)).leasePolicy).toBe("retain_on_failure");
+    expect((await getLease(lease.id)).metadata?.workspaceRestoreRecovery).toBeTruthy();
+    expect((await getLease(lease.id)).providerLeaseId).toBe(lease.providerLeaseId);
+    expect((await getLease(lease.id)).metadata).toMatchObject({ workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", runId: run.id } });
+    await quiet();
+  });
+  it("settles stranded agent status at its saved grace", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "succeeded", finishedAt: old });
+    await db.transaction(async tx => {
+      await notifyStrandedAgentStatus(tx, { companyId: f.companyId, agentId: f.agent.id });
+      await tx.update(agents).set({ status: "running", updatedAt: old }).where(eq(agents.id, f.agent.id));
+    });
+    await start(); await vi.waitFor(async () => expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"));
+    await quiet();
+  });
+  it("retains only skipped agent lock work until a lock-only transaction releases it", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, status: "succeeded", finishedAt: old });
+    await db.update(agents).set({ status: "running", updatedAt: old }).where(eq(agents.id, f.agent.id));
+    let release!: () => void, locked!: () => void;
+    const held = new Promise<void>(resolve => { release = resolve; });
+    const ready = new Promise<void>(resolve => { locked = resolve; });
+    const lock = db.transaction(async tx => {
+      await tx.select({ id: agents.id }).from(agents).where(eq(agents.id, f.agent.id)).for("update");
+      locked(); await held;
+    });
+    try {
+      await ready; await start();
+      expect(deadlines.get("stranded-agent-status")!()).toBe(Date.now() + 30_000);
+      expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("running");
+    } finally { release(); await lock; }
+    await advance(29_999);
+    expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("running");
+    await advance(1);
+    await vi.waitFor(async () => expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"));
+    expect(await db.select().from(activityLog).where(and(eq(activityLog.entityId, f.agent.id),
+      eq(activityLog.action, "agent.status_reconciled")))).toHaveLength(1);
+    await quiet();
+  });
+  it("moves the silence threshold on actual output and folds same-run completion only at strict snooze expiry", async () => {
+    const f = await fixture(), now = new Date(), old = new Date(now.getTime() - 10 * 60_000);
+    await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
+    await db.update(agents).set({ status: "running" }).where(eq(agents.id, f.agent.id));
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issue.id },
+      responsibleUserId: f.actorId, startedAt: old, processStartedAt: old, updatedAt: old,
+      controllerBootId: randomUUID(), controllerLeaseExpiresAt: new Date(now.getTime() + 60 * 60_000) }).returning();
+    const output = createHeartbeatRunOutput(db, { run, runId: run.id, issueId: f.issue.id,
+      runLogStore: { begin: async () => ({ store: "local_file", logRef: "fixture-output.ndjson" }), append: async (_handle, event) => Buffer.byteLength(event.chunk) },
+      identityRedactor: { chunk: (_stream, chunk) => chunk, redact: value => value }, secretKeys: new Set(),
+      appendRunEvent: async (_run, event) => { await appendHeartbeatRunEvent(db, { companyId: f.companyId, agentId: f.agent.id, runId: run.id, ...event }); },
+      getCurrentUserRedactionOptions: async () => ({ enabled: false }) });
+    await (await output.begin(run)).onLog("stdout", "fresh actual output");
+    const progressed = await getRun(run.id), snoozedUntil = new Date(progressed.lastOutputAt!.getTime() + 5 * 60_000 + 2_000);
+    expect(progressed.lastOutputSeq).toBe(1); expect(progressed.lastOutputAt!.getTime()).toBeGreaterThan(old.getTime());
+    await issueService(db).update(f.issue.id, { status: "done" });
+    await persistActivity(db, { companyId: f.companyId, actorType: "agent", actorId: f.agent.id, agentId: f.agent.id,
+      runId: run.id, entityType: "issue", entityId: f.issue.id, action: "issue.updated", details: { status: "done" } });
+    await recoveryService(db, { enqueueWakeup: heartbeat.wakeup }).recordWatchdogDecision({ runId: run.id,
+      actor: { type: "board" }, decision: "snooze", snoozedUntil, reason: "Wait for the exact saved deadline" });
+    await start();
+    await advance(snoozedUntil.getTime() - Date.now() - 1);
+    expect((await getRun(run.id)).status).toBe("running");
+    await advance(1);
+    await vi.waitFor(async () => expect((await getRun(run.id)).status).toBe("succeeded"));
+    expect((await getRun(run.id)).resultJson).toMatchObject({ sourceResolvedWatchdogFold: { sourceIssueStatus: "done", sameRunEvidenceKind: "activity" } });
+    await quiet();
+  });
+  it("re-reads a concurrently moved stranded-agent grace before settlement", async () => {
+    const f = await fixture(), initialAt = new Date();
+    await db.update(agents).set({ status: "running", updatedAt: initialAt }).where(eq(agents.id, f.agent.id));
+    await start(); await advance(60_000);
+    const moved = new Date();
+    await db.update(agents).set({ updatedAt: moved }).where(eq(agents.id, f.agent.id));
+    await advance(initialAt.getTime() + 5 * 60_000 - Date.now());
+    expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("running");
+    await advance(moved.getTime() + 5 * 60_000 - Date.now());
+    await vi.waitFor(async () => expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"));
+    await quiet();
+  });
+  it("reaches a silent run behind more than 100 suppressed live runs and leaves only real controller deadlines", async () => {
+    const f = await fixture(), now = new Date(), old = new Date(now.getTime() - 20 * 60_000);
+    await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, f.issue.id));
+    const expiry = new Date(now.getTime() + 60 * 60_000);
+    const suppressed = await db.insert(heartbeatRuns).values(Array.from({ length: 105 }, () => ({ companyId: f.companyId,
+      agentId: f.agent.id, status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: f.issue.id },
+      startedAt: old, updatedAt: old, controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry }))).returning();
+    const [source] = await db.insert(issues).values({ companyId: f.companyId, status: "done", title: "Actual completed source", assigneeAgentId: f.agent.id }).returning();
+    const [target] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", runtimeMode: "legacy", contextSnapshot: { issueId: source.id },
+      startedAt: new Date(old.getTime() + 60_000), updatedAt: old,
+      controllerBootId: randomUUID(), controllerLeaseExpiresAt: expiry }).returning();
+    await db.insert(activityLog).values({ companyId: f.companyId, actorType: "agent", actorId: f.agent.id,
+      agentId: f.agent.id, runId: target.id, entityType: "issue", entityId: source.id, action: "issue.updated", details: { status: "done" } });
+    const broad = await recoveryService(db, { enqueueWakeup: heartbeat.wakeup }).scanSilentActiveRuns({ companyId: f.companyId });
+    expect(broad.scanned).toBe(100); expect((await getRun(target.id)).status).toBe("running");
+    await start(); await advance(31_000);
+    await vi.waitFor(async () => expect((await getRun(target.id)).status).toBe("succeeded"));
+    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "running"))).toHaveLength(suppressed.length);
+    expect(deadlines.get("active-run-silence")!()).toBeNull();
+    const [agentState] = await db.select().from(agents).where(eq(agents.id, f.agent.id));
+    const grace = agentState.updatedAt.getTime() + 5 * 60_000;
+    expect(agentState.status).toBe("running");
+    expect(deadlines.get("stranded-agent-status")!()).toBe(grace);
+    await advance(grace - Date.now() + 1);
+    expect(deadlines.get("stranded-agent-status")!()).toBe(expiry.getTime());
+    expect(coordinator.nextWakeAt(), JSON.stringify([...deadlines].map(([name, at]) => [name, at()]))).toBe(expiry.getTime());
+    const queries: string[] = [], prior = db.$client.options.debug;
+    db.$client.options.debug = (_connection, query) => { queries.push(query); };
+    try { await advance(20 * 60_000); expect(queries).toEqual([]); }
+    finally { db.$client.options.debug = prior; }
+  });
+  it("reconciles nested rollback and dispatches only the outer committed orphan producer", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000), rolledBack = randomUUID();
+    await start(); await quiet();
+    await expect(db.transaction(async outer => {
+      await outer.transaction(async inner => {
+        await notifyLiveResourceRecovery(inner, { companyId: f.companyId, leaseId: rolledBack });
+        await inner.insert(environmentLeases).values({ id: rolledBack, companyId: f.companyId, status: "active", provider: "local", updatedAt: old });
+      });
+      throw new Error("abort outer producer");
+    })).rejects.toThrow("abort outer producer");
+    await advance(31_000); expect(await getLease(rolledBack)).toBeUndefined(); await quiet();
+    const committed = randomUUID();
+    await db.transaction(async outer => {
+      await outer.transaction(async inner => {
+        await notifyLiveResourceRecovery(inner, { companyId: f.companyId, leaseId: committed });
+        await inner.insert(environmentLeases).values({ id: committed, companyId: f.companyId, status: "active", provider: "local", updatedAt: old });
+      });
+      expect(await getLease(committed)).toBeUndefined();
+    });
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await getLease(committed)).status).toBe("pending_cleanup"));
+    await quiet();
+  });
+  it("retains and re-reads a moved native controller deadline while awaiting recovery evidence", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000), expiry = new Date(Date.now() + 3_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id },
+      startedAt: old, updatedAt: old }).returning();
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id,
+      phase: "observed", recoveryState: "awaiting_evidence", controllerBootId: randomUUID(), controllerPid: process.pid + 100_000,
+      controllerProcessStartedAt: old, leaseExpiresAt: expiry });
+    await start(); expect(deadlines.get("active-execution-recovery")!()).toBe(expiry.getTime());
+    const moved = new Date(expiry.getTime() + 10_000);
+    await db.update(nativeRunFinalizations).set({ leaseExpiresAt: moved }).where(eq(nativeRunFinalizations.runId, run.id));
+    await advance(expiry.getTime() - Date.now());
+    expect((await getRun(run.id)).status).toBe("running"); expect((await getRun(run.id)).errorCode).toBeNull();
+    expect(deadlines.get("active-execution-recovery")!()).toBe(moved.getTime());
+    await db.transaction(async tx => {
+      await notifyExecutionRunWork(tx, { companyId: f.companyId, runId: run.id, agentId: f.agent.id });
+      await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
+      await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", recoveryState: "blocked", leaseExpiresAt: null })
+        .where(eq(nativeRunFinalizations.runId, run.id));
+    });
+    await quiet();
+  });
+  async function preparedOrphan() {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "failed", runtimeMode: "native", nativeIssueId: f.issue.id, issueId: f.issue.id,
+      finishedAt: old, updatedAt: old }).returning();
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId,
+      heartbeatRunId: run.id, issueId: f.issue.id, status: "active", provider: "daytona",
+      providerLeaseId: `sandbox-${run.id}`, metadata: { driver: "sandbox", remoteCwd: "/workspace" }, updatedAt: old }).returning();
+    await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: {
+      schema: "paperclip.native-workspace-sync/v3", state: "prepared", descriptorSha256: "a".repeat(64),
+      baselineSha256: "b".repeat(64), finalHostSha256: null, workspaceId: randomUUID(), leaseId: lease.id,
+      providerLeaseId: lease.providerLeaseId, remoteCwd: "/workspace", resourceDisposition: "destroy",
+    } } }).where(eq(heartbeatRuns.id, run.id));
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id,
+      runId: run.id, phase: "terminal_failure", recoveryState: "blocked" });
+    return { f, run, lease };
+  }
+  it.each(["task lock", "statement timeout"])("retries workspace preservation after real PostgreSQL %s without a producer mutation", async kind => {
+    const { f, run, lease } = await preparedOrphan();
+    const warnings = vi.spyOn(logger, "warn");
+    let retryAt = 0;
+    if (kind === "statement timeout") {
+      await db.execute(sql`create sequence live_preservation_slow_once`);
+      await db.execute(sql.raw(`create function live_preservation_slow_once() returns trigger language plpgsql as $$
+        begin if new.id = '${run.id}'::uuid and new.result_json->>'workspaceRestoreFailure' = 'restore_failed'
+          and nextval('live_preservation_slow_once') = 1 then perform pg_sleep(16); end if; return new; end $$`));
+      await db.execute(sql`create trigger live_preservation_slow_once before update on heartbeat_runs for each row execute function live_preservation_slow_once()`);
+      extraStops.push(async () => {
+        await db.execute(sql`drop trigger if exists live_preservation_slow_once on heartbeat_runs`);
+        await db.execute(sql`drop function if exists live_preservation_slow_once()`);
+        await db.execute(sql`drop sequence if exists live_preservation_slow_once`);
+      });
+    }
+    let release!: () => void, locked!: () => void;
+    const held = new Promise<void>(resolve => { release = resolve; });
+    const ready = new Promise<void>(resolve => { locked = resolve; });
+    const lock = kind === "task lock" ? db.transaction(async tx => {
+      await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, f.issue.id)).for("update");
+      locked(); await held;
+    }) : Promise.resolve();
+    if (kind !== "task lock") locked();
+    try {
+      await ready; await start();
+      expect((await getRun(run.id)).resultJson).toBeNull();
+      expect((await getLease(lease.id)).status).toBe("active");
+      expect(warnings).toHaveBeenCalledWith(expect.objectContaining({ runId: run.id,
+        code: kind === "task lock" ? "55P03" : "57014" }), expect.any(String));
+      retryAt = deadlines.get("orphan-active-leases")!()!;
+      expect(retryAt).toBeGreaterThan(Date.now());
+      expect(retryAt).toBeLessThanOrEqual(Date.now() + 30_000);
+      if (kind === "task lock") expect(retryAt).toBe(Date.now() + 30_000);
+    } finally { release(); await lock; }
+    await advance(retryAt - Date.now() - 1); expect((await getRun(run.id)).resultJson).toBeNull();
+    await advance(1);
+    await vi.waitFor(async () => expect((await getRun(run.id)).resultJson).toMatchObject({
+      workspaceRestoreFailure: "restore_failed", workspaceRestoreRecovery: { leaseIds: [lease.id] },
+    }));
+    expect(await db.select().from(activityLog).where(and(eq(activityLog.runId, run.id),
+      eq(activityLog.action, "issue.workspace_recovery_required")))).toHaveLength(1);
+    expect(await getLease(lease.id)).toMatchObject({ status: "pending_cleanup", leasePolicy: "retain_on_failure" });
+    await quiet();
+    expect(await db.select().from(activityLog).where(and(eq(activityLog.runId, run.id),
+      eq(activityLog.action, "issue.workspace_recovery_required")))).toHaveLength(1);
+  });
+  it.each(["company", "claim", "manual"])("parks workspace preservation authority gates without timeout retries: %s", async gate => {
+    const { f, run, lease } = await preparedOrphan();
+    if (gate === "company") {
+      const other = await fixture();
+      await db.update(environmentLeases).set({ companyId: other.companyId }).where(eq(environmentLeases.id, lease.id));
+    } else if (gate === "claim") {
+      await db.update(nativeRunFinalizations).set({ leaseOwner: "another-workspace-owner",
+        leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, run.id));
+    } else {
+      await db.update(nativeRunFinalizations).set({ phase: "observed", recoveryState: "blocked" })
+        .where(eq(nativeRunFinalizations.runId, run.id));
+    }
+    await start();
+    expect((await getRun(run.id)).resultJson).toBeNull();
+    expect((await getLease(lease.id)).status).toBe("active");
+    expect(deadlines.get("orphan-active-leases")!()).toBeNull();
+    expect(await db.select().from(activityLog).where(eq(activityLog.action, "issue.workspace_recovery_required"))).toEqual([]);
+    await quiet();
+  });
+  it("retains failed orphan work with bounded backoff after a real PostgreSQL write failure", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, status: "active", provider: "local", updatedAt: old }).returning();
+    await db.execute(sql`create sequence live_recovery_failure_once`);
+    await db.execute(sql.raw(`create function live_recovery_failure_once() returns trigger language plpgsql as $$
+      begin if new.id = '${lease.id}'::uuid and new.status = 'pending_cleanup' and nextval('live_recovery_failure_once') = 1
+        then raise exception 'fixture cleanup persistence unavailable'; end if; return new; end $$`));
+    await db.execute(sql`create trigger live_recovery_failure_once before update on environment_leases for each row execute function live_recovery_failure_once()`);
+    extraStops.push(async () => {
+      await db.execute(sql`drop trigger if exists live_recovery_failure_once on environment_leases`);
+      await db.execute(sql`drop function if exists live_recovery_failure_once()`);
+      await db.execute(sql`drop sequence if exists live_recovery_failure_once`);
+    });
+    await start();
+    expect(errors).toHaveLength(1); expect(errors.shift()).toBeInstanceOf(AggregateError);
+    expect((await getLease(lease.id)).status).toBe("active");
+    await advance(29_000); expect((await getLease(lease.id)).status).toBe("active");
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await getLease(lease.id)).status).toBe("pending_cleanup"));
+    await quiet();
+  });
+  it("parks suppressed worktree recovery without probes and reconstructs it on the committed feature change", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const runtimeEnv = { PAPERCLIP_IN_WORKTREE: "true", PAPERCLIP_INSTANCE_ID: "live-recovery-worktree" };
+    const settings = instanceSettingsService(db, { runtimeEnv });
+    await settings.updateExperimental({ enableWorktreeRunExecution: false });
+    heartbeat = heartbeatService(db, { runtimeEnv });
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, status: "active", provider: "local", updatedAt: old }).returning();
+    await start(); expect((await getLease(lease.id)).status).toBe("active"); await quiet();
+    try {
+      await settings.updateExperimental({ enableWorktreeRunExecution: true });
+      await advance(1_000);
+      await vi.waitFor(async () => expect((await getLease(lease.id)).status).toBe("pending_cleanup"));
+    } finally { await settings.updateExperimental({ enableWorktreeRunExecution: false }); }
+    await quiet();
+  });
+  it.each(["agent", "hold"] as const)("revalidates parked native session work on ordinary %s eligibility", async gate => {
+    const f = await fixture(), tree = issueTreeControlService(db);
+    const lifecycleWorker = configureAgentLifecycle(db, { requiredPluginIds: async () => [], runHost: async () => "complete", runPlugin: async () => "complete" });
+    extraStops.push(() => lifecycleWorker.stop());
+    await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
+    const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
+    const hold = gate === "hold" ? await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor }) : null;
+    if (gate === "agent") { await createAgentLifecycle(db).pauseAgent(f.agent.id); await lifecycleWorker.process(f.agent.id); }
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "failed", runtimeMode: "native", nativeIssueId: f.issue.id, contextSnapshot: { issueId: f.issue.id },
+      finishedAt: new Date(), errorCode: "saved_native_failure" }).returning();
+    await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, f.issue.id));
+    await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId: run.id,
+      phase: "retryable_failure", nextAttemptAt: new Date(Date.now() - 1_000) });
+    await start(); await quiet();
+    expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0].phase).toBe("retryable_failure");
+    const before = timeline.filter(event => Array.isArray(event) && event[0] === "finished" && event[1] === "native-process-sessions").length;
+    if (gate === "agent") { await createAgentLifecycle(db).resumeAgent(f.agent.id); await lifecycleWorker.process(f.agent.id); }
+    else await tree.releaseHold(f.companyId, f.issue.id, hold!.hold.id, { actor, reason: "Resume native recovery" });
+    await advance(1_000);
+    const deadline = performance.now() + 5_000;
+    while (timeline.filter(event => Array.isArray(event) && event[0] === "finished" && event[1] === "native-process-sessions").length <= before && performance.now() < deadline)
+      await realDelay(5);
+    expect(timeline.filter(event => Array.isArray(event) && event[0] === "finished" && event[1] === "native-process-sessions").length).toBeGreaterThan(before);
+    const expectedPhase = gate === "agent" ? "retryable_failure" : "terminal_failure";
+    expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)))[0].phase).toBe(expectedPhase);
+    if (gate === "agent") expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].status).toBe("blocked");
+    expect((await getRun(run.id)).errorCode).toBe("saved_native_failure");
+    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1);
+    await quiet();
+  });
+  it("discovers an active lease from the ordinary terminal run commit before any lease write", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "running", runtimeMode: "legacy", startedAt: new Date(), updatedAt: new Date(),
+      controllerBootId: legacyControllerBootId, controllerLeaseExpiresAt: new Date(Date.now() + 60 * 60_000) }).returning();
+    const [lease] = await db.insert(environmentLeases).values({ companyId: f.companyId, heartbeatRunId: run.id,
+      status: "active", provider: "local", updatedAt: old }).returning();
+    await start(); expect(deadlines.get("orphan-active-leases")!()).toBeNull();
+    const ended = await terminalizeLegacyExecution({ db, run, status: "failed", patch: { finishedAt: new Date(), errorCode: "owner_process_failure" } });
+    expect(ended?.status).toBe("failed"); expect((await getLease(lease.id)).status).toBe("active");
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await getLease(lease.id)).status).toBe("pending_cleanup"));
+    await quiet();
+  });
+  it.each(["accounting", "wake", "assignment_cancel"] as const)("settles parked agent status when its last %s receipt commits", async receipt => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    await db.update(agents).set({ status: "running", updatedAt: old }).where(eq(agents.id, f.agent.id));
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
+      status: "succeeded", finishedAt: old, costAccountingPending: receipt === "accounting",
+      usageJson: { accountingProviderWorkStarted: false } }).returning();
+    const [wake] = receipt !== "accounting" ? await db.insert(agentWakeupRequests).values({ companyId: f.companyId,
+      agentId: f.agent.id, runId: run.id, status: "claimed", source: "automation", reason: "fixture-last-receipt",
+      requestedByActorType: receipt === "assignment_cancel" ? "user" : "system", requestedByActorId: f.actorId, payload: { issueId: f.issue.id } }).returning() : [];
+    await start(); await quiet();
+    expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("running");
+    if (receipt === "accounting") expect(await accountRunCost(db, run.id)).toBe(true);
+    else if (receipt === "assignment_cancel") await issueService(db).update(f.issue.id, { assigneeAgentId: null, assigneeUserId: f.actorId });
+    else await createHeartbeatLifecycle(db, {} as Parameters<typeof createHeartbeatLifecycle>[1])
+      .setWakeupStatus(wake.id, "completed", { finishedAt: new Date() });
+    await advance(1_000);
+    await vi.waitFor(async () => expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle"));
+    await quiet();
+  });
+  it("retains first producer debt during drain and warm standby, then resumes without duplicate cleanup", async () => {
+    const f = await fixture(), old = new Date(Date.now() - 6 * 60_000);
+    draining = true; await start();
+    const [lease] = await writeEnvironmentCleanup(db, tx => tx.insert(environmentLeases).values({ companyId: f.companyId,
+      status: "active", provider: "local", updatedAt: old }).returning(), { companyId: f.companyId });
+    await advance(1_000); await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
+    expect((await getLease(lease.id)).status).toBe("active");
+    standby = true; draining = false; await advance(60_000); expect((await getLease(lease.id)).status).toBe("active");
+    standby = false; await advance(30_000);
+    await vi.waitFor(async () => expect((await getLease(lease.id)).status).toBe("pending_cleanup")); await quiet();
+  });
+});

server/src/__tests__/native-cancellation-request.integration.test.ts

@@ -2,19 +2,21 @@ import { randomUUID } from "node:crypto";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import { and, eq, inArray, sql } from "drizzle-orm";
 import { agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
 import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
 import { rethrowNativeCancellationLockConflict, nativeRetryCancellationCommitCondition, cancellationIntentId, claimCancellationRequest, startupCancellationFence } from "../services/native-runtime/native-cancellation-request.js";
 
 describe("atomic caller cancellation request ownership", () => {
-  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
   let db: ReturnType<typeof createDb>;
   let cancelNativeSession: typeof import("../services/native-runtime/native-session-executor.js").cancelNativeSession;
   beforeAll(async () => {
     ({ cancelNativeSession } = await import("../services/native-runtime/native-session-executor.js"));
-    database = await startEmbeddedPostgresTestDatabase("caller-stop-");
-    db = createDb(database.connectionString);
+    const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_CANCELLATION_TEST_DATABASE_URL");
+    if (!external) database = await startEmbeddedPostgresTestDatabase("caller-stop-");
+    db = createDb(external ?? database!.connectionString, { maxConnections: 6 });
   }, 90_000);
-  afterAll(async () => { await database?.cleanup(); });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
   async function fixture() {
     const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID();
     await db.insert(companies).values({ id: companyId, name: "Stop claims", issuePrefix: randomUUID().slice(0, 8) });

server/src/__tests__/native-settled-process-release.test.ts

@@ -1,9 +1,10 @@
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
 import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
 import { randomBytes, randomUUID } from "node:crypto";
 import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
 import { spawn } from "node:child_process";
 import os from "node:os";
-import { and, eq } from "drizzle-orm";
+import { and, eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
 import {
   agents,
@@ -39,8 +40,9 @@ vi.mock("../adapters/index.js", async () => {
   return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute })) };
 });
 
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_SETTLED_PROCESS_TEST_DATABASE_URL");
 const support = await getEmbeddedPostgresTestSupport();
-const suite = support.supported ? describe : describe.skip;
+const suite = external || support.supported ? describe : describe.skip;
 
 function startDetached() {
   const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" });
@@ -57,14 +59,15 @@ async function exitedPid() {
 
 suite("settled native run process identity", () => {
   let db: ReturnType<typeof createDb>;
-  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
   let companyId: string;
   let agentId: string;
   const children: ReturnType<typeof startDetached>[] = [];
 
   beforeAll(async () => {
-    temporary = await startEmbeddedPostgresTestDatabase("native-settled-process-");
-    db = createDb(temporary.connectionString);
+    if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-settled-process-");
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
+    if (external) await db.execute(sql`truncate companies cascade`);
     companyId = randomUUID();
     agentId = randomUUID();
     await db.insert(companies).values({ id: companyId, name: "Settled processes", issuePrefix: "SPR", defaultResponsibleUserId: "owner" });
@@ -242,6 +245,17 @@ suite("settled native run process identity", () => {
     },
   );
 
+  it.each(["absent", "live", "foreign host", "known birth mismatch"])("preserves exact legacy null-birth compatibility without clearing unsafe identity: %s", async kind => {
+    const child = kind === "live" ? startDetached() : undefined;
+    if (child) children.push(child);
+    const { runId, pid } = await seed({ ...(child ? { pid: child.pid! } : {}),
+      ...(kind === "foreign host" ? { hostname: `${os.hostname()}-foreign` } : {}) });
+    await db.update(heartbeatRuns).set({ processStartedAt: kind === "known birth mismatch" ? new Date(0) : null })
+      .where(eq(heartbeatRuns.id, runId));
+    expect(await releaseSettledNativeRunProcess(db, runId)).toBe(kind === "absent" ? "released" : kind === "live" ? "pending" : "skipped");
+    expect((await readRun(runId)).processPid).toBe(kind === "absent" ? null : pid);
+    expect(await hasNativeLocalProcessStop(db, companyId, runId)).toBe(kind === "absent");
+  });
   it("keeps the identity while the process is still alive, then releases it after exit", async () => {
     const child = startDetached();
     children.push(child);

server/src/__tests__/server-startup-feedback-export.test.ts

@@ -144,6 +144,8 @@ const {
     })),
     reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
     reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
+    registerResidualRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
+    registerActiveRunSilenceWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
@@ -993,6 +995,8 @@ describe("startServer feedback export wiring", () => {
     loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
     await startServer(); await settleStartupHousekeeping();
     expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
+    expect(heartbeatServiceMock.registerResidualRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
+    expect(heartbeatServiceMock.registerActiveRunSilenceWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
     expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
     expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
     expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
@@ -1439,7 +1443,8 @@ describe("startServer feedback export wiring", () => {
       intervalCallback?.();
       await new Promise<void>((resolve) => setImmediate(resolve));
       expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
-      expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
+      expect(heartbeatServiceMock.reapOrphanedRuns).not.toHaveBeenCalled();
+      expect(heartbeatServiceMock.scanSilentActiveRuns).not.toHaveBeenCalled();
       expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
       expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
       expect(retiredDetector).not.toHaveBeenCalled();
@@ -1756,20 +1761,30 @@ describe("startServer feedback export wiring", () => {
     }
   });
 
-  it("does not replay hot-restart adoption when the orphan reaper retries", async () => {
+  it("keeps residual recovery independent after a partial hot-restart adoption", async () => {
     loadConfigMock.mockReturnValue(buildTestConfig({
       heartbeatSchedulerEnabled: true,
       heartbeatSchedulerIntervalMs: 30000,
     }));
     heartbeatServiceMock.reconcileHotRestartAdoption.mockRejectedValueOnce(new Error("partial adoption"));
-    heartbeatServiceMock.reapOrphanedRuns
-      .mockRejectedValueOnce(new Error("transient reap failure"))
-      .mockResolvedValueOnce({ reaped: 0, runIds: [] });
+
 
     await startServer();
 
     expect(heartbeatServiceMock.reconcileHotRestartAdoption).toHaveBeenCalledTimes(1);
-    expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenCalledTimes(2);
+    expect(heartbeatServiceMock.reapOrphanedRuns).not.toHaveBeenCalled();
+    expect(heartbeatServiceMock.registerResidualRecoveryWork).toHaveBeenCalledTimes(1);
+    expect(heartbeatServiceMock.registerActiveRunSilenceWork).toHaveBeenCalledTimes(1);
+  });
+
+  it.each(["recoverPendingSessionGoalActions", "recoverActiveSessionGoals"] as const)("continues independent recovery after %s startup failure", async method => {
+    loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true }));
+    heartbeatServiceMock[method].mockRejectedValueOnce(new Error("goal owner unavailable"));
+    await startServer(); await settleStartupHousekeeping();
+    expect(heartbeatServiceMock.registerResidualRecoveryWork).toHaveBeenCalledTimes(1);
+    expect(heartbeatServiceMock.registerActiveRunSilenceWork).toHaveBeenCalledTimes(1);
+    expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(1);
+    expect(fakeServer.close).not.toHaveBeenCalled();
   });
 
   it("closes the bound listener when native startup recovery fails", async () => {

server/src/__tests__/slack-conversation-lifecycle.test.ts

@@ -451,8 +451,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
       let entered = false;
       await adapter.withIssueExecutionLock({ companyId: f.companyId, runId: f.runId, now: new Date() }, async (_locked, ports) => {
         entered = true;
-        if (operation === "cancel_wake") await ports.transaction.cancelDeferredWake({ companyId: f.companyId, wakeId, reason: "cancelled", now: new Date() });
-        if (operation === "fail_wake") await ports.transaction.failDeferredWake({ companyId: f.companyId, wakeId, now: new Date() });
+        if (operation === "cancel_wake") await ports.transaction.cancelDeferredWake({ companyId: f.companyId, wakeId, agentId: f.agentId, reason: "cancelled", now: new Date() });
+        if (operation === "fail_wake") await ports.transaction.failDeferredWake({ companyId: f.companyId, wakeId, agentId: f.agentId, now: new Date() });
         return { outcome: { kind: "released" as const }, postCommitEffects: [] };
       });
       expect(entered).toBe(true);

server/src/index.ts

@@ -67,7 +67,6 @@ import { chatCompletionDeliveryService } from "./services/chat-completion-delive
 // instrumentationReady before opening DB connections or constructing the
 // HTTP server, so trace coverage does not depend on incidental timing.
 import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js";
-import { createTrackedSingleFlight } from "./lib/single-flight.js";
 import { sentryReady, shutdownSentry, captureException } from "./sentry.js";
 import { setSentryRunLinkResolver } from "./sentry-links.js";
 import { createSentryRunLinkResolver } from "./services/sentry-run-links.js";
@@ -1265,16 +1264,6 @@ async function startServerWithDatabaseTeardown(
       });
     heartbeatSchedulerInFlight.add(tracked);
   };
-  const runPeriodicRecovery = createTrackedSingleFlight({
-    trackWork: trackHeartbeatSchedulerWork,
-    onSkipped: (inFlightForMs) => {
-      if (inFlightForMs >= 3 * config.heartbeatSchedulerIntervalMs) {
-        logger.error({ inFlightForMs }, "periodic heartbeat recovery remains in flight; duplicate pass suppressed");
-      } else {
-        logger.warn({ inFlightForMs }, "skipped periodic heartbeat recovery tick; previous chain still in flight");
-      }
-    },
-  });
   const waitForHeartbeatSchedulerIdle = async () => {
     while (heartbeatSchedulerInFlight.size > 0) {
       await Promise.allSettled([...heartbeatSchedulerInFlight]);
@@ -1651,7 +1640,7 @@ async function startServerWithDatabaseTeardown(
         // workers before cleanup or retry admission, including unmanaged installs.
         await timeStartupRecovery("plugin_workers", () => app.locals.bundledPluginsStartup);
         try {
-          const nativeRecovery = await timeStartupRecovery("native_recovery", () => heartbeat.recoverNativeRunsAfterRestart());
+          const nativeRecovery = await timeStartupRecovery("native_recovery", () => heartbeat.recoverNativeRunsAfterRestart(new Date(), undefined, { skipDelegatedWork: true }));
           if (nativeRecovery.dispositions.length > 0) {
             logger.info(
               {
@@ -1686,31 +1675,14 @@ async function startServerWithDatabaseTeardown(
         } catch (err) {
           logger.error(
             { err },
-            "startup hot-restart adoption reconciliation failed - orphan reaper will serve as degraded backstop",
+            "startup hot-restart adoption reconciliation failed - recovery deadlines remain registered",
           );
         }
 
-        for (let attempt = 1; attempt <= 2; attempt++) {
-          try {
-            const result = await timeStartupRecovery("orphan_reaping", () => heartbeat.reapOrphanedRuns());
-            logger.info(
-              { reaped: result.reaped, runIds: result.runIds },
-              "startup reap of orphaned heartbeat runs complete",
-            );
-            break;
-          } catch (err) {
-            if (attempt < 2) {
-              logger.warn({ err, attempt }, "startup reap failed, retrying");
-            } else {
-              logger.error(
-                { err },
-                "startup reap of orphaned heartbeat runs failed after retry - periodic reaper will serve as degraded backstop",
-              );
-            }
-          }
-        }
-
-        const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
+        const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions()).catch(err => {
+          logger.error({ err }, "startup session-goal action recovery failed");
+          return { enqueued: 0, invalid: 0 };
+        });
         if (
           recoveredGoalActions.enqueued > 0 ||
           recoveredGoalActions.invalid > 0
@@ -1720,17 +1692,16 @@ async function startServerWithDatabaseTeardown(
             "startup session-goal action outbox recovery reconciled pending controls",
           );
         }
-        const recoveredGoals = await timeStartupRecovery("active_goals", () => heartbeat.recoverActiveSessionGoals());
+        const recoveredGoals = await timeStartupRecovery("active_goals", () => heartbeat.recoverActiveSessionGoals()).catch(err => {
+          logger.error({ err }, "startup active session-goal recovery failed");
+          return { enqueued: 0 };
+        });
         if (recoveredGoals.enqueued > 0) {
           logger.warn(
             recoveredGoals,
             "startup session-goal recovery resumed durable agent goals",
           );
         }
-        const scanned = await heartbeat.scanSilentActiveRuns();
-        if (scanned.created > 0 || scanned.escalated > 0) {
-          logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
-        }
       })().catch((err) => {
         logger.error({ err }, "startup heartbeat recovery failed");
         throw err;
@@ -1739,6 +1710,10 @@ async function startServerWithDatabaseTeardown(
       await startupHeartbeatRecovery;
     }
 
+    if (config.heartbeatSchedulerEnabled) await heartbeat.registerResidualRecoveryWork(deliveryWork,
+      () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
+    if (config.heartbeatSchedulerEnabled) await heartbeat.registerActiveRunSilenceWork(deliveryWork,
+      () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
     if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
       () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
     if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
@@ -1893,22 +1868,6 @@ async function startServerWithDatabaseTeardown(
         if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
 
 
-        if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
-        if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
-          // Periodically reap orphaned runs (5-min staleness threshold) and make sure
-          // persisted queued work is still being driven forward.
-          runPeriodicRecovery(() => heartbeat
-            .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
-            .then(async () => {
-              const scanned = await heartbeat.scanSilentActiveRuns();
-              if (scanned.created > 0 || scanned.escalated > 0) {
-                logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
-              }
-            })
-            .catch((err) => {
-              logger.error({ err }, "periodic heartbeat recovery failed");
-            }));
-        }
       })().catch((err) => {
         logger.error({ err }, "heartbeat scheduler tick failed");
       }));

server/src/modules/active-run-watchdog/adapters/deadline-work.ts

@@ -0,0 +1,62 @@
+import { and, asc, eq, gt, sql } from "drizzle-orm";
+import { heartbeatRuns, type Db } from "@paperclipai/db";
+import type { createDeliveryWorkCoordinator } from "../../../services/delivery-work-coordinator.js";
+import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
+import type { LiveRunRecoveryScope } from "../../../services/live-run-recovery-notifications.js";
+import { registerScopedDeadlineWork } from "../../../services/scoped-deadline-work.js";
+import type { ScanSilentActiveRunsOptions } from "../application/use-cases.js";
+import type { WatchdogRunReader } from "../application/ports.js";
+
+type Entry = { id: string; companyId: string; issueId: string | null; dueAt: number };
+
+export function registerActiveRunSilenceWork(db: Db,
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  owner: { reader: WatchdogRunReader; scan(input: ScanSilentActiveRunsOptions): Promise<unknown>;
+    suspicionThresholdMs: number; enabled(): Promise<boolean> },
+  canRun: () => boolean,
+) {
+  const cursors = new Map<string, string>();
+  let enabled = false;
+  return registerScopedDeadlineWork<Entry, LiveRunRecoveryScope>(coordinator, {
+    queue: DELIVERY_QUEUES.activeRunSilence,
+    reconstructionIntervalMs: 30_000,
+    beforePass: async () => { enabled = await owner.enabled(); },
+    decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
+    matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.runId || entry.id === scope.runId) && (!scope.issueId || entry.issueId === scope.issueId),
+    load: async scope => {
+      const key = scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId }) : "*";
+      if (!enabled) { cursors.delete(key); return []; }
+      const cursor = cursors.get(key);
+      const issueId = sql<string | null>`coalesce(${heartbeatRuns.contextSnapshot}->>'issueId', ${heartbeatRuns.contextSnapshot}->>'taskId')`;
+      const rows = await db.select({ issueId, id: heartbeatRuns.id, companyId: heartbeatRuns.companyId,
+        lastActivity: sql<Date>`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.processStartedAt},
+          ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
+      }).from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "running"),
+        scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
+        scope?.runId ? eq(heartbeatRuns.id, scope.runId) : undefined,
+        scope?.issueId ? eq(issueId, scope.issueId) : undefined,
+        cursor ? gt(heartbeatRuns.id, cursor) : undefined,
+      )).orderBy(asc(heartbeatRuns.id)).limit(100);
+      const complete = rows.length < 100;
+      if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
+      return { complete, rows: rows.map(row => ({ id: row.id, companyId: row.companyId, issueId: row.issueId,
+        dueAt: row.lastActivity.getTime() + owner.suspicionThresholdMs })) };
+    },
+    check: async entry => {
+      if (!enabled) return null;
+      const run = await owner.reader.findRunForCompany(entry.companyId, entry.id);
+      if (!run || run.status !== "running") return null;
+      const started = run.lastOutputAt ?? run.processStartedAt ?? run.startedAt ?? run.createdAt;
+      if (!started) return null;
+      const due = started.getTime() + owner.suspicionThresholdMs;
+      if (due > Date.now()) return { ...entry, dueAt: due };
+      const decision = await owner.reader.findLatestDecision(entry.companyId, entry.id, new Date());
+      if (decision.quietUntilDecision) return { ...entry, dueAt: decision.quietUntilDecision.snoozedUntil.getTime() };
+      let next: number | null = null;
+      await owner.scan({ companyId: entry.companyId, runId: entry.id, onDeferredDeadline: at => {
+        next = next === null ? at : Math.min(next, at);
+      } });
+      return next === null ? null : { ...entry, dueAt: next };
+    },
+  }, canRun);
+}

server/src/modules/active-run-watchdog/adapters/postgres.ts

@@ -1,3 +1,4 @@
+import { notifyActiveRunSilence, notifyLiveRunRecovery, notifyLiveResourceRecovery } from "../../../services/live-run-recovery-notifications.js";
 import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js";
 import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
@@ -101,6 +102,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
 
   async function findCandidateSilentRuns(input: {
     companyId?: string;
+    runId?: string;
     suspicionBefore: Date;
     issueCreatedAtGte?: Date | null;
   }): Promise<RunSnapshot[]> {
@@ -110,6 +112,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
       .where(
         and(
           input.companyId ? eq(heartbeatRuns.companyId, input.companyId) : undefined,
+          input.runId ? eq(heartbeatRuns.id, input.runId) : undefined,
           eq(heartbeatRuns.status, "running"),
           sql`coalesce(${heartbeatRuns.lastOutputAt}, ${heartbeatRuns.processStartedAt}, ${heartbeatRuns.startedAt}, ${heartbeatRuns.createdAt}) <= ${input.suspicionBefore.toISOString()}::timestamptz`,
         ),
@@ -283,8 +286,9 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
   }
 
   async function recordDecision(companyId: string, input: RecordDecisionInput): Promise<WatchdogDecisionRecord> {
-    const [row] = await db
-      .insert(heartbeatRunWatchdogDecisions)
+    const [row] = await db.transaction(async tx => {
+      await notifyActiveRunSilence(tx, { companyId, runId: input.runId });
+      return tx.insert(heartbeatRunWatchdogDecisions)
       .values({
         companyId,
         runId: input.runId,
@@ -297,6 +301,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
         createdByRunId: input.createdByRunId,
       })
       .returning();
+    });
 
     await logActivity(db, {
       companyId,
@@ -344,6 +349,8 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
     };
 
     const transactionResult = await db.transaction(async (tx) => {
+      await notifyLiveRunRecovery(tx, input.run);
+      await notifyLiveResourceRecovery(tx, input.run);
       await notifyInstructionCollection(tx, { companyId, runId: input.run.id });
       await notifyExternalAgentSettlement(tx, { companyId, runId: input.run.id });
       await notifyCancelledWorkspacePreservation(tx, { companyId, runId: input.run.id });

server/src/modules/active-run-watchdog/application/ports.ts

@@ -13,6 +13,7 @@ import type {
 
 export type FindCandidateSilentRunsInput = {
   companyId?: string;
+  runId?: string;
   suspicionBefore: Date;
   issueCreatedAtGte?: Date | null;
 };

server/src/modules/active-run-watchdog/application/types.ts

@@ -114,7 +114,7 @@ export type FoldSourceResolvedRunInput = {
 
 export type FoldOutcome =
   | { kind: "folded"; evaluationIssueId: string | null }
-  | { kind: "stale" };
+  | { kind: "stale"; retryAt?: number };
 
 export type WatchdogDecisionActor =
   | { type: "board"; userId?: string | null; runId?: string | null }

server/src/modules/active-run-watchdog/application/use-cases.ts

@@ -100,7 +100,7 @@ export function createFoldSourceResolvedRun(deps: FoldSourceResolvedRunDeps) {
       fallbackProcessGroupId: input.run.processGroupId,
     });
 
-    return deps.writer.foldSourceResolvedRun(input.run.companyId, {
+    const outcome = await deps.writer.foldSourceResolvedRun(input.run.companyId, {
       run: input.run,
       sourceIssue: input.sourceIssue,
       evidence: input.evidence,
@@ -110,6 +110,8 @@ export function createFoldSourceResolvedRun(deps: FoldSourceResolvedRunDeps) {
       cleanup,
       now: input.now,
     });
+    return outcome.kind === "stale" && cleanup.attempted && cleanup.outcome === "termination_sent_still_running"
+      ? { ...outcome, retryAt: input.now.getTime() + 30_000 } : outcome;
   };
 }
 
@@ -227,12 +229,14 @@ export type ScanSilentActiveRunsDeps = {
 
 export type ScanSilentActiveRunsOptions = {
   now?: Date;
+  runId?: string;
+  onDeferredDeadline?: (at: number) => void;
   companyId?: string;
   issueCreatedAtGte?: Date | null;
 };
 
 type InspectOutcome =
-  | { kind: "skipped" }
+  | { kind: "skipped"; retryAt?: number }
   | { kind: "existing"; evaluationIssueId: string }
   | { kind: "folded"; evaluationIssueId: string | null };
 
@@ -283,7 +287,7 @@ export function createScanSilentActiveRuns(deps: ScanSilentActiveRunsDeps) {
         });
         return foldOutcome.kind === "folded"
           ? { kind: "folded", evaluationIssueId: foldOutcome.evaluationIssueId }
-          : { kind: "skipped" };
+          : { kind: "skipped", ...(foldOutcome.retryAt === undefined ? {} : { retryAt: foldOutcome.retryAt }) };
       }
     }
 
@@ -305,6 +309,7 @@ export function createScanSilentActiveRuns(deps: ScanSilentActiveRunsDeps) {
     const suspicionBefore = new Date(now.getTime() - deps.suspicionThresholdMs);
     const candidates = await deps.reader.findCandidateSilentRuns({
       companyId: opts?.companyId,
+      runId: opts?.runId,
       suspicionBefore,
       issueCreatedAtGte: opts?.issueCreatedAtGte,
     });
@@ -331,6 +336,7 @@ export function createScanSilentActiveRuns(deps: ScanSilentActiveRunsDeps) {
         now,
         dismissedFalsePositive: decisionState.dismissedFalsePositive,
       });
+      if (outcome.kind === "skipped" && outcome.retryAt !== undefined) opts?.onDeferredDeadline?.(outcome.retryAt);
       if (outcome.kind === "existing") result.existing += 1;
       else if (outcome.kind === "folded") result.folded += 1;
       else result.skipped += 1;

server/src/modules/active-run-watchdog/index.ts

@@ -1,3 +1,4 @@
+import { registerActiveRunSilenceWork } from "./adapters/deadline-work.js";
 import type { Db } from "@paperclipai/db";
 import { createPostgresWatchdogAdapter } from "./adapters/postgres.js";
 import { createProcessAdapter } from "./adapters/process.js";
@@ -29,17 +30,19 @@ export function createActiveRunWatchdog(db: Db, config: ActiveRunWatchdogConfig)
     processController,
   });
 
+  const scanSilentActiveRuns = createScanSilentActiveRuns({ reader: postgresAdapter, foldSourceResolvedRun,
+    suspicionThresholdMs: config.suspicionThresholdMs });
+
   return {
+    registerSilenceWork: (coordinator: Parameters<typeof registerActiveRunSilenceWork>[1], canRun: () => boolean,
+      scan = scanSilentActiveRuns, enabled = async () => true) => registerActiveRunSilenceWork(db, coordinator, { reader: postgresAdapter,
+        scan, enabled, suspicionThresholdMs: config.suspicionThresholdMs }, canRun),
     buildRunOutputSilence: createBuildRunOutputSilence({
       reader: postgresAdapter,
       suspicionThresholdMs: config.suspicionThresholdMs,
       criticalThresholdMs: config.criticalThresholdMs,
     }),
-    scanSilentActiveRuns: createScanSilentActiveRuns({
-      reader: postgresAdapter,
-      foldSourceResolvedRun,
-      suspicionThresholdMs: config.suspicionThresholdMs,
-    }),
+    scanSilentActiveRuns,
     recordWatchdogDecision: createRecordWatchdogDecision({
       reader: postgresAdapter,
       writer: postgresAdapter,

server/src/modules/run-dispatch/adapters/postgres.ts

@@ -1,3 +1,4 @@
+import { notifyExecutionRunWork } from "../../../services/execution-control-notifications.js";
 import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
 import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
 import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
@@ -687,6 +688,7 @@ export function createPostgresRunDispatchAdapter(
     | { applied: true; run: HeartbeatRun; postCommitEffects: PostCommitEffect[] }
     | { applied: false }
   > {
+    await notifyExecutionRunWork(tx, { companyId: input.companyId, runId: input.runId });
     await notifyInstructionCollection(tx, { companyId: input.companyId, runId: input.runId });
     await notifyCancelledWorkspacePreservation(tx, { companyId: input.companyId, runId: input.runId });
     await notifyScheduledRetryCancellation(tx, { companyId: input.companyId, runId: input.runId });
@@ -732,6 +734,7 @@ export function createPostgresRunDispatchAdapter(
     tx: Db,
     input: CancelSuppressedRetryInput,
   ): Promise<CancelSuppressedRetryResult> {
+    await notifyExecutionRunWork(tx, { companyId: input.companyId, runId: input.runId });
     await notifyInstructionCollection(tx, { companyId: input.companyId, runId: input.runId });
     await notifyExternalAgentSettlement(tx, { companyId: input.companyId, runId: input.runId });
     await notifyCancelledWorkspacePreservation(tx, { companyId: input.companyId, runId: input.runId });
@@ -1018,6 +1021,7 @@ export function createPostgresRunDispatchAdapter(
     now: Date,
     providerDispatchNotStarted = false,
   ): Promise<CancelStaleQueuedRunOutcome> {
+      await notifyExecutionRunWork(tx, { companyId: run.companyId, agentId: run.agentId, runId: run.id });
       await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id });
       await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id });
       await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });

server/src/modules/wake-queue/adapters/postgres.test.ts

@@ -871,7 +871,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
       expect(candidate?.id).toBe(wakeId);
       const agent = await ports.transaction.findInvokableAgent({ companyId, agentId: foreignAgentId });
       expect(agent).toBeNull();
-      const failed = await ports.transaction.failDeferredWake({ companyId, wakeId: candidate!.id, now: new Date() });
+      const failed = await ports.transaction.failDeferredWake({ companyId, wakeId: candidate!.id, agentId: candidate!.agentId, now: new Date() });
       expect(failed).toBe(true);
       return { outcome: { kind: "released" as const }, postCommitEffects: [] };
     });
@@ -903,6 +903,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
       async (_locked, ports) => {
         const cancelledUnderWrongCompany = await ports.transaction.cancelDeferredWake({
           companyId: otherCompanyId,
+          agentId,
           wakeId,
           reason: "cross-company cancel attempt",
           now: new Date(),
@@ -911,6 +912,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
 
         const failedUnderWrongCompany = await ports.transaction.failDeferredWake({
           companyId: otherCompanyId,
+          agentId,
           wakeId,
           now: new Date(),
         });

server/src/modules/wake-queue/adapters/postgres.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "../../../services/live-run-recovery-notifications.js";
 import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
 import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
 import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
@@ -275,6 +276,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
             throw error;
           }
           const now = new Date();
+          await notifyStrandedAgentStatus(tx, { companyId, agentId: row.agentId });
           await notifyChatInboundWakeNotices(tx, { companyId, issueId });
           await tx
             .update(agentWakeupRequests)
@@ -320,7 +322,8 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
       return { liveNonSelfCommentIds, containedSelfAuthoredComment };
     },
 
-    async cancelDeferredWake({ companyId, wakeId, reason, now }) {
+    async cancelDeferredWake({ companyId, wakeId, agentId, reason, now }) {
+      await notifyStrandedAgentStatus(tx, { companyId, agentId });
       await notifyChatInboundWakeNotices(tx, { companyId });
       const rows = await tx
         .update(agentWakeupRequests)
@@ -328,6 +331,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
         .where(
           and(
             eq(agentWakeupRequests.id, wakeId),
+            eq(agentWakeupRequests.agentId, agentId),
             eq(agentWakeupRequests.companyId, companyId),
             eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
           ),
@@ -352,7 +356,8 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
       return row ? toDeferredWakeCandidate(row) : null;
     },
 
-    async failDeferredWake({ companyId, wakeId, now }) {
+    async failDeferredWake({ companyId, wakeId, agentId, now }) {
+      await notifyStrandedAgentStatus(tx, { companyId, agentId });
       await notifyChatInboundWakeNotices(tx, { companyId });
       const rows = await tx
         .update(agentWakeupRequests)
@@ -365,6 +370,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
         .where(
           and(
             eq(agentWakeupRequests.id, wakeId),
+            eq(agentWakeupRequests.agentId, agentId),
             eq(agentWakeupRequests.companyId, companyId),
             eq(agentWakeupRequests.status, DEFERRED_WAKE_STATUS),
           ),

server/src/modules/wake-queue/adapters/queued-comment-postgres.ts

@@ -1,3 +1,5 @@
+import { notifyStrandedAgentStatus } from "../../../services/live-run-recovery-notifications.js";
+import { notifyExecutionRunWork } from "../../../services/execution-control-notifications.js";
 import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
 import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
 import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
@@ -53,7 +55,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = {
   syncCommentExternalObjectsSafely(commentId: string, tx: Db): Promise<void>;
 };
 
-function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction {
+function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps, wakeAgentId: string): QueuedCommentQueueTransaction {
   return {
     async updateCommentBody({ issueId, commentId, body, updatedAt }) {
       await notifyStatusCardIssueChange(tx, companyId);
@@ -103,6 +105,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu
     },
 
     async cancelWake({ wakeId, reason, now }) {
+      await notifyStrandedAgentStatus(tx, { companyId, agentId: wakeAgentId });
       await notifyChatInboundWakeNotices(tx, { companyId, issueId });
       await tx
         .update(agentWakeupRequests)
@@ -111,6 +114,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu
     },
 
     async cancelQueueRun({ queueRunId, reason, now }) {
+      await notifyExecutionRunWork(tx, { companyId, runId: queueRunId });
       await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId });
       await notifyInstructionCollection(tx, { companyId, runId: queueRunId });
       await notifyExternalAgentSettlement(tx, { companyId, runId: queueRunId });
@@ -311,7 +315,7 @@ export function createQueuedCommentIssueLockWriter(db: Db, deps: QueuedCommentQu
               .then((rows) => rows[0] ?? null)
           : null;
 
-        const transaction = buildTransaction(tx, companyId, input.issue.id, deps);
+        const transaction = buildTransaction(tx, companyId, input.issue.id, deps, wakeRow.agentId);
         const wake = toWakeRow(wakeRow);
         const queueRun = queueRunRow ? toRunRow(queueRunRow) : null;
         const activeRun = activeRunRow ? toRunRow(activeRunRow) : null;

server/src/modules/wake-queue/application/ports.ts

@@ -123,6 +123,7 @@ export interface WakeQueueTransaction {
   cancelDeferredWake(input: {
     companyId: string;
     wakeId: string;
+    agentId: string;
     reason: string;
     now: Date;
   }): Promise<boolean>;
@@ -135,7 +136,7 @@ export interface WakeQueueTransaction {
     now: Date;
   }): Promise<DeferredWakeCandidate | null>;
   /** Sets `status = 'failed'` guarded by the current `deferred_issue_execution` status. */
-  failDeferredWake(input: { companyId: string; wakeId: string; now: Date }): Promise<boolean>;
+  failDeferredWake(input: { companyId: string; wakeId: string; agentId: string; now: Date }): Promise<boolean>;
   getPauseHoldFacts(input: {
     companyId: string;
     issueId: string;

server/src/modules/wake-queue/application/use-cases.ts

@@ -191,6 +191,7 @@ async function runReleaseDrain(
       await ports.transaction.cancelDeferredWake({
         companyId: run.companyId,
         wakeId: candidate.id,
+        agentId: candidate.agentId,
         reason: "Delegation closing note refers to completed child work",
         now: input.now,
       });
@@ -212,6 +213,7 @@ async function runReleaseDrain(
         await ports.transaction.cancelDeferredWake({
           companyId: run.companyId,
           wakeId: candidate.id,
+          agentId: candidate.agentId,
           reason: "Deferred task messages now belong to the current assignee",
           now: input.now,
         });
@@ -260,6 +262,7 @@ async function runReleaseDrain(
       await ports.transaction.cancelDeferredWake({
         companyId: run.companyId,
         wakeId: candidate.id,
+        agentId: candidate.agentId,
         reason: commentAction.selfAuthored
           ? "Deferred wake contained only comments authored by the finishing run"
           : "Queued messages were discarded before promotion",
@@ -288,7 +291,7 @@ async function runReleaseDrain(
     });
 
     if (wakeOutcome.kind === "fail_not_invokable") {
-      await ports.transaction.failDeferredWake({ companyId: run.companyId, wakeId: workingCandidate.id, now: input.now });
+      await ports.transaction.failDeferredWake({ companyId: run.companyId, wakeId: workingCandidate.id, agentId: workingCandidate.agentId, now: input.now });
       continue;
     }
 
@@ -300,7 +303,7 @@ async function runReleaseDrain(
     if (wakeOutcome.kind === "cancel_pause_hold") {
       await ports.transaction.cancelDeferredWake({
         companyId: run.companyId,
-        wakeId: workingCandidate.id,
+        wakeId: workingCandidate.id, agentId: workingCandidate.agentId,
         reason: "Deferred wake suppressed by active subtree pause hold",
         now: input.now,
       });
@@ -374,7 +377,7 @@ async function promoteDeferredWake(
   ) {
     await ports.transaction.cancelDeferredWake({
       companyId: run.companyId,
-      wakeId: workingCandidate.id,
+      wakeId: workingCandidate.id, agentId: workingCandidate.agentId,
       reason: "Deferred execution wake no longer applies to a terminal task",
       now: input.now,
     });

server/src/services/activity-log.ts

@@ -1,3 +1,4 @@
+import { notifyActiveRunSilence } from "./live-run-recovery-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { beginRunPhase } from "./execution-preparation-timing.js";
 import type { ActivityPublication } from "../types/activity-publication.js";
@@ -181,6 +182,8 @@ export async function persistActivity(db: Db, input: LogActivityInput) {
   const [activity] = input.entityType === "issue"
     ? await db.transaction(async tx => {
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, `task:${input.entityId}`);
+      if (input.runId && input.action === "issue.updated" && ["done", "cancelled"].includes(String(input.details?.status)))
+        await notifyActiveRunSilence(tx, { companyId: input.companyId, runId: input.runId });
       return insert(tx as unknown as Db);
     })
     : await insert(db);

server/src/services/agent-configuration.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
@@ -87,6 +88,7 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
       existing.adapterConfig,
     );
   }
+  await notifyStrandedAgentStatus(db, { companyId: existing.companyId, agentId: existing.id });
   await notifyRunDispatch(db, { companyId: existing.companyId });
   await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
   if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {

server/src/services/agent-instruction-working-copies.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { notifyInstructionCollection, type InstructionCollectionScope } from "./instruction-collection-notifications.js";
 import { notifyDirectoryCleanup, type DirectoryCleanupScope } from "./directory-cleanup-notifications.js";
 import { notifyInstructionCommit, type InstructionCommitScope } from "./instruction-commit-notifications.js";
@@ -101,6 +102,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
     const signalsCleanup = directoryCleanupEntry(row) !== null || directoryCleanupEntry({ ...row, ...values }) !== null;
     const signalsCommit = row.state === "pending_commit" || values.state === "pending_commit";
     const write = async (tx: Db) => {
+      await notifyStrandedAgentStatus(tx, { companyId: row.companyId, agentId: row.agentId });
       if (signalsCommit)
         await notifyInstructionCommit(tx, { companyId: row.companyId, runId: row.runId });
       if (signalsCleanup) await notifyDirectoryCleanup(tx, { companyId: row.companyId, runId: row.runId });
@@ -112,7 +114,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
       )).returning();
       return updated;
     };
-    return signalsCommit || signalsCleanup || signalsCollection ? db.transaction(tx => write(tx as unknown as Db)) : write(db);
+    return db.transaction(tx => write(tx as unknown as Db));
   }
   async function patch(row: Copy, values: Partial<typeof copies.$inferInsert>) {
     const updated = await tryPatch(row, values);
@@ -204,6 +206,7 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
     if (!existing) {
       await db.transaction(async tx => {
         await notifyInstructionCollection(tx, { companyId: input.companyId, runId: input.runId });
+        await notifyStrandedAgentStatus(tx, { companyId: input.companyId, agentId: input.agentId });
       await tx.insert(copies).values({
         runId: input.runId, companyId: input.companyId, agentId: input.agentId,
         responsibleUserId: bound.onBehalfOfUserId!, entryFile,

server/src/services/agent-lifecycle.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
@@ -43,6 +44,7 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
     async beforeStatusChange(tx, companyId, agentId, before, after) {
       await notifyAttentionRetentionWork(tx, { companyId });
       if (before !== after) {
+        await notifyStrandedAgentStatus(tx, { companyId, agentId });
         await notifyRunDispatch(tx, { companyId });
         await notifyDeferredExecutionEligibility(tx, { companyId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);

server/src/services/chat-channels/failed-run-retry.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js";
 import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
 import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
 import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
@@ -1923,6 +1924,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
           // Receipt existence proves the earlier wait, not current authority.
           // Retire only this unadmitted wake; an admitted run keeps its history.
           await db.transaction(async tx => {
+            await notifyStrandedAgentStatus(tx, { companyId: claimed.companyId, agentId: receipt.agentId });
             await notifyChatInboundWakeNotices(tx, {
               companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
             });

server/src/services/chat-channels/slack/session-stop.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "../../live-run-recovery-notifications.js";
 import { notifyChatInboundWakeNotices } from "../../chat-inbound-wake-notice-notifications.js";
 import type { EndpointRow, DbTransaction, DbOrTransaction, RuntimeContext, LifecycleRuntimeFence } from "../runtime-types.js";
 
@@ -585,6 +586,7 @@ export function createSlackSessionStop(db: Db, options: SlackSessionStopOptions)
           ["queued", "deferred_issue_execution"].includes(wakeup.status) &&
           !issue.executionRunId
         ) {
+          await notifyStrandedAgentStatus(tx, { companyId: current.companyId, agentId: wakeup.agentId });
           await notifyChatInboundWakeNotices(tx, { companyId: current.companyId, issueId: issue.id });
           await tx
             .update(agentWakeupRequests)

server/src/services/deferred-execution-notifications.ts

@@ -24,6 +24,7 @@ export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecu
 }
 
 export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeProcessSessions, scope ? JSON.stringify({ companyId: scope.companyId }) : "*");
   if (scope) {
     await notifyDeferredExecutionWork(tx, scope);
     await notifyNativeStatusWakeWork(tx, scope);

server/src/services/delivery-work-notifications.ts

@@ -2,6 +2,14 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db/work-
 
 /** Topics for existing durable queues; register intent before their writes. */
 export const DELIVERY_QUEUES = {
+  nativeProcessSessions: "native-process-sessions",
+  nativeQuestionCancellations: "native-question-cancellations",
+  activeExecutionRecovery: "active-execution-recovery",
+  settledNativeProcesses: "settled-native-processes",
+  orphanActiveLeases: "orphan-active-leases",
+  legacyShutdownSettlement: "legacy-shutdown-settlement",
+  strandedAgentStatus: "stranded-agent-status",
+  activeRunSilence: "active-run-silence",
   deferredExecutionReceipts: "deferred-execution-receipts",
   taskRetryReceiptSettlement: "task-retry-receipt-settlement",
   nativeStatusWakeIntents: "native-status-wake-intents",

server/src/services/environment-cleanup-notifications.ts

@@ -1,14 +1,38 @@
+import { and, eq, or, sql } from "drizzle-orm";
+import { environmentLeases } from "@paperclipai/db";
+import { notifyLiveResourceRecovery } from "./live-run-recovery-notifications.js";
 import { notifyNativeEnvironmentSettlement, type NativeEnvironmentSettlementScope } from "./native-environment-settlement-notifications.js";
 import type { Db } from "@paperclipai/db";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 
-export async function notifyEnvironmentCleanup(transaction: object, scope?: NativeEnvironmentSettlementScope): Promise<void> {
+export type EnvironmentCleanupWorkScope = NativeEnvironmentSettlementScope & { leaseId?: string };
+
+export async function notifyEnvironmentCleanup(transaction: object, scope?: EnvironmentCleanupWorkScope): Promise<void> {
+  await notifyLiveResourceRecovery(transaction, scope);
+  if (scope && (scope.runId || scope.leaseId) && typeof (transaction as Db).select === "function") {
+    // A stopped/released allocation can still owe shutdown workspace preservation.
+    // Its peer's owning release must reactivate that exact marker as well.
+    const pendingShutdown = sql<boolean>`${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,schema}' = 'paperclip.legacy-shutdown-workspace-settlement.v1'
+      and ${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending'
+      and ${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,runId}' = ${environmentLeases.heartbeatRunId}::text`;
+    const peers = await (transaction as Db).select({ id: environmentLeases.id, companyId: environmentLeases.companyId,
+      status: environmentLeases.status, pendingShutdown }).from(environmentLeases).where(and(
+      or(eq(environmentLeases.status, "active"), pendingShutdown),
+      sql`exists (select 1 from environment_leases changed where changed.company_id = ${scope.companyId}
+        and ${scope.leaseId ? sql`changed.id = ${scope.leaseId}` : sql`changed.heartbeat_run_id = ${scope.runId}`} and changed.provider_lease_id is not null
+        and changed.provider = ${environmentLeases.provider} and changed.provider_lease_id = ${environmentLeases.providerLeaseId})`));
+    for (const peer of peers) {
+      const hint = JSON.stringify({ companyId: peer.companyId, leaseId: peer.id });
+      if (peer.status === "active") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.orphanActiveLeases, hint);
+      if (peer.pendingShutdown) await notifyDeliveryWork(transaction, DELIVERY_QUEUES.legacyShutdownSettlement, hint);
+    }
+  }
   await notifyNativeEnvironmentSettlement(transaction, scope);
   await notifyDeliveryWork(transaction, DELIVERY_QUEUES.environmentCleanup);
   await notifyDeliveryWork(transaction, DELIVERY_QUEUES.historicalCleanup);
 }
 
-export function writeEnvironmentCleanup<T>(db: Db, write: (tx: Db) => PromiseLike<T>, scope: NativeEnvironmentSettlementScope): Promise<T> {
+export function writeEnvironmentCleanup<T>(db: Db, write: (tx: Db) => PromiseLike<T>, scope: EnvironmentCleanupWorkScope): Promise<T> {
   return db.transaction(async tx => {
     await notifyEnvironmentCleanup(tx, scope);
     return write(tx as unknown as Db);

server/src/services/execution-control-notifications.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery, notifyLiveResourceRecovery } from "./live-run-recovery-notifications.js";
 import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { notifyRunDispatch } from "./run-dispatch-notifications.js";
@@ -14,6 +15,8 @@ export async function notifyExecutionRunWork(transaction: object, scope?: Execut
     await notifyRunDispatch(transaction, scope);
     await notifyStatusCardIssueChange(transaction, scope.companyId);
     await notifyDeferredExecutionEligibility(transaction, scope);
+    await notifyLiveRunRecovery(transaction, scope);
+    await notifyLiveResourceRecovery(transaction, scope);
     await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
   }
   for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
@@ -31,10 +34,14 @@ export function writeExecutionControl<T>(
 ): Promise<T> {
   return db.transaction(async transaction => {
     await notifyRunDispatch(transaction, scope);
-    if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
-    if (scope) await notifyAttentionRetentionWork(transaction, scope);
-    if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
-    if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
+    if (scope) {
+      await notifyStatusCardIssueChange(transaction, scope.companyId);
+      await notifyAttentionRetentionWork(transaction, scope);
+      await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId);
+      await notifyDeferredExecutionEligibility(transaction, scope);
+      await notifyLiveRunRecovery(transaction, scope);
+      await notifyLiveResourceRecovery(transaction, scope);
+    }
     await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
     for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
     return write(transaction as unknown as Db);

server/src/services/heartbeat-run-events.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js";
 import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
 import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js";
 import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
@@ -78,10 +79,11 @@ export async function appendHeartbeatRunEvent(
   input: AppendHeartbeatRunEventInput,
 ): Promise<AppendHeartbeatRunEventResult> {
   const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
+  const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType);
   const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
   const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
   const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
-  if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) {
+  if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRecovery && !projectsRetention) {
     const allocated = db.$with("allocated_run_event", {}).as(sql`
       update ${heartbeatRuns}
       set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1,
@@ -121,6 +123,8 @@ export async function appendHeartbeatRunEvent(
       throw new Error("heartbeat_run_event_binding_mismatch");
     }
 
+    if (projectsRecovery) await notifyLiveRunRecovery(tx, input);
+
     if (input.retryExhaustion && !input.nativeSource) {
       // The run lock also serializes concurrent recovery checks across server
       // instances. Reusing the receipt must not allocate a sequence or publish

server/src/services/heartbeat.ts

@@ -1030,6 +1030,7 @@ export function heartbeatService(
   });
 
   const {
+    registerResidualRecoveryWork: registerResidualRecoveryOwner,
     sweepPendingCleanupLeases,
     markNativeOwnershipUnverified,
     prepareHotRestartShutdown,
@@ -4233,6 +4234,13 @@ export function heartbeatService(
     computeTaskDrain,
     applyTaskDrain,
 
+    registerResidualRecoveryWork: (coordinator: Parameters<typeof registerResidualRecoveryOwner>[0], canRun: () => boolean) =>
+      registerResidualRecoveryOwner(coordinator, () => canRun() && readTaskDrain(new Date()) === null,
+        async () => !(await getSchedulingSuppression(true)).suppressed),
+    registerActiveRunSilenceWork: (coordinator: Parameters<typeof recovery.registerActiveRunSilenceWork>[0], canRun: () => boolean) =>
+      recovery.registerActiveRunSilenceWork(coordinator, () => canRun() && readTaskDrain(new Date()) === null,
+        async scope => recovery.scanSilentActiveRuns({ ...scope, issueCreatedAtGte: await getWorktreeExecutionCutoff() }),
+        async () => !(await getSchedulingSuppression(true)).suppressed),
     registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
       registerDeferredReceiptWork(db, coordinator, {
         resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),

server/src/services/heartbeat/agent-status.test.ts

@@ -6,23 +6,29 @@ import {
 } from "@paperclipai/db";
 import { and, eq, sql } from "drizzle-orm";
 import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest";
+import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
 import { createHeartbeatAgentStatus, settledAgentWorkCondition } from "./agent-status.js";
 
-const support = await getEmbeddedPostgresTestSupport();
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_STATUS_TEST_DATABASE_URL");
+const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
 const describePostgres = support.supported ? describe : describe.skip;
 if (!support.supported) console.warn(`Skipping agent status database tests: ${support.reason}`);
 
 describePostgres("heartbeat agent status", () => {
   let db: Db;
-  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
   let service: ReturnType<typeof createHeartbeatAgentStatus>;
   const live = new Set<string>();
   const old = () => new Date(Date.now() - 60 * 60_000);
 
   beforeAll(async () => {
-    database = await startEmbeddedPostgresTestDatabase("paperclip-agent-status-");
-    db = createDb(database.connectionString);
+    if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-agent-status-");
+    db = createDb(external ?? database!.connectionString, { maxConnections: 6 });
+    if (external) {
+      db.$client.options.onnotice = () => {};
+      await db.execute(sql`truncate companies cascade`);
+    }
     service = createHeartbeatAgentStatus(db);
   }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
   beforeEach(async () => {
@@ -38,7 +44,7 @@ describePostgres("heartbeat agent status", () => {
     await db.delete(agents);
     await db.delete(companies);
   });
-  afterAll(async () => { await database?.cleanup(); });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
 
   async function seed(status = "interrupted") {
     const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(), issueId = randomUUID();

server/src/services/heartbeat/agent-status.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js";
 import { notifyAttentionRetentionWork } from "../retention-work-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
 import { agents, heartbeatRuns, type Db } from "@paperclipai/db";
@@ -92,6 +93,7 @@ export function createHeartbeatAgentStatus(db: Db) {
       )).limit(1);
       const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure);
       const now = new Date();
+      await notifyStrandedAgentStatus(tx, { companyId: existing.companyId, agentId });
       await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
       const [updated] = await tx.update(agents).set({
@@ -111,13 +113,15 @@ export function createHeartbeatAgentStatus(db: Db) {
     publishStatus(result.updated, outcome);
   }
 
-  async function reconcileStrandedAgentStatuses(liveRunIds: () => ReadonlySet<string>) {
+  async function reconcileStrandedAgentStatuses(liveRunIds: () => ReadonlySet<string>, scope: { companyId?: string; agentId?: string; onDeferredDeadline?: (at: number) => void } = {}) {
     const now = new Date();
     const cutoff = new Date(now.getTime() - RECONCILIATION_GRACE_MS);
     const candidates = await db.transaction(async tx => {
       await tx.execute(sql`SET LOCAL statement_timeout = '2s'`);
       return tx.select({ id: agents.id, companyId: agents.companyId }).from(agents)
       .where(and(eq(agents.status, "running"), eq(agents.lifecycleState, "ready"),
+        scope.companyId ? eq(agents.companyId, scope.companyId) : undefined,
+        scope.agentId ? eq(agents.id, scope.agentId) : undefined,
         lte(agents.updatedAt, cutoff), settledAgentWorkCondition(liveRunIds(), now)))
       .orderBy(asc(agents.updatedAt), asc(agents.id)).limit(RECONCILIATION_BATCH_SIZE);
     });
@@ -130,7 +134,13 @@ export function createHeartbeatAgentStatus(db: Db) {
         const [locked] = await tx.select({ id: agents.id }).from(agents).where(and(
           eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
         )).for("update", { skipLocked: true });
-        if (!locked) return null;
+        if (!locked) {
+          const [pending] = await tx.select({ id: agents.id }).from(agents).where(and(
+            eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
+          ));
+          if (pending) scope.onDeferredDeadline?.(Date.now() + 30_000);
+          return null;
+        }
         const [current] = await tx.select().from(agents).where(and(
           eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId),
           eq(agents.status, "running"), eq(agents.lifecycleState, "ready"),
@@ -159,6 +169,21 @@ export function createHeartbeatAgentStatus(db: Db) {
       for (const publication of publications) publishActivity(publication);
       publishStatus(updated, "reconciled");
     }
+    if (scope.companyId && scope.agentId && reconciled.length === 0 && scope.onDeferredDeadline) {
+      const rows = await db.execute(sql`select min(deadline) as deadline from (
+        select r.controller_lease_expires_at as deadline from heartbeat_runs r
+          where r.company_id = ${scope.companyId} and r.agent_id = ${scope.agentId}
+        union all
+        select f.lease_expires_at as deadline from native_run_finalizations f
+          inner join heartbeat_runs r on r.id = f.run_id and r.company_id = f.company_id
+          where r.company_id = ${scope.companyId} and r.agent_id = ${scope.agentId}
+      ) authority where deadline > ${now.toISOString()}::timestamptz`);
+      const deadline = rows[0]?.deadline;
+      if (deadline) {
+        const at = deadline instanceof Date ? deadline.getTime() : Date.parse(String(deadline));
+        if (Number.isFinite(at) && at > Date.now()) scope.onDeferredDeadline(at);
+      }
+    }
     return reconciled;
   }
 

server/src/services/heartbeat/live-recovery-work.ts

@@ -0,0 +1,138 @@
+import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js";
+import { settledNativeProcessIdentityCondition } from "../native-local-process-stop.js";
+import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
+import { agents, environmentLeases, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
+import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
+import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
+import type { LiveResourceRecoveryScope, LiveRunRecoveryScope } from "../live-run-recovery-notifications.js";
+import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
+
+const TERMINAL = ["succeeded", "failed", "cancelled", "timed_out", "interrupted"];
+type Scope = LiveRunRecoveryScope & LiveResourceRecoveryScope;
+type Entry = { id: string; companyId: string; runId?: string | null; dueAt: number };
+type CheckScope = { companyId: string; runId?: string; leaseId?: string; agentId?: string;
+  onDeferredDeadline(at: number): void; onError(error: unknown): void };
+type RunKind = "native" | "question" | "active" | "settled";
+
+export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType<typeof createDeliveryWorkCoordinator>, owner: {
+  native(scope: CheckScope): Promise<unknown>;
+  question(scope: CheckScope): Promise<unknown>;
+  active(scope: CheckScope): Promise<unknown>;
+  settled(scope: CheckScope): Promise<unknown>;
+  orphanLease(scope: CheckScope): Promise<unknown>;
+  shutdown(scope: CheckScope): Promise<unknown>;
+  agent(scope: CheckScope): Promise<unknown>;
+  enabled(): Promise<boolean>;
+}, canRun: () => boolean) {
+  function register(queue: DeliveryQueue, kind: RunKind | "orphanLease" | "shutdown" | "agent") {
+    const cursors = new Map<string, string>();
+    let enabled = false;
+    async function load(scope?: Scope) {
+        const key = scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId,
+          leaseId: scope.leaseId, agentId: scope.agentId }) : "*";
+        if (!enabled) { cursors.delete(key); return []; }
+        const cursor = cursors.get(key);
+        let rows: Entry[];
+        if (kind === "agent") {
+          const candidates = await db.select({ id: agents.id, companyId: agents.companyId, updatedAt: agents.updatedAt })
+            .from(agents).where(and(eq(agents.status, "running"), eq(agents.lifecycleState, "ready"),
+              scope ? eq(agents.companyId, scope.companyId) : undefined,
+              scope?.agentId ? eq(agents.id, scope.agentId) : undefined,
+              cursor ? gt(agents.id, cursor) : undefined,
+            )).orderBy(asc(agents.id)).limit(100);
+          rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, dueAt: row.updatedAt.getTime() + 5 * 60_000 }));
+        } else if (kind === "orphanLease" || kind === "shutdown") {
+          const candidates = await db.select({ id: environmentLeases.id, companyId: environmentLeases.companyId,
+            runId: environmentLeases.heartbeatRunId, updatedAt: environmentLeases.updatedAt, metadata: environmentLeases.metadata }).from(environmentLeases)
+            .leftJoin(heartbeatRuns, and(eq(heartbeatRuns.id, environmentLeases.heartbeatRunId),
+              eq(heartbeatRuns.companyId, environmentLeases.companyId)))
+            .where(and(scope ? eq(environmentLeases.companyId, scope.companyId) : undefined,
+              scope?.leaseId ? eq(environmentLeases.id, scope.leaseId) : undefined,
+              scope?.runId ? eq(environmentLeases.heartbeatRunId, scope.runId) : undefined,
+              cursor ? gt(environmentLeases.id, cursor) : undefined,
+              kind === "orphanLease" ? and(eq(environmentLeases.status, "active"),
+                or(isNull(environmentLeases.heartbeatRunId), inArray(heartbeatRuns.status, TERMINAL)))
+                : sql`${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,schema}' = 'paperclip.legacy-shutdown-workspace-settlement.v1'
+                  and ${environmentLeases.metadata}#>>'{legacyShutdownWorkspaceSettlement,state}' = 'pending'`,
+            )).orderBy(asc(environmentLeases.id)).limit(100);
+          rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, runId: row.runId,
+            dueAt: kind === "orphanLease" ? row.updatedAt.getTime() + 5 * 60_000
+              : Date.parse(String((row.metadata?.legacyShutdownWorkspaceSettlement as Record<string, unknown>)?.deadlineAt)) || Date.now(),
+          }));
+        } else {
+          const candidates = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId,
+            nextAttemptAt: nativeRunFinalizations.nextAttemptAt, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt })
+            .from(heartbeatRuns).leftJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
+              eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId)))
+            .where(and(scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
+              scope?.runId ? eq(heartbeatRuns.id, scope.runId) : undefined,
+              cursor ? gt(heartbeatRuns.id, cursor) : undefined,
+              kind === "native" ? and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["running", "failed"]),
+                eq(nativeRunFinalizations.phase, "retryable_failure"), isNull(nativeRunFinalizations.resultId),
+                sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`)
+                : kind === "question" ? and(or(inArray(heartbeatRuns.status, ["queued", "running", "scheduled_retry"]),
+                    pendingFailedNativeRetryCancellationCondition()),
+                  sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is not null`)
+                  : kind === "active" ? and(eq(heartbeatRuns.status, "running"),
+                    sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`)
+                    : and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]),
+                      isNotNull(heartbeatRuns.finishedAt), or(isNotNull(heartbeatRuns.processPid),
+                        isNotNull(heartbeatRuns.processGroupId), isNotNull(heartbeatRuns.processStartedAt)),
+                      settledNativeProcessIdentityCondition(db)),
+            )).orderBy(asc(heartbeatRuns.id)).limit(100);
+          rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, dueAt: kind === "native"
+            ? Math.max(Date.now(), row.nextAttemptAt?.getTime() ?? 0, row.leaseExpiresAt?.getTime() ?? 0) : Date.now() }));
+        }
+        const complete = rows.length < 100;
+        if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
+        return { complete, rows };
+    }
+    return registerScopedDeadlineWork<Entry, Scope>(coordinator, {
+      queue,
+      reconstructionIntervalMs: 30_000,
+      beforePass: async () => { enabled = await owner.enabled(); },
+      decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
+      matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (
+        kind === "agent" ? !scope.agentId || entry.id === scope.agentId
+          : kind === "orphanLease" || kind === "shutdown" ? (!scope.leaseId || entry.id === scope.leaseId) && (!scope.runId || entry.runId === scope.runId)
+            : !scope.runId || entry.id === scope.runId),
+      load,
+      check: async entry => {
+        if (!enabled) return null;
+        const currentScope = { companyId: entry.companyId,
+          ...(kind === "agent" ? { agentId: entry.id } : kind === "orphanLease" || kind === "shutdown"
+            ? { leaseId: entry.id } : { runId: entry.id }) };
+        const loaded = await load(currentScope);
+        const current = (Array.isArray(loaded) ? loaded : loaded.rows).find(row => row.id === entry.id);
+        if (!current) return null;
+        if (current.dueAt > Date.now()) return current;
+        let next: number | null = null;
+        const errors: unknown[] = [];
+        const scope: CheckScope = { companyId: entry.companyId,
+          ...(kind === "agent" ? { agentId: entry.id } : kind === "orphanLease" || kind === "shutdown"
+            ? { leaseId: entry.id } : { runId: entry.id }),
+          onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
+          onError: error => { errors.push(error); },
+        };
+        await owner[kind](scope);
+        if (errors.length) throw new AggregateError(errors, `Recovery obligation ${entry.id} failed`);
+        if (kind === "native") {
+          const refreshed = await load(currentScope);
+          const remaining = (Array.isArray(refreshed) ? refreshed : refreshed.rows).find(row => row.id === entry.id);
+          if (remaining && remaining.dueAt > Date.now()) next = next === null ? remaining.dueAt : Math.min(next, remaining.dueAt);
+        }
+        return next === null ? null : { ...entry, dueAt: next };
+      },
+    }, canRun);
+  }
+  const workers = [register(DELIVERY_QUEUES.nativeProcessSessions, "native"),
+    register(DELIVERY_QUEUES.nativeQuestionCancellations, "question"),
+    register(DELIVERY_QUEUES.activeExecutionRecovery, "active"),
+    register(DELIVERY_QUEUES.settledNativeProcesses, "settled"),
+    register(DELIVERY_QUEUES.orphanActiveLeases, "orphanLease"),
+    register(DELIVERY_QUEUES.legacyShutdownSettlement, "shutdown"),
+    register(DELIVERY_QUEUES.strandedAgentStatus, "agent")];
+  return { ready: Promise.all(workers.map(worker => worker.ready)),
+    wake() { for (const worker of workers) worker.invalidate(); },
+    async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
+}

server/src/services/heartbeat/queue.ts

@@ -1,3 +1,4 @@
+import { notifyExecutionRunWork } from "../execution-control-notifications.js";
 import { PersistedExecutionWaitRefusal, writeDeferredExecutionWait } from "../deferred-execution-wait.js";
 import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
 import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
@@ -872,6 +873,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id });
         await notifyExternalAgentSettlement(tx, { companyId: current.companyId, runId: current.id });
         await notifyCancelledWorkspacePreservation(tx, { companyId: current.companyId, runId: current.id });
+        await notifyExecutionRunWork(tx, { companyId: current.companyId, agentId: current.agentId, runId: current.id });
         await notifyUnsafeWorkspaceExportRunChange(tx, current);
         [terminal] = await tx
           .update(heartbeatRuns)
@@ -998,6 +1000,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
           await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id });
           await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id });
           await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
+          await notifyExecutionRunWork(tx, { companyId: run.companyId, agentId: run.agentId, runId: run.id });
           await notifyUnsafeWorkspaceExportRunChange(tx, run);
           const [row] = await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(),
             errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() })
@@ -1363,7 +1366,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
                   await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id });
                   await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id });
                   await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
-          await notifyUnsafeWorkspaceExportRunChange(tx, run);
+                  await notifyExecutionRunWork(tx, { companyId: run.companyId, agentId: run.agentId, runId: run.id });
+                  await notifyUnsafeWorkspaceExportRunChange(tx, run);
                   const [cancelled] = await tx
                     .update(heartbeatRuns)
                     .set({
@@ -1488,7 +1492,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
                 await notifyInstructionCollection(tx, { companyId: run.companyId, runId: run.id });
                 await notifyExternalAgentSettlement(tx, { companyId: run.companyId, runId: run.id });
                 await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id });
-          await notifyUnsafeWorkspaceExportRunChange(tx, run);
+                await notifyExecutionRunWork(tx, { companyId: run.companyId, agentId: run.agentId, runId: run.id });
+                await notifyUnsafeWorkspaceExportRunChange(tx, run);
                 const [cancelled] = await tx
                   .update(heartbeatRuns)
                   .set({

server/src/services/heartbeat/recovery.ts

@@ -1,3 +1,5 @@
+import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js";
+import { registerLiveRecoveryWork } from "./live-recovery-work.js";
 import { NATIVE_RETAINED_OWNER_WAIT } from "../native-runtime/native-controller-recovery.js";
 import { nativeEnvironmentSettlementDisposition, nativeEnvironmentSettlementWork } from "../native-runtime/native-environment-settlement-work.js";
 import { nativeWorkspaceFinalizationWork } from "../native-runtime/native-workspace-finalization-work.js";
@@ -19,11 +21,11 @@ import {
   hasStopOnlyCleanup,
   settleStopOnlyCleanup,
 } from "../sandbox-stop-and-retain.js";
-import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStop, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
+import { discoverSettledNativeRunProcesses, PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, recordNativeLocalProcessStop, releaseSettledNativeRunProcess, retryPendingSettledNativeRunProcesses } from "../native-local-process-stop.js";
 import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js";
 import {
   legacyControllerBootId,
-  hasLiveLegacyController,
+  liveLegacyControllerExpiresAt,
   revokeExpiredLegacyControllerWithState,
 } from "../legacy-controller-lease.js";
 import { remoteTerminationReceipt } from "../remote-execution-termination.js";
@@ -919,8 +921,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
     };
   }
 
-  async function recoverNativeRunsAfterRestart(now = new Date(), runIds?: string[]) {
-    if (!runIds) {
+  async function recoverNativeRunsAfterRestart(now = new Date(), runIds?: string[], options?: { skipDelegatedWork?: boolean }) {
+    if (!runIds && !options?.skipDelegatedWork) {
       await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
       await reconcileNativeFinalizations(db, undefined, {
         environmentRuntime,
@@ -953,6 +955,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       )
       .where(
         and(
+          runIds ? inArray(nativeRunFinalizations.runId, runIds) : undefined,
           eq(heartbeatRuns.runtimeMode, "native"),
           inArray(heartbeatRuns.status, ["running", "failed"]),
           isNull(nativeRunFinalizations.resultId),
@@ -1500,6 +1503,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
   // data already on the lease row.
   async function sweepOrphanedActiveLeases(opts: {
     backoffMs: number;
+    companyId?: string;
+    leaseId?: string;
+    onDeferredDeadline?: (at: number) => void;
   }): Promise<{ recovered: number }> {
     const cutoff = new Date(Date.now() - opts.backoffMs);
 
@@ -1513,6 +1519,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       .where(
         and(
           eq(environmentLeases.status, "active"),
+          opts.companyId ? eq(environmentLeases.companyId, opts.companyId) : undefined,
+          opts.leaseId ? eq(environmentLeases.id, opts.leaseId) : undefined,
           or(
             isNull(environmentLeases.heartbeatRunId),
             inArray(heartbeatRuns.status, [
@@ -1530,12 +1538,12 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       if (await nativeWorkspaceExportPendingForLease(db, lease)) {
         if (lease.heartbeatRunId && !activeRunExecutions.has(lease.heartbeatRunId)
           && !adapterExecutionControls.has(lease.heartbeatRunId)
-          && await preserveOrphanedNativeWorkspace(db, lease)) recovered += 1;
-        await deferOrphanedActiveLease(lease.id);
+          && await preserveOrphanedNativeWorkspace(db, lease, { onDeferredDeadline: opts.onDeferredDeadline })) recovered += 1;
+        if (!opts.leaseId) await deferOrphanedActiveLease(lease.id);
         continue;
       }
       if (!(await allowLegacyShutdownWorkspaceCleanup(db, lease))) {
-        await deferOrphanedActiveLease(lease.id);
+        if (!opts.leaseId) await deferOrphanedActiveLease(lease.id);
         continue;
       }
       // A provider resource id names one physical sandbox. A different lease
@@ -1562,7 +1570,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
           // Defer this row so the fixed-size page reaches the rows behind
           // it next tick, instead of re-selecting the same guarded rows
           // forever.
-          await deferOrphanedActiveLease(lease.id);
+          if (!opts.leaseId) await deferOrphanedActiveLease(lease.id);
           continue;
         }
       }
@@ -1584,7 +1592,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
             eq(environmentLeases.status, "active"),
           ),
         )
-        .returning({ id: environmentLeases.id }), { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined });
+        .returning({ id: environmentLeases.id }), { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined, leaseId: lease.id });
       if (flipped.length > 0) recovered += 1;
     }
 
@@ -2007,63 +2015,19 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
     void cleanup.finally(() => activeRunExecutionPromises.delete(cleanup));
   }
 
-  let settledProcessesDiscovered = false;
+  type RecoveryScope = {
+    companyId?: string;
+    runId?: string;
+    onDeferredDeadline?: (at: number) => void;
+    onError?: (error: unknown) => void;
+  };
 
-  async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
-    if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
-    const staleThresholdMs = opts?.staleThresholdMs ?? 0;
+  async function resumeRetryableNativeProcessSessions(scope: RecoveryScope = {}, beforeDispatch?: () => Promise<void>) {
     const now = new Date();
-    if (!opts?.skipScheduledRetryCancellation) await scheduledRetryCancellation.sweep();
-    // Recovery never launches a provider or infers stopped ownership from
-    // terminal status. Uncaptured local copies require durable stop evidence.
-    if (!opts?.skipStoppedInstructionCollection) await instructionCopies.recoverStopped().catch(error => {
-      logger.warn({ err: error }, "failed to recover stopped instruction copies");
-    });
-    await instructionCopies.recoverCaptured({ skipCommit: opts?.skipCapturedInstructionCommit, skipDirectoryCleanup: opts?.skipInstructionDirectoryCleanup }).catch(error => {
-      logger.warn({ err: error }, "failed to retry captured instruction revisions");
-    });
-
-    // Complete persisted native results before generic orphan recovery. The
-    // reconciler reads the durable workspace barrier and persisted runtime
-    // mode, never the current feature flag.
-    await reconcileNativeFinalizations(db, undefined, {
-      skipObsoletePolicyReviews: opts?.skipObsoletePolicyReviews,
-      skipAutomaticCompletionReviews: opts?.skipAutomaticCompletionReviews,
-      skipUnsafeWorkspaceExports: opts?.skipUnsafeWorkspaceExports,
-      skipNativePendingEffects: opts?.skipNativePendingEffects,
-      skipNativeResponsePresentation: opts?.skipNativeResponsePresentation,
-      skipNativeReviewResponse: opts?.skipNativeReviewResponse,
-      skipNativeStatusReassessment: opts?.skipNativeStatusReassessment,
-      skipNativeWorkspaceFinalization: opts?.skipNativeWorkspaceFinalization,
-      skipNativeResultCommit: opts?.skipNativeResultCommit,
-      skipNativeRunProjection: opts?.skipNativeRunProjection,
-      skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
-      skipExternalAgentSettlement: opts?.skipExternalAgentSettlement,
-      skipNativeRecoveryNotices: opts?.skipNativeRecoveryNotices,
-      environmentRuntime,
-      pluginWorkerManager: dependencies.pluginWorkerManager,
-      onWorkspaceSettled: input => settleRecoveredNativeWorkspace(input, {
-        skipEnvironmentSettlement: opts?.skipNativeEnvironmentSettlement,
-      }),
-    }).catch((error) => {
-      logger.warn(
-        { err: error },
-        "failed to reconcile persisted native finalizations before orphan reaping",
-      );
-    });
-    if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
-    await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
-    if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
-      logger.warn(
-        { err: error },
-        "failed to dispatch persisted native status wake intents before orphan reaping",
-      );
-    });
-
     // Retryable runs can retain process identifiers from the failed attempt.
     // Only verified local identities may be checked and cleared on this host.
-    // Due remote retries instead re-enter the existing restart-recovery owner,
-    // which verifies the saved checkpoint, controller lease, and remote target.
+    // Due remote retries re-enter the existing restart-recovery owner, which
+    // verifies the saved checkpoint, controller lease, and remote target.
     const retryableNativeProcesses = await db
       .select({ run: heartbeatRuns, nextAttemptAt: nativeRunFinalizations.nextAttemptAt })
       .from(heartbeatRuns)
@@ -2073,10 +2037,13 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       )
       .where(
         and(
+          scope.companyId ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
+          scope.runId ? eq(heartbeatRuns.id, scope.runId) : undefined,
           inArray(heartbeatRuns.status, ["running", "failed"]),
           eq(heartbeatRuns.runtimeMode, "native"),
           eq(nativeRunFinalizations.phase, "retryable_failure"),
           isNull(nativeRunFinalizations.resultId),
+          sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`,
         ),
       );
     const claimableNativeRunIds = new Set<string>();
@@ -2111,6 +2078,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
           processPidAlive,
           processGroupAlive,
         });
+        scope.onDeferredDeadline?.(Date.now() + 30_000);
         continue;
       }
       const cleared = await db.transaction(async tx => {
@@ -2146,21 +2114,11 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       if (cleared) claimableNativeRunIds.add(cleared.id);
     }
 
-    const remoteRetryClaims = remoteRetryRunIds.length
-      ? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds)).claims
-      : [];
+    await beforeDispatch?.();
 
-    const executingRun = (runId: string) => activeRunExecutions.has(runId) || runningProcesses.has(runId);
-    if (!settledProcessesDiscovered) {
-      settledProcessesDiscovered = await discoverSettledNativeRunProcesses(db, { skip: executingRun }).then(() => true, (error) => {
-        logger.warn({ err: error }, "failed to discover settled native process identities");
-        return false;
-      });
-    } else {
-      await retryPendingSettledNativeRunProcesses(db, { skip: executingRun }).catch((error) => {
-        logger.warn({ err: error }, "failed to retry settled native process identity release");
-      });
-    }
+    const remoteRetryClaims = remoteRetryRunIds.length
+      ? (await recoverNativeRunsAfterRestart(now, remoteRetryRunIds, { skipDelegatedWork: true })).claims
+      : [];
 
     // An explicit result-less retryable failure resumes on the original run.
     // The database lease is claimed before dispatch so concurrent service
@@ -2175,6 +2133,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
               runtimeEnv.PAPERCLIP_INSTANCE_ID?.trim() || "paperclip-heartbeat",
             now,
             runIds: [...claimableNativeRunIds],
+            deferIneligible: !scope.runId,
+            onError: scope.onError,
             dispatch: (claim) => {
               const execution = executeRun(claim.runId, {
                 nativeLeaseOwner: claim.leaseOwner,
@@ -2190,6 +2150,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
               );
             },
           }).catch((error) => {
+            scope.onError?.(error);
             logger.warn(
               { err: error },
               "failed to claim persisted native session resumptions",
@@ -2200,6 +2161,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       [...nativeResumeClaims, ...remoteRetryClaims].map((claim) => claim.runId),
     );
 
+    return resumedRunIds;
+  }
+
+  async function recoverQuestionCancellationRequests(scope: RecoveryScope = {}) {
     // A terminal issue transition writes this intent in the same transaction
     // that closes the question's task, even when its card is retained. Consume
     // it before generic orphan recovery so a restart preserves cancellation.
@@ -2211,9 +2176,11 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       .from(heartbeatRuns)
       .where(
         and(
-          inArray(heartbeatRuns.status, [
+          scope.companyId ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
+          scope.runId ? eq(heartbeatRuns.id, scope.runId) : undefined,
+          or(inArray(heartbeatRuns.status, [
             ...CANCELLABLE_HEARTBEAT_RUN_STATUSES,
-          ]),
+          ]), pendingFailedNativeRetryCancellationCondition()),
           sql`${heartbeatRuns.contextSnapshot} -> ${NATIVE_QUESTION_CANCELLATION_CONTEXT_KEY} is not null`,
         ),
       );
@@ -2249,6 +2216,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
           },
         });
       } catch (err) {
+        scope.onError?.(err);
         // Keep the marker intact for the next startup/periodic sweep.
         logger.warn(
           { err, runId: request.id },
@@ -2257,6 +2225,14 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       }
     }
 
+  }
+
+  async function recoverActiveExecutions(scope: RecoveryScope = {}, options: {
+    staleThresholdMs?: number; resumedRunIds?: Set<string>;
+  } = {}) {
+    const now = new Date();
+    const staleThresholdMs = options.staleThresholdMs ?? 5 * 60_000;
+    const resumedRunIds = options.resumedRunIds ?? new Set<string>();
     // Find all runs stuck in "running" state (queued runs are legitimately waiting; resumeQueuedRuns handles them)
     const activeRuns = await db
       .select({
@@ -2278,7 +2254,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
         nativeRunFinalizations,
         eq(nativeRunFinalizations.runId, heartbeatRuns.id),
       )
-      .where(eq(heartbeatRuns.status, "running"));
+      .where(and(eq(heartbeatRuns.status, "running"),
+        scope.companyId ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
+        scope.runId ? eq(heartbeatRuns.id, scope.runId) : undefined));
 
     // Reuse the orphan sweep's active-run inventory. Only an expired, untracked
     // controller creates recovery work; idle sweeps add no checkpoint query.
@@ -2338,6 +2316,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       nativeControllerLeaseExpiresAt,
       nativeWorkspaceFinalizationPending,
     } of activeRuns) {
+      if (parseObject(run.contextSnapshot).nativeQuestionCancellation) continue;
       // Native recovery owns these dispositions. Its writes happened after this
       // inventory was read; do not reap or overwrite them using the old row.
       if (remoteRecoveryRunIds.has(run.id)) continue;
@@ -2369,6 +2348,11 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
         runningProcesses.has(run.id) ||
         activeRunExecutions.has(run.id) ||
         coordinatorOwnedByCurrentController;
+      if (nativeRun && !locallyTracked && nativeControllerBootId && nativeControllerPid
+        && nativeControllerProcessStartedAt && nativeControllerLeaseExpiresAt && nativeControllerLeaseExpiresAt > now) {
+        scope.onDeferredDeadline?.(nativeControllerLeaseExpiresAt.getTime());
+        continue;
+      }
       if (
         nativeRun &&
         ([
@@ -2379,6 +2363,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
         ].includes(nativeRecoveryState ?? "") ||
           nativeCoordinatorPhase === "retryable_failure")
       ) {
+        if (nativeControllerLeaseExpiresAt && nativeControllerLeaseExpiresAt > now)
+          scope.onDeferredDeadline?.(nativeControllerLeaseExpiresAt.getTime());
         continue;
       }
       const observedOwnerUnverified =
@@ -2409,17 +2395,27 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
           processPidAlive: nativeProcessPidAlive,
           processGroupAlive: nativeProcessGroupAlive,
         });
+        if (nativeControllerLeaseExpiresAt && nativeControllerLeaseExpiresAt > now)
+          scope.onDeferredDeadline?.(nativeControllerLeaseExpiresAt.getTime());
         continue;
       }
       if (nativeRun && !locallyTracked && (run.processPid || run.processGroupId) && !nativeLocalProcess) continue;
       if (resumedRunIds.has(run.id)) continue;
-      if (locallyTracked) continue;
-      if (await hasLiveLegacyController(db, run)) continue;
+      if (locallyTracked) {
+        if (!activeRunExecutions.has(run.id) && !runningProcesses.has(run.id) && nativeControllerLeaseExpiresAt)
+          scope.onDeferredDeadline?.(nativeControllerLeaseExpiresAt.getTime());
+        continue;
+      }
+      const legacyLease = await liveLegacyControllerExpiresAt(db, run);
+      if (legacyLease) { scope.onDeferredDeadline?.(legacyLease.getTime()); continue; }
 
       // Apply staleness threshold to avoid false positives
       if (staleThresholdMs > 0) {
         const refTime = run.updatedAt ? new Date(run.updatedAt).getTime() : 0;
-        if (now.getTime() - refTime < staleThresholdMs) continue;
+        if (now.getTime() - refTime < staleThresholdMs) {
+          scope.onDeferredDeadline?.(refTime + staleThresholdMs);
+          continue;
+        }
       }
 
       const currentAdapterTracksLocalChild =
@@ -2444,6 +2440,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
         (processPidAlive || processGroupAlive) &&
         readHotRestartAdoptionMetadata(parseObject(run.resultJson))
       ) {
+        scope.onDeferredDeadline?.(Date.now() + 30_000);
         continue;
       }
       if (processPidAlive || processGroupAlive) {
@@ -2469,6 +2466,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
             });
           }
         }
+        scope.onDeferredDeadline?.(Date.now() + 30_000);
         continue;
       }
 
@@ -2618,6 +2616,80 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
       );
     }
 
+    return reaped;
+  }
+
+  let settledProcessesDiscovered = false;
+
+  async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
+    if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
+    const staleThresholdMs = opts?.staleThresholdMs ?? 0;
+    const now = new Date();
+    if (!opts?.skipScheduledRetryCancellation) await scheduledRetryCancellation.sweep();
+    // Recovery never launches a provider or infers stopped ownership from
+    // terminal status. Uncaptured local copies require durable stop evidence.
+    if (!opts?.skipStoppedInstructionCollection) await instructionCopies.recoverStopped().catch(error => {
+      logger.warn({ err: error }, "failed to recover stopped instruction copies");
+    });
+    await instructionCopies.recoverCaptured({ skipCommit: opts?.skipCapturedInstructionCommit, skipDirectoryCleanup: opts?.skipInstructionDirectoryCleanup }).catch(error => {
+      logger.warn({ err: error }, "failed to retry captured instruction revisions");
+    });
+
+    // Complete persisted native results before generic orphan recovery. The
+    // reconciler reads the durable workspace barrier and persisted runtime
+    // mode, never the current feature flag.
+    await reconcileNativeFinalizations(db, undefined, {
+      skipObsoletePolicyReviews: opts?.skipObsoletePolicyReviews,
+      skipAutomaticCompletionReviews: opts?.skipAutomaticCompletionReviews,
+      skipUnsafeWorkspaceExports: opts?.skipUnsafeWorkspaceExports,
+      skipNativePendingEffects: opts?.skipNativePendingEffects,
+      skipNativeResponsePresentation: opts?.skipNativeResponsePresentation,
+      skipNativeReviewResponse: opts?.skipNativeReviewResponse,
+      skipNativeStatusReassessment: opts?.skipNativeStatusReassessment,
+      skipNativeWorkspaceFinalization: opts?.skipNativeWorkspaceFinalization,
+      skipNativeResultCommit: opts?.skipNativeResultCommit,
+      skipNativeRunProjection: opts?.skipNativeRunProjection,
+      skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
+      skipExternalAgentSettlement: opts?.skipExternalAgentSettlement,
+      skipNativeRecoveryNotices: opts?.skipNativeRecoveryNotices,
+      environmentRuntime,
+      pluginWorkerManager: dependencies.pluginWorkerManager,
+      onWorkspaceSettled: input => settleRecoveredNativeWorkspace(input, {
+        skipEnvironmentSettlement: opts?.skipNativeEnvironmentSettlement,
+      }),
+    }).catch((error) => {
+      logger.warn(
+        { err: error },
+        "failed to reconcile persisted native finalizations before orphan reaping",
+      );
+    });
+    if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
+    await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
+    if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
+      logger.warn(
+        { err: error },
+        "failed to dispatch persisted native status wake intents before orphan reaping",
+      );
+    });
+
+    const resumedRunIds = await resumeRetryableNativeProcessSessions({}, async () => {
+    const executingRun = (runId: string) => activeRunExecutions.has(runId) || runningProcesses.has(runId);
+    if (!settledProcessesDiscovered) {
+      settledProcessesDiscovered = await discoverSettledNativeRunProcesses(db, { skip: executingRun }).then(() => true, (error) => {
+        logger.warn({ err: error }, "failed to discover settled native process identities");
+        return false;
+      });
+    } else {
+      await retryPendingSettledNativeRunProcesses(db, { skip: executingRun }).catch((error) => {
+        logger.warn({ err: error }, "failed to retry settled native process identity release");
+      });
+    }
+
+    });
+
+    await recoverQuestionCancellationRequests();
+    const reaped = await recoverActiveExecutions({}, { staleThresholdMs, resumedRunIds });
+
     // Recover an active lease whose run already ended before the same-tick
     // pending_cleanup sweep, so this tick can stop the recovered sandbox.
     // Isolate the sweep so its failure never hides the reaper result.
@@ -2675,7 +2747,31 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
     return { reaped: reaped.length, runIds: reaped };
   }
 
+  function registerResidualRecoveryWork(coordinator: Parameters<typeof registerLiveRecoveryWork>[1],
+    canRun: () => boolean, enabled: () => Promise<boolean>) {
+    return registerLiveRecoveryWork(db, coordinator, {
+      native: scope => resumeRetryableNativeProcessSessions(scope),
+      question: recoverQuestionCancellationRequests,
+      active: scope => recoverActiveExecutions(scope),
+      settled: async scope => {
+        if (!scope.runId || activeRunExecutions.has(scope.runId) || runningProcesses.has(scope.runId)
+          || adapterExecutionControls.has(scope.runId)) {
+          scope.onDeferredDeadline(Date.now() + 30_000);
+          return;
+        }
+        await releaseSettledNativeRunProcess(db, scope.runId, scope);
+      },
+      orphanLease: scope => sweepOrphanedActiveLeases({ ...scope, backoffMs: 5 * 60_000 }),
+      shutdown: scope => sweepLegacyShutdownWorkspaceSettlements(db, new Date(), scope),
+      agent: scope => createHeartbeatAgentStatus(db).reconcileStrandedAgentStatuses(() => new Set([
+        ...activeRunExecutions, ...runningProcesses.keys(), ...adapterExecutionControls.keys(),
+      ]), scope),
+      enabled,
+    }, canRun);
+  }
+
   return {
+    registerResidualRecoveryWork,
     registerRetainedNativeSessionCleanupWork: retainedNativeCleanup.register,
     registerNativeWorkspaceFinalizationWork: nativeWorkspaceFinalization.register,
     registerNativeEnvironmentSettlementWork: nativeEnvironmentSettlement.register,
@@ -2686,6 +2782,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
     reconcileHotRestartAdoption,
     recoverNativeRunsAfterRestart,
     reapOrphanedRuns,
+    resumeRetryableNativeProcessSessions,
+    recoverQuestionCancellationRequests,
+    recoverActiveExecutions,
     sweepOrphanedActiveLeases,
     drainRunningRunsForShutdown,
   };

server/src/services/heartbeat/run-control.ts

@@ -1249,18 +1249,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
     // under lock, and the final CAS requires its own unchanged acknowledged
     // retry fence. Re-reading only retryable_failure here would strand replay.
     const pendingNativeRetry =
-      run.runtimeMode === "native" && run.status === "failed" && (
+      run.runtimeMode === "native" && run.status === "failed" && Boolean(run.nativeIssueId) && (
         caseHandoffRetryClaimed || Boolean(options.cancellationRequestId) || await db
-          .select({ runId: nativeRunFinalizations.runId })
+          .select({ phase: nativeRunFinalizations.phase, failureCode: nativeRunFinalizations.failureCode })
           .from(nativeRunFinalizations)
           .where(
             and(
               eq(nativeRunFinalizations.runId, run.id),
               eq(nativeRunFinalizations.companyId, run.companyId),
-              eq(nativeRunFinalizations.phase, "retryable_failure"),
+              eq(nativeRunFinalizations.issueId, run.nativeIssueId!),
             ),
           )
-          .then((rows) => rows.length > 0)
+          .then(([coordinator]) => nativeRetryCancellationEligible({ runId: run!.id, companyId: run!.companyId,
+            issueId: run!.nativeIssueId!, resultJson: run!.resultJson }, coordinator))
       );
     if (
       !pendingNativeRetry &&

server/src/services/heartbeat/run-lifecycle.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js";
 import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
 import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
@@ -1419,12 +1420,13 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
   ) {
     if (!wakeupRequestId) return;
     await db.transaction(async tx => {
-      const [current] = await tx.select({ companyId: agentWakeupRequests.companyId, status: agentWakeupRequests.status })
+      const [current] = await tx.select({ companyId: agentWakeupRequests.companyId, agentId: agentWakeupRequests.agentId, status: agentWakeupRequests.status })
         .from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeupRequestId)).for("update");
       if (!current || current.status === "cancelled") return;
       if (current.status !== (patch?.status ?? status) || patch?.payload !== undefined || patch?.runId !== undefined) {
         await notifyExternalConversationSettlement(tx, { companyId: current.companyId });
       }
+      await notifyStrandedAgentStatus(tx, { companyId: current.companyId, agentId: current.agentId });
       await tx.update(agentWakeupRequests)
         .set({ status, ...patch, updatedAt: new Date() })
 

server/src/services/heartbeat/run-output.test.ts

@@ -46,6 +46,7 @@ function createMemoryProvider() {
 // way the run row would carry it into a later attempt.
 function createRunRowDb(row: Partial<Run>) {
   return {
+    async transaction<T>(write: (tx: Db) => Promise<T>): Promise<T> { return write(this as unknown as Db); },
     update: () => ({
       set: (values: Partial<Run>) => ({
         where: async () => {
@@ -215,6 +216,7 @@ describe("object-backed diagnostic recording", () => {
     const row = { ...newRunRow };
     const writes: Array<Partial<Run>> = [];
     const db = {
+      async transaction<T>(write: (tx: Db) => Promise<T>): Promise<T> { return write(this as unknown as Db); },
       update: () => ({
         set: (values: Partial<Run>) => ({
           where: async () => {

server/src/services/heartbeat/run-output.ts

@@ -1,3 +1,4 @@
+import { notifyActiveRunSilence } from "../live-run-recovery-notifications.js";
 import { eq } from "drizzle-orm";
 import { heartbeatRuns, type Db } from "@paperclipai/db";
 import type { AdapterInvocationMeta, AdapterRuntimeEvent } from "../../adapters/index.js";
@@ -94,8 +95,9 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
       pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >=
         ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS;
     if (!shouldFlush) return;
-    progressWrite = db
-      .update(heartbeatRuns)
+    progressWrite = db.transaction(async tx => {
+      await notifyActiveRunSilence(tx, { companyId: run.companyId, runId: run.id });
+      await tx.update(heartbeatRuns)
       .set({
         lastOutputAt: pendingOutputProgress.at,
         lastOutputSeq: pendingOutputProgress.seq,
@@ -103,7 +105,8 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput)
         lastOutputBytes: pendingOutputProgress.bytes,
         updatedAt: new Date(),
       })
-      .where(eq(heartbeatRuns.id, run.id))
+      .where(eq(heartbeatRuns.id, run.id));
+    })
       .then(() => {
         lastOutputFlushAt = pendingOutputProgress.at;
         if (outputProgressState.pending === pendingOutputProgress)

server/src/services/instance-settings.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery, notifyLiveResourceRecovery } from "./live-run-recovery-notifications.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
@@ -648,6 +649,8 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
         if (patch.enableWorktreeRunExecution !== undefined) {
           await notifyRunDispatch(tx);
           await notifyDeferredExecutionEligibility(tx);
+          await notifyLiveRunRecovery(tx);
+          await notifyLiveResourceRecovery(tx);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyExternalAgentSettlement(tx);
           await notifyInstructionCommit(tx);

server/src/services/issue-tree-control.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
 import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
@@ -1226,6 +1227,11 @@ export function issueTreeControlService(db: Db) {
     if (issueIds.length === 0) return [];
     const now = new Date();
     return db.transaction(async tx => {
+      const affected = await tx.select({ agentId: agentWakeupRequests.agentId }).from(agentWakeupRequests).where(and(
+        eq(agentWakeupRequests.companyId, companyId), inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]),
+        isNull(agentWakeupRequests.runId), inArray(sql<string | null>`${agentWakeupRequests.payload} ->> 'issueId'`, issueIds),
+      )).for("update");
+      for (const agentId of new Set(affected.map(wake => wake.agentId))) await notifyStrandedAgentStatus(tx, { companyId, agentId });
       await notifyChatInboundWakeNotices(tx, { companyId });
       return tx
         .update(agentWakeupRequests)

server/src/services/issues.ts

@@ -1,3 +1,5 @@
+import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js";
+import { finishedIdleWake } from "./idle-work-proof.js";
 import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { notifyRunDispatch } from "./run-dispatch-notifications.js";
@@ -11524,6 +11526,15 @@ export function issueService(db: Db) {
           // Invalidate human direction at the common assignment boundary, including
           // plugin/service writes that do not go through HTTP run cancellation.
           // Keep the requester attribution for audit; cancellation revokes its use.
+          const affectedWakes = await tx.select({ agentId: agentWakeupRequests.agentId }).from(agentWakeupRequests).where(and(
+            eq(agentWakeupRequests.companyId, receiptExisting.companyId), eq(agentWakeupRequests.requestedByActorType, "user"),
+            sql`(${sql.raw(finishedIdleWake("agent_wakeup_requests"))}) IS NOT TRUE`,
+            sql`coalesce(${agentWakeupRequests.payload}->>'issueId', ${agentWakeupRequests.payload}->>'taskId',
+              ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'issueId',
+              ${agentWakeupRequests.payload}->'_paperclipWakeContext'->>'taskId') = ${id}`,
+          )).for("update");
+          for (const agentId of new Set<string>(affectedWakes.map((wake: { agentId: string }) => wake.agentId)))
+            await notifyStrandedAgentStatus(tx, { companyId: receiptExisting.companyId, agentId });
           await notifyChatInboundWakeNotices(tx, { companyId: receiptExisting.companyId });
           await tx.update(agentWakeupRequests).set({ status: "cancelled", finishedAt: new Date(), updatedAt: new Date() })
             .where(and(eq(agentWakeupRequests.companyId, receiptExisting.companyId),
@@ -11560,6 +11571,7 @@ export function issueService(db: Db) {
           || blockedByIssueIds !== undefined) {
           await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
           await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
+          await notifyActiveRunSilence(tx, { companyId: receiptExisting.companyId, issueId: id });
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyRoutineExecutionStatus(tx, id);
         }
@@ -12042,7 +12054,10 @@ export function issueService(db: Db) {
           await notifyDecisionTargetIssueRemoval(tx, id);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
           await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
-          if (owner) await notifyDeferredExecutionEligibility(tx, owner);
+          if (owner) {
+            await notifyDeferredExecutionEligibility(tx, owner);
+            await notifyActiveRunSilence(tx, { companyId: owner.companyId, issueId: id });
+          }
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyRoutineExecutionStatus(tx, id);
           removedIssue = await tx

server/src/services/legacy-controller-lease.test.ts

@@ -1,8 +1,10 @@
 import { randomUUID } from "node:crypto";
 import { eq, sql } from "drizzle-orm";
 import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-import { agents, closeLivenessDb, companies, createDb, heartbeatRuns, livenessDb } from "@paperclipai/db";
+import { agents, closeLivenessDb, companies, createDb, heartbeatRuns, livenessDb, subscribeDatabaseWork } from "@paperclipai/db";
 import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
+import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js";
+import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
 import { heartbeatService } from "./heartbeat.js";
 import { hasLiveLegacyController, LEGACY_CONTROLLER_LEASE_MS, LEGACY_CONTROLLER_RENEW_MS, legacyControllerBootId, legacyControllerClaim, LegacyControllerLeaseLostError,
   renewLegacyControllerLease, revokeExpiredLegacyController, revokeExpiredLegacyControllerWithState,
@@ -16,15 +18,16 @@ function fakeRenewalDb(returning: () => Promise<unknown>) {
   return fake as unknown as WatchArgs[0];
 }
 
-const support = await getEmbeddedPostgresTestSupport();
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_LEGACY_CONTROLLER_TEST_DATABASE_URL");
+const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
 (support.supported ? describe : describe.skip)("durable legacy controller ownership", () => {
   let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
   let db: ReturnType<typeof createDb>;
   beforeAll(async () => {
-    database = await startEmbeddedPostgresTestDatabase("legacy-controller-");
-    db = createDb(database.connectionString);
+    if (!external) database = await startEmbeddedPostgresTestDatabase("legacy-controller-");
+    db = createDb(external ?? database.connectionString, { maxConnections: 6 });
   }, 30000);
-  afterAll(async () => { await database?.cleanup(); });
+  afterAll(async () => { await closeLivenessDb(db, 1); await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
   async function seed() {
     const companyId = randomUUID(), agentId = randomUUID();
     await db.insert(companies).values({ id: companyId, name: "Controller test", issuePrefix: `C${companyId.slice(0, 7)}` });
@@ -68,6 +71,26 @@ const support = await getEmbeddedPostgresTestSupport();
     expect(saved.executionStage).toBe("dispatching");
     expect(await revokeExpiredLegacyController(db, run)).toBe(false);
   });
+  it("renews through one private transaction with only the active execution topic", async () => {
+    const run = await seed(), settled: string[] = [], queries: string[] = [];
+    const queues = [DELIVERY_QUEUES.nativeProcessSessions, DELIVERY_QUEUES.nativeQuestionCancellations,
+      DELIVERY_QUEUES.activeExecutionRecovery, DELIVERY_QUEUES.settledNativeProcesses,
+      DELIVERY_QUEUES.strandedAgentStatus, DELIVERY_QUEUES.activeRunSilence];
+    const stops = queues.map(queue => subscribeDatabaseWork(db, queue, event => { if (event === "settled") settled.push(queue); }));
+    const pool = livenessDb(db), prior = pool.$client.options.debug;
+    await pool.execute(sql`select 1`);
+    pool.$client.options.debug = (_connection, query) => { queries.push(query); };
+    const watch = watchLegacyControllerLease(db, run, new AbortController());
+    try {
+      await watch.assertOwned();
+      expect(settled).toEqual([DELIVERY_QUEUES.activeExecutionRecovery]);
+      expect(queries.filter(query => /savepoint/i.test(query))).toEqual([]);
+      expect(queries.filter(query => /pg_current_xact_id/.test(query))).toHaveLength(1);
+      expect(queries.filter(query => /set_config.*statement_timeout/.test(query))).toHaveLength(1);
+      expect(queries.filter(query => /^update /i.test(query))).toHaveLength(1);
+      expect(queries).toHaveLength(5);
+    } finally { watch.stop(); pool.$client.options.debug = prior; stops.forEach(stop => stop()); }
+  });
   it("an expired controller cannot renew or dispatch even before a reaper claims it", async () => {
     const run = await seed();
     await expire(run.id);
@@ -139,7 +162,7 @@ const support = await getEmbeddedPostgresTestSupport();
   it("renews without queueing behind a slow statement on a saturated shared pool", async () => {
     const run = await seed();
     // postgres.js pipelines onto a busy connection when none is idle.
-    const shared = createDb(database.connectionString, { maxConnections: 1 });
+    const shared = createDb(external ?? database.connectionString, { maxConnections: 1 });
     const slow = shared.execute(sql`select pg_sleep(5)`).then(() => undefined); // drizzle sends on then()
     const controller = new AbortController();
     const watch = watchLegacyControllerLease(shared, run, controller);
@@ -155,7 +178,7 @@ const support = await getEmbeddedPostgresTestSupport();
 
   it("renews on one private pool per client that shutdown can end", async () => {
     const run = await seed();
-    const owner = createDb(database.connectionString);
+    const owner = createDb(external ?? database.connectionString, { maxConnections: 6 });
     const pool = livenessDb(owner);
     expect(pool).not.toBe(owner);
     expect(livenessDb(owner)).toBe(pool);

server/src/services/legacy-controller-lease.ts

@@ -1,3 +1,4 @@
+import { notifyActiveExecutionRecovery, notifyLiveRunRecovery } from "./live-run-recovery-notifications.js";
 import { randomUUID } from "node:crypto";
 import { and, eq, gt, lte, sql } from "drizzle-orm";
 import { heartbeatRuns, livenessDb, type Db } from "@paperclipai/db";
@@ -35,12 +36,13 @@ export function legacyControllerClaim(runtimeMode: string) {
   };
 }
 
-export async function renewLegacyControllerLease(
-  db: Db,
+async function renewLegacyControllerLeaseInTransaction(
+  tx: Db,
   run: Pick<Run, "id" | "companyId" | "controllerBootId">,
   stage?: "dispatching",
 ): Promise<boolean> {
-  const [renewed] = await db.update(heartbeatRuns).set({
+  await notifyActiveExecutionRecovery(tx, { companyId: run.companyId, runId: run.id });
+  const [renewed] = await tx.update(heartbeatRuns).set({
     controllerLeaseExpiresAt: sql`clock_timestamp() + interval '60 seconds'`,
     ...(stage ? { executionStage: stage } : {}),
   }).where(and(
@@ -52,6 +54,14 @@ export async function renewLegacyControllerLease(
   return Boolean(renewed);
 }
 
+export async function renewLegacyControllerLease(
+  db: Db,
+  run: Pick<Run, "id" | "companyId" | "controllerBootId">,
+  stage?: "dispatching",
+): Promise<boolean> {
+  return db.transaction(tx => renewLegacyControllerLeaseInTransaction(tx as unknown as Db, run, stage));
+}
+
 /** One attempt may wait at most this long; a timed-out attempt is retried
  * until the lease deadline instead of holding the renewal for the whole lease. */
 export const LEGACY_CONTROLLER_RENEWAL_STATEMENT_TIMEOUT_MS = 10_000;
@@ -73,7 +83,7 @@ async function renewOnLivenessPool(
 ): Promise<boolean> {
   return livenessDb(db).transaction(async (tx) => {
     await tx.execute(sql`select set_config('statement_timeout', ${`${statementTimeoutMs}ms`}, true)`);
-    return renewLegacyControllerLease(tx as unknown as Db, run, stage);
+    return renewLegacyControllerLeaseInTransaction(tx as unknown as Db, run, stage);
   });
 }
 
@@ -105,19 +115,23 @@ export async function revokeExpiredLegacyControllerWithState(
   if (run.runtimeMode === "native" || !run.controllerBootId) {
     return { executionStage: run.executionStage, processPid: run.processPid, processGroupId: run.processGroupId };
   }
-  const [revoked] = await db.update(heartbeatRuns).set({
+  const controllerBootId = run.controllerBootId;
+  const [revoked] = await db.transaction(async tx => {
+    await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id });
+    return tx.update(heartbeatRuns).set({
     controllerBootId: randomUUID(),
     controllerLeaseExpiresAt: sql`clock_timestamp() + interval '60 seconds'`,
   }).where(and(
     eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
     eq(heartbeatRuns.status, "running"),
-    eq(heartbeatRuns.controllerBootId, run.controllerBootId),
+    eq(heartbeatRuns.controllerBootId, controllerBootId),
     lte(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
   )).returning({
     executionStage: heartbeatRuns.executionStage,
     processPid: heartbeatRuns.processPid,
     processGroupId: heartbeatRuns.processGroupId,
   });
+  });
   return revoked ?? null;
 }
 

server/src/services/legacy-shutdown-workspace-settlement.ts

@@ -1,3 +1,4 @@
+import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js";
 import { and, eq, gt, inArray, ne, or, sql } from "drizzle-orm";
 import { environmentLeases, heartbeatRuns, issues, type Db } from "@paperclipai/db";
 import { adapterExecutionControls } from "./adapter-execution-control.js";
@@ -33,6 +34,7 @@ export function maintainLegacyShutdownWorkspaceSettlement(db: Db, run: Run) {
     pending = db.transaction(async tx => {
       await tx.execute(sql`SET LOCAL lock_timeout = '1s'`);
       await tx.execute(sql`SET LOCAL statement_timeout = '5s'`);
+      await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
       await tx.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata},
         '{legacyShutdownWorkspaceSettlement,deadlineAt}', ${JSON.stringify(new Date(Date.now() + SETTLEMENT_LEASE_MS).toISOString())}::jsonb)` })
         .where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
@@ -73,39 +75,47 @@ export async function legacyShutdownWorkspaceResourceProtected(db: Db,
 }
 
 /** Released stop receipts still need reconciliation if export never settled. */
-export async function sweepLegacyShutdownWorkspaceSettlements(db: Db, now = new Date()) {
+export async function sweepLegacyShutdownWorkspaceSettlements(db: Db, now = new Date(), scope: { companyId?: string; leaseId?: string; onDeferredDeadline?: (at: number) => void } = {}) {
   const rows = await db.select().from(environmentLeases).where(and(
+    scope.companyId ? eq(environmentLeases.companyId, scope.companyId) : undefined,
+    scope.leaseId ? eq(environmentLeases.id, scope.leaseId) : undefined,
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`,
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`,
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt' <= ${now.toISOString()}`,
   )).orderBy(sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'deadlineAt'`).limit(50);
-  for (const lease of rows) await allowLegacyShutdownWorkspaceCleanup(db, lease, now);
+  for (const lease of rows) await allowLegacyShutdownWorkspaceCleanup(db, lease, now, { onDeferredDeadline: scope.onDeferredDeadline });
 }
 
 /** Commit the fence while the run is still live, before exposing interruption. */
 export async function beginLegacyShutdownWorkspaceSettlement(db: Db, run: Run, deadlineAt: Date) {
-  await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) ||
+  await db.transaction(async tx => {
+    await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
+    await tx.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) ||
     ${JSON.stringify({ legacyShutdownWorkspaceSettlement: { schema: SCHEMA, state: "pending", runId: run.id,
       controllerBootId: legacyControllerBootId, deadlineAt: deadlineAt.toISOString() } })}::jsonb`,
   }).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
     eq(environmentLeases.status, "active"), ne(environmentLeases.provider, "local"),
     or(sql`${environmentLeases.metadata}->>'driver' = 'sandbox'`, sql`${environmentLeases.metadata}->>'sandboxProviderPlugin' = 'true'`)));
+  });
 }
 
 /** The owner joins settlement, or records exact retention before timeout cleanup. */
 export async function finishLegacyShutdownWorkspaceSettlement(db: Db, run: Run, state: "settled" | "expired" = "settled") {
-  await db.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata},
+  await db.transaction(async tx => {
+    await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
+    await tx.update(environmentLeases).set({ metadata: sql`jsonb_set(${environmentLeases.metadata},
     '{legacyShutdownWorkspaceSettlement,state}', ${JSON.stringify(state)}::jsonb)` }).where(and(
     eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'schema' = ${SCHEMA}`,
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'controllerBootId' = ${legacyControllerBootId}`,
     sql`${environmentLeases.metadata}->'legacyShutdownWorkspaceSettlement'->>'state' = 'pending'`,
   ));
+  });
 }
 
 /** A restart or another cleanup worker must never infer saved files from timeout. */
 export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease, now = new Date(),
-  options: { ownerStopOnly?: boolean } = {}) {
+  options: { ownerStopOnly?: boolean; onDeferredDeadline?: (at: number) => void } = {}) {
   if (!pendingLegacyShutdownWorkspaceSettlement(lease)) return true;
   return db.transaction(async tx => {
     const [hint] = await tx.select().from(environmentLeases).where(and(
@@ -154,12 +164,17 @@ export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease,
       if (current.status === "pending_cleanup" && !readStopOnlyCleanup(current)) {
         const expiresAt = current.metadata?.pendingCleanupLeaseExpiresAtMs;
         if (current.metadata?.pendingCleanupInFlight === true
-          && (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt > now.getTime())) return false;
+          && (typeof expiresAt !== "number" || !Number.isFinite(expiresAt) || expiresAt > now.getTime())) {
+          if (typeof expiresAt === "number" && Number.isFinite(expiresAt)) options.onDeferredDeadline?.(expiresAt);
+          return false;
+        }
         // A failed ordinary release can race the shutdown marker. Re-enter the
         // recorder's active-source path within this claim transaction so it
         // replaces destructive retry with an exact stop-only recovery intent.
+        await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
         await tx.update(environmentLeases).set({ status: "active" }).where(eq(environmentLeases.id, current.id));
       } else {
+        await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
         await tx.update(environmentLeases).set({ leasePolicy: "retain_on_failure",
           metadata: sql`${environmentLeases.metadata} || ${JSON.stringify({ workspaceRestoreRecovery: {
             schema: LEGACY_WORKSPACE_RECOVERY_SCHEMA, runId: run.id,
@@ -167,6 +182,7 @@ export async function allowLegacyShutdownWorkspaceCleanup(db: Db, lease: Lease,
         }).where(eq(environmentLeases.id, current.id));
       }
     }
+    if (current.status === "active") await notifyEnvironmentCleanup(tx, { companyId: run.companyId, runId: run.id });
     const { terminalizeLegacyExecution } = await import("./legacy-execution-recovery.js");
     const preserved = await terminalizeLegacyExecution({ db: tx as unknown as Db, run, status: run.status,
       recordRestoreFailureOnly: true, patch: { resultJson: { workspaceRestoreFailure: "restore_failed" } } });

server/src/services/live-run-recovery-notifications.ts

@@ -0,0 +1,40 @@
+import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
+
+export type LiveRunRecoveryScope = { companyId: string; runId?: string; issueId?: string; agentId?: string };
+
+export async function notifyActiveRunSilence(tx: object, scope?: LiveRunRecoveryScope) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.activeRunSilence, scope
+    ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId }) : "*");
+}
+
+export async function notifyActiveExecutionRecovery(tx: object, scope: { companyId: string; runId: string }) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.activeExecutionRecovery,
+    JSON.stringify({ companyId: scope.companyId, runId: scope.runId }));
+}
+
+export async function notifyLiveRunRecovery(tx: object, scope?: LiveRunRecoveryScope) {
+  const hint = scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId }) : "*";
+  for (const queue of [DELIVERY_QUEUES.nativeProcessSessions, DELIVERY_QUEUES.nativeQuestionCancellations,
+    DELIVERY_QUEUES.activeExecutionRecovery, DELIVERY_QUEUES.settledNativeProcesses])
+    await notifyDeliveryWork(tx, queue, hint);
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.strandedAgentStatus, scope
+    ? JSON.stringify({ companyId: scope.companyId, agentId: scope.agentId }) : "*");
+  await notifyActiveRunSilence(tx, scope);
+}
+
+export type LiveResourceRecoveryScope = { companyId: string; runId?: string; leaseId?: string };
+
+export async function notifyLiveResourceRecovery(tx: object, scope?: LiveResourceRecoveryScope) {
+  const hint = scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, leaseId: scope.leaseId }) : "*";
+  for (const queue of [DELIVERY_QUEUES.orphanActiveLeases, DELIVERY_QUEUES.legacyShutdownSettlement])
+    await notifyDeliveryWork(tx, queue, hint);
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.settledNativeProcesses, scope
+    ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId }) : "*");
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.strandedAgentStatus, scope
+    ? JSON.stringify({ companyId: scope.companyId }) : "*");
+}
+
+export async function notifyStrandedAgentStatus(tx: object, scope: { companyId: string; agentId: string }) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.strandedAgentStatus,
+    JSON.stringify({ companyId: scope.companyId, agentId: scope.agentId }));
+}

server/src/services/native-finalization-notifications.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js";
 import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
 import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
 import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
@@ -17,6 +18,7 @@ export type NativeFinalizationWorkScope = { companyId: string; runId?: string; i
 export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) {
   await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId });
   if (scope.issueId || scope.runId) {
+    await notifyLiveRunRecovery(transaction, scope);
     await notifyNativeEnvironmentSettlement(transaction, scope);
     await notifyExternalAgentSettlement(transaction, scope);
     await notifyExternalConversationSettlement(transaction, scope);

server/src/services/native-local-process-stop.ts

@@ -1,4 +1,5 @@
-import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
+import { notifyLiveRunRecovery } from "./live-run-recovery-notifications.js";
+import { and, asc, desc, eq, exists, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
 import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db";
 import { notifyRetainedNativeSessionCleanup } from "./retained-native-session-cleanup-notifications.js";
 import { appendHeartbeatRunEvent } from "./heartbeat-run-events.js";
@@ -145,8 +146,24 @@ export async function readNativeLocalProcessStop(db: Db, companyId: string, runI
 
 type SettledProcessRelease = "released" | "pending" | "skipped";
 
-export async function releaseSettledNativeRunProcess(db: Db, runId: string): Promise<SettledProcessRelease> {
-  return db.transaction(async transaction => {
+export function settledNativeProcessIdentityCondition(db: Db) {
+  const identity = db.select({ payload: heartbeatRunEvents.payload }).from(heartbeatRunEvents).where(and(
+    eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), eq(heartbeatRunEvents.runId, heartbeatRuns.id),
+    eq(heartbeatRunEvents.eventType, PROCESS_IDENTITY_RECORDED), isNull(heartbeatRunEvents.sourceEventId),
+  )).orderBy(desc(heartbeatRunEvents.seq)).limit(1).as("settled_process_identity");
+  return or(isNotNull(heartbeatRuns.processStartedAt), exists(db.select({ id: environmentLeases.id })
+    .from(environmentLeases).where(and(eq(environmentLeases.companyId, heartbeatRuns.companyId),
+      eq(environmentLeases.heartbeatRunId, heartbeatRuns.id), sql`${environmentLeases.provider} <> 'local'`))),
+    and(gt(heartbeatRuns.processPid, 1), eq(heartbeatRuns.processPid, heartbeatRuns.processGroupId),
+      exists(db.select({ payload: identity.payload }).from(identity).where(and(
+        sql`${identity.payload}->>'targetKind' = 'local'`, sql`${identity.payload}->>'hostname' = ${os.hostname()}`,
+        sql`${identity.payload}->>'processPid' = ${heartbeatRuns.processPid}::text`,
+        sql`${identity.payload}->>'processGroupId' = ${heartbeatRuns.processGroupId}::text`,
+      )))));
+}
+
+export async function releaseSettledNativeRunProcess(db: Db, runId: string, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise<SettledProcessRelease> {
+  const outcome = await db.transaction(async transaction => {
     const tx = transaction as unknown as Db;
     const [run] = await tx.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId,
       runtimeMode: heartbeatRuns.runtimeMode, status: heartbeatRuns.status, finishedAt: heartbeatRuns.finishedAt,
@@ -172,15 +189,23 @@ export async function releaseSettledNativeRunProcess(db: Db, runId: string): Pro
         eq(heartbeatRunEvents.eventType, PROCESS_IDENTITY_RECORDED), isNull(heartbeatRunEvents.sourceEventId),
       )).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
       if (identity?.payload?.targetKind !== "local" || identity.payload.hostname !== os.hostname() ||
-          identity.payload.processPid !== run.processPid || identity.payload.processGroupId !== run.processGroupId) return "skipped";
-      if (!(await recordNativeLocalProcessStop(tx, run))) return "pending";
+          identity.payload.processPid !== run.processPid || identity.payload.processGroupId !== run.processGroupId ||
+          (run.processStartedAt ? identity.payload.processStartedAt !== run.processStartedAt.toISOString()
+            : !Number.isSafeInteger(run.processPid) || run.processPid! <= 1 || run.processGroupId !== run.processPid)) return "skipped";
+      if (!(await recordNativeLocalProcessStop(tx, run))) {
+        options.onDeferredDeadline?.(Date.now() + 30_000);
+        return "pending";
+      }
     }
+    await notifyLiveRunRecovery(tx, { companyId: run.companyId, runId: run.id, agentId: run.agentId });
     await notifyRetainedNativeSessionCleanup(tx, { companyId: run.companyId, runId: run.id });
     await tx.update(heartbeatRuns).set({
       processPid: null, processGroupId: null, processStartedAt: null, updatedAt: new Date(),
     }).where(eq(heartbeatRuns.id, run.id));
-    return "released";
+    return "released" as const;
   });
+  if (outcome !== "pending") pendingSettledProcessReleases.delete(runId);
+  return outcome;
 }
 
 const MAX_SETTLED_PROCESS_RELEASE_ATTEMPTS = 20;

server/src/services/native-runtime/native-cancellation-request.ts

@@ -41,6 +41,32 @@ export function nativeRetryCancellationEligible(input: {
       && !!native.acknowledgementAuditId && native.acknowledgementAuditId !== native.intentAuditId));
 }
 
+export function pendingFailedNativeRetryCancellationCondition() {
+  const native = sql`${heartbeatRuns.resultJson}->'nativeCancellation'`;
+  const continuingStop = sql`${nativeRunFinalizations.phase} = 'terminal_failure'
+    and ${nativeRunFinalizations.failureCode} = 'native_retry_cancelled'
+    and ${native}->>'schema' = 'paperclip.native-cancellation.v1'
+    and jsonb_typeof(${native}->'intentId') = 'string' and length(${native}->>'intentId') > 0
+    and ${native}->>'runId' = ${heartbeatRuns.id}::text
+    and ${native}->>'companyId' = ${heartbeatRuns.companyId}::text
+    and ${native}->>'issueId' = ${heartbeatRuns.nativeIssueId}::text
+    and ${native}->>'scope' = 'run'
+    and ${native}->>'dispatchState' in ('pending', 'acknowledged')
+    and jsonb_typeof(${native}->'dispatched') = 'boolean'
+    and jsonb_typeof(${native}->'intentAuditId') = 'string' and length(${native}->>'intentAuditId') > 0
+    and (${native}->>'dispatchState' = 'pending' or (
+      jsonb_typeof(${native}->'acknowledgementAuditId') = 'string'
+      and length(${native}->>'acknowledgementAuditId') > 0
+      and ${native}->>'acknowledgementAuditId' <> ${native}->>'intentAuditId'))`;
+  return sql`${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed'
+    and exists (select 1 from ${nativeRunFinalizations}
+      where ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}
+        and ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId}
+        and ${nativeRunFinalizations.issueId} = ${heartbeatRuns.nativeIssueId}
+        and (${nativeRunFinalizations.phase} = 'retryable_failure' or (${continuingStop}))
+        and ${nativeRunFinalizations.resultId} is null)`;
+}
+
 /** Final failed -> cancelled CAS: a later board/recovery outcome must win.
  * The row-locking subquery prevents a coordinator-only writer from changing
  * the retry disposition between this check and the status update. */

server/src/services/native-runtime/native-execution-lease.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
 import type { Db } from "@paperclipai/db";
 import { heartbeatRuns, livenessDb, nativeRunFinalizations } from "@paperclipai/db";
 import { and, eq, gt, sql } from "drizzle-orm";
@@ -74,6 +75,7 @@ async function renewNativeSessionExecutionLeaseDeadline(
     // but conservatively include UPDATE/commit response time without assuming
     // that this server's wall clock agrees with the database's clock.
     confirmedDeadline = performance.now() + leaseTtlMs;
+    await notifyLiveRunRecovery(tx, { companyId: input.companyId, runId: input.runId });
     return tx.update(nativeRunFinalizations)
       .set({
         leaseExpiresAt: sql`clock_timestamp() + (${leaseTtlMs} * interval '1 millisecond')`,

server/src/services/native-runtime/native-failed-workspace-recovery.ts

@@ -99,7 +99,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick<typeo
 }
 
 export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof environmentLeases.$inferSelect,
-  "companyId" | "heartbeatRunId">): Promise<boolean> {
+  "companyId" | "heartbeatRunId">, options: { onDeferredDeadline?: (at: number) => void } = {}): Promise<boolean> {
   if (!lease.heartbeatRunId) return false;
   const [run] = await db.select().from(heartbeatRuns).where(and(
     eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId),
@@ -122,6 +122,7 @@ export async function preserveOrphanedNativeWorkspace(db: Db, lease: Pick<typeof
     if (error instanceof NativeWorkspaceFinalizationBusyError) return false;
     const code = retentionContentionCode(error);
     if (!code) throw error;
+    options.onDeferredDeadline?.(Date.now() + 30_000);
     logger.warn({ runId: run.id, code }, "Native workspace retention is waiting for its current owner");
     return false;
   }

server/src/services/native-runtime/native-finalization-reconciler.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
 import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
 import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
 import { pendingNativeRunProjectionCondition } from "./native-run-projection.js";
@@ -80,6 +81,8 @@ export async function dispatchNativeSessionResumptions(input: {
   now?: Date;
   limit?: number;
   dispatch: (claim: NativeSessionResumeClaim) => void;
+  deferIneligible?: boolean;
+  onError?: (error: unknown) => void;
 }) {
   const claims = await claimNativeSessionResumptions(input);
   for (const claim of claims) input.dispatch(claim);
@@ -98,6 +101,8 @@ export async function claimNativeSessionResumptions(input: {
   runIds?: string[];
   now?: Date;
   limit?: number;
+  deferIneligible?: boolean;
+  onError?: (error: unknown) => void;
 }): Promise<NativeSessionResumeClaim[]> {
   const now = input.now ?? new Date();
   const candidates = await input.db
@@ -136,6 +141,7 @@ export async function claimNativeSessionResumptions(input: {
   for (const candidate of candidates) {
     const leaseOwner = `${input.runnerInstanceId}:resume:${randomUUID()}`;
     let terminalRunToEmit: typeof heartbeatRuns.$inferSelect | null = null;
+    let claimFailed = false;
     const claimed = await input.db.transaction(async (tx) => {
       await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
       const [issue] = await tx.select().from(issues).where(and(
@@ -245,6 +251,7 @@ export async function claimNativeSessionResumptions(input: {
       ) {
         const failureCode = originalFailureCode;
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, row.run.companyId);
+        await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
         await tx.update(nativeRunFinalizations).set({
           phase: "terminal_failure",
           leaseOwner: null,
@@ -313,6 +320,7 @@ export async function claimNativeSessionResumptions(input: {
       }
 
       const leaseExpiresAt = new Date(now.getTime() + 20 * 60_000);
+      await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
       const updated = await tx.update(nativeRunFinalizations).set({
         phase: "observed",
         leaseOwner,
@@ -350,6 +358,8 @@ export async function claimNativeSessionResumptions(input: {
       });
       return true;
     }).catch((error: unknown) => {
+      claimFailed = true;
+      input.onError?.(error);
       terminalRunToEmit = null;
       logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
         "failed to claim native session recovery candidate");
@@ -362,8 +372,9 @@ export async function claimNativeSessionResumptions(input: {
       void reportRunFailure(input.db, terminalRunToEmit);
     }
     if (claimed) claims.push({ runId: candidate.runId, leaseOwner });
-    else await input.db.transaction(async tx => {
+    else if (claimFailed || input.deferIneligible !== false) await input.db.transaction(async tx => {
       await tx.execute(sql`select set_config('statement_timeout', '10000', true), set_config('lock_timeout', '2000', true)`);
+      await notifyLiveRunRecovery(tx, { companyId: candidate.companyId, runId: candidate.runId });
       await tx.update(nativeRunFinalizations).set({
         nextAttemptAt: new Date(now.getTime() + 30_000),
         updatedAt: now,
@@ -377,6 +388,7 @@ export async function claimNativeSessionResumptions(input: {
         or(isNull(nativeRunFinalizations.leaseOwner), isNull(nativeRunFinalizations.leaseExpiresAt), lte(nativeRunFinalizations.leaseExpiresAt, now)),
       ));
     }).catch((error: unknown) => {
+      input.onError?.(error);
       logger.warn({ err: error, companyId: candidate.companyId, runId: candidate.runId },
         "failed to defer native session recovery candidate");
     });

server/src/services/native-runtime/native-question-bridge.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
 import { and, eq, inArray, sql } from "drizzle-orm";
 
 import type { Db } from "@paperclipai/db";
@@ -348,6 +349,7 @@ export async function requestNativeQuestionRunCancellation(
     ...cause,
     requestedAt: new Date().toISOString(),
   });
+  await notifyLiveRunRecovery(db, { companyId: interaction.companyId, runId: run.id });
   return db.update(heartbeatRuns).set({
     contextSnapshot: sql`jsonb_set(
       case

server/src/services/native-runtime/native-restart-recovery.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
 import { NATIVE_RETAINED_OWNER_WAIT } from "./native-controller-recovery.js";
 import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
 import { remoteWorkspaceRunReferenceSchema } from "@paperclipai/shared";
@@ -664,6 +665,7 @@ export async function claimNativeRestartRecoveries(input: {
           hasProviderEvidence: false,
           stderrTail: row.run.stderrExcerpt,
         });
+        await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
         await tx
           .update(nativeRunFinalizations)
           .set({
@@ -1048,6 +1050,7 @@ export async function claimNativeRestartRecoveries(input: {
         recycledProviderPids: providerProcesses.recycledPids,
         stderrTail: row.run.stderrExcerpt,
       });
+      await notifyLiveRunRecovery(tx, { companyId: row.run.companyId, runId: row.run.id, agentId: row.run.agentId });
       const claimed = await tx
         .update(nativeRunFinalizations)
         .set({

server/src/services/native-runtime/native-runner-checkpoint.ts

@@ -1,3 +1,4 @@
+import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js";
 import { and, eq, exists, gt, isNull, lte, inArray, notExists, or, sql } from "drizzle-orm";
 import { checkpointDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db";
 import { inspectDurablePrpCheckpointHeader, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint, type DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js";
@@ -105,6 +106,7 @@ export function nativeRunnerCheckpointWriter(db: Db, owner: NativeRunnerCheckpoi
           eq(heartbeatRuns.nativeSessionId, checkpoint.predecessor.normalizedSessionId)));
         if (!prior) throw new Error("native_runner_checkpoint_binding_mismatch");
       }
+      await notifyLiveRunRecovery(tx, { companyId: owner.companyId, runId: owner.runId });
       if (checkpoint.reset) await tx.delete(nativeRunnerCheckpoints).where(and(
         eq(nativeRunnerCheckpoints.companyId, owner.companyId), eq(nativeRunnerCheckpoints.runId, owner.runId)));
       else if (checkpoint.removedRecords.length) await tx.delete(nativeRunnerCheckpoints).where(and(

server/src/services/recovery/service.ts

@@ -2384,11 +2384,7 @@ export function recoveryService(
     });
   }
 
-  async function scanSilentActiveRuns(opts?: {
-    now?: Date;
-    companyId?: string;
-    issueCreatedAtGte?: Date | null;
-  }) {
+  async function scanSilentActiveRuns(opts?: Parameters<typeof watchdog.scanSilentActiveRuns>[0]) {
     return watchdog.scanSilentActiveRuns(opts);
   }
 
@@ -6568,6 +6564,7 @@ export function recoveryService(
     escalateStrandedAssignedIssue,
     recordWatchdogDecision,
     scanSilentActiveRuns,
+    registerActiveRunSilenceWork: watchdog.registerSilenceWork,
     reconcileStrandedAssignedIssues,
     reconcileUnassignedBlockingIssues,
     reconcileActiveRecoveryActions,

server/src/services/run-cost-accounting.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { usdToUnits, unitsToCents } from "@paperclipai/shared";
 import { and, asc, eq, inArray, or, sql } from "drizzle-orm";
 import { accountingRuntimeBaselines, budgetReservations, agentRuntimeState, agents, costEvents, heartbeatRuns, nativeRunFinalizations, issues, projects, decisionInvocations, promptInjectionInvocations, type Db } from "@paperclipai/db";
@@ -38,6 +39,7 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService
     let [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).for("update");
     if (!run?.costAccountingPending || !terminalStatuses.includes(run.status)) return false;
     if (run.costAccountedAt) {
+      await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: run.agentId });
       await tx.update(heartbeatRuns).set({ costAccountingPending: false }).where(eq(heartbeatRuns.id, run.id));
       await budgetServiceInTransaction(tx, publications).evaluateAffectedScopes(run.companyId, run.agentId,
         text(object(object(run.usageJson).ledgerScope).projectId));
@@ -69,6 +71,7 @@ export async function accountRunCost(db: Db, runId: string, hooks: BudgetService
     if (!preProviderFailure && object(run.usageJson).accountingReceiptReady === false) return false;
     // Acknowledgement is part of the same transaction. Budget checks inside
     // this transaction must not mistake this receipt for missing accounting.
+    await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: run.agentId });
     await tx.update(heartbeatRuns).set({ costAccountingPending: false, costAccountedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
     await tx.update(budgetReservations).set({ state: preProviderFailure ? "released" : "settled", settledAt: new Date() })
       .where(and(eq(budgetReservations.companyId, run.companyId), eq(budgetReservations.runId, run.id), eq(budgetReservations.state, "held")));

server/src/services/run-dispatch-notifications.ts

@@ -6,6 +6,7 @@ export type RunDispatchScope = { companyId: string };
 export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) {
   await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined);
   const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
+  await notifyDeliveryWork(transaction, DELIVERY_QUEUES.nativeProcessSessions, hint);
   await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint);
 }
 

server/src/services/task-retry.ts

@@ -1,3 +1,4 @@
+import { notifyStrandedAgentStatus } from "./live-run-recovery-notifications.js";
 import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
 import { writeExecutionControl } from "./execution-control-notifications.js";
 import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
@@ -183,6 +184,12 @@ export async function retireSupersededTaskRetry(db: Db, input: {
       sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${input.issueId}`),
   )).orderBy(desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1);
   if (latest?.id === input.sourceRunId) return false;
+  const [wake] = await db.select({ agentId: agentWakeupRequests.agentId }).from(agentWakeupRequests).where(and(
+    eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.id, input.requestId),
+    inArray(agentWakeupRequests.status, ["deferred_issue_execution", "queued", "claimed"]),
+  )).for("update");
+  if (!wake) return false;
+  await notifyStrandedAgentStatus(db, { companyId: input.companyId, agentId: wake.agentId });
   await notifyExternalConversationSettlement(db, input);
   const retired = await db.update(agentWakeupRequests).set({ status: "skipped", error: "task_retry_superseded",
     finishedAt: new Date(), updatedAt: new Date() }).where(and(
@@ -375,6 +382,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?
       updatedAt: new Date(),
     }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution")));
     if (receipt?.runId || failed) await db.transaction(async tx => {
+      await notifyStrandedAgentStatus(tx, { companyId: wake.companyId, agentId: wake.agentId });
       await notifyExternalConversationSettlement(tx, { companyId: wake.companyId,
         issueId: typeof wake.payload?.issueId === "string" ? wake.payload.issueId : undefined });
       await write(tx);