merge: preserve status-card and recovery producers in the review stack

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/scripts/tests/docker-canary-promotion.test.mjs

@@ -11,7 +11,7 @@ const script = job.split("        run: |\n")[1].split("\n").map(line => line.rep
 const sha = "a".repeat(40);
 
 test("canary promotion waits for the standard manifest and keeps its serialized channel", () => {
-  assert.match(job, /needs: \[merge-and-push\]/);
+  assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/);
   assert.match(job, /group: docker-canary-channel-promotion/);
   assert.match(job, /cancel-in-progress: false/);
 });

.github/workflows/docker.yml

@@ -33,6 +33,35 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  reuse_canary_image:
+    if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      packages: write
+      attestations: read
+    outputs:
+      reused: ${{ steps.reuse.outputs.reused }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
+        with:
+          node-version: 24
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Reuse the verified master image when present
+        id: reuse
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: node scripts/reuse-canary-image.mjs
+
   # Multi-arch by native runner, not QEMU.
   #
   # This was one job building linux/amd64,linux/arm64 together on an x86
@@ -56,6 +85,8 @@ jobs:
   # digest; `merge` assembles the manifest list. Both architectures remain
   # available to self-hosted installations and downstream image composers.
   build-and-push:
+    needs: [reuse_canary_image]
+    if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }}
     strategy:
       # Independent legs: one architecture failing should still publish
       # nothing, but it must not also hide the other's logs behind a
@@ -277,6 +308,7 @@ jobs:
   # tags. Nothing is publicly tagged until this runs, so a half-published
   # multi-arch image is not a state anything can pull.
   merge-and-push:
+    if: ${{ !cancelled() && needs.build-and-push.result == 'success' }}
     needs: build-and-push
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -407,11 +439,8 @@ jobs:
   # promotion is gated on its own completed pushes). Every interleaving
   # converges the Docker channel onto the npm channel.
   promote_canary_channel:
-    if: startsWith(github.ref, 'refs/tags/canary/v')
-    # merge-and-push, not build-and-push: the per-arch legs push untagged
-    # digests, and the production `sha-*` tags this promotion retags only
-    # exist once the manifest merge has named them.
-    needs: [merge-and-push]
+    if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }}
+    needs: [reuse_canary_image, merge-and-push]
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:

.github/workflows/pr-trusted.yml

@@ -202,7 +202,7 @@ jobs:
         run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs
 
       - name: Test release verify workflow wiring
-        run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
+        run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs
 
       - name: Test standalone package build concurrency
         run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs

doc/RELEASING.md

@@ -86,7 +86,7 @@ internal traceability tag.
   moves them to the versioned name
 - canaries, nightlies, and betas never create GitHub Releases
 - canaries, nightlies, and betas never require changelog generation
-- Docker `:latest` moves only on stable releases; master builds publish
+- Docker `:latest` moves only on stable releases; canary dispatches promote
   `:canary`, nightly builds `:nightly`, and beta builds `:beta`
 
 ## TL;DR
@@ -310,12 +310,12 @@ The workflow:
 
 ## Docker Image Tags
 
-[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted
-image and the `-cloud` variant with the same lane mapping:
+[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image:
 
 | Build ref | Tags |
 | --- | --- |
-| `master` push | `:canary`, `:sha-<short>` |
+| `master` push | `:sha-<short>`, `:sha-<full>` |
+| `canary/v*` dispatch | `:sha-<short>`, then `:canary` for the current npm canary |
 | `nightly/v*` tag | `:nightly`, `:sha-<short>` |
 | `beta/v*` tag | `:beta`, `:sha-<short>` |
 | `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-<short>` |
@@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs
 therefore dispatch `docker.yml` explicitly at the new tag ref; the tag
 mapping keys off `github.ref` either way.
 
+Canary dispatches first look up the full-SHA master image. Reuse requires a
+valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching
+registry digest, and signed provenance from this repository's `docker.yml`
+on `master` at the exact source commit. The image is retagged by immutable
+digest. This preserves the original build inputs and skips both native builds
+and the manifest merge. A missing full-SHA image runs the normal build instead;
+registry errors or invalid provenance fail the run. Rerun after the master
+image's attestation finishes if the image became visible before its proof.
+
+Both paths retain the serialized promotion that resolves the current npm
+`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and
+master builds retain their existing build paths. Downstream image builds can
+continue to consume the signed full-SHA master image independently of npm
+canary publication.
+
 The Docker build matrix and the release preview image builder configure
 BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers
 both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as

doc/evals.md

@@ -76,6 +76,12 @@ one bound continuation, and an exact saved confirmation. Its four development
 cells use the recorded Codex and Claude models. Registration and calibration do
 not establish Slack, staging authentication, warm lifecycle, or live qualification.
 
+The explicit `followup-while-running` [Product journey](../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up)
+checks a saved message during independently observed native work, one attributed
+reply with the late requirement, and persistence after reload. Its attached and
+managed targets share the same story and permit one steered run or one normal
+successor. Registration and calibration are separate from a live staging pass.
+
 [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit
 deployment targets and starting-state recipes. The shared task-properties
 journey uses no model; Cloud onboarding supplies reusable verified state. Cloud

doc/idle-sleep-safety.md

@@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers.
 Caller timeouts retain their slots until the actual worker reply or exit;
 timeouts cannot accumulate additional provider reads in later sweeps.
 
+If a failed ephemeral lease has no usable local inspection path, a managed
+Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks
+the stack's historical credential organization and reads the exact allocation
+with its manager credential. Only confirmed absence can produce a signed receipt
+bound to the stack, company, environment, lease, provider handle and nullable run.
+The separate cleanup type and audience cannot authorize run recovery. Older
+Cloud servers, unsupported providers, invalid signatures, changed organizations
+and failed reads leave the lease unresolved. Existing bounded cleanup batches
+and retry deadlines own these reads; no new idle polling is added. An explicit
+local provider observation is retained rather than replaced by this fallback.
+
 Host-owned verification metadata binds the company, environment, resource,
 provider handle, optional run, and exact row version. Concurrent resume,
 replacement, cleanup, or configuration changes invalidate verification.

doc/observability.md

@@ -328,6 +328,14 @@ once known, `paperclip.native_reconciliation.candidates` (a count). A failed
 pass marks the pass span and the phase that was running with an error status.
 Faster passes record nothing.
 
+Heartbeat configuration preparation overlaps independent reads. Skill preparation
+runs alongside connector assignments; standing connection instructions wait for
+assignment catalog repairs, but can overlap the skill branch. These spans can
+finish out of order. Do not add their durations as serial elapsed time. The
+historical `skills.prepare` span ends when the skill branch settles, rather than
+when its concurrent branches finish. Each preparation attempt settles its started
+branches before it returns an error or releases acquired credentials.
+
 ### Detailed execution phases
 
 The `paperclip.execution` tracer adds these content-free spans beneath the current

doc/plans/2026-10-30-deferred-execution-sleep.md

@@ -0,0 +1,23 @@
+# Deferred execution receipt ownership
+
+This slice removes the periodic `resumeQueuedRuns` sweep after queue dispatch migrated separately. It also delegates native status wake intents out of the periodic orphan reaper. Existing queue admission, authorization, task recovery, cancellation, resource and controller checks remain the mutation authority.
+
+| Duty | Durable pending truth | Committing producers | Deadline and exit |
+| --- | --- | --- | --- |
+| Task Retry | deferred task_retry wake with certified actor/generation | requestTaskRetry through execution control; execution, issue, agent, company and budget eligibility | immediate; current admission links canonical intent to one dispatch/run, supersession or terminal policy retires it; blocked input awaits its actual gate mutation |
+| Saved comments and interaction responses | deferred wake carrying saved comment IDs or interaction mutation | wake admission insert; merge only genuinely new comment IDs or interaction identity; issue/lock/assignment/dependency and execution eligibility | immediate after terminal source; original release owner revalidates comment/card authority, process/cleanup evidence and issue claim |
+| Execution wait | deferred wake with executionWait | original admission/restored source receipt; actual recovery action, execution and eligibility changes | original updatedAt + 30 seconds, then current owner revalidation; unchanged blocked receipt carries no recurring scan |
+| Interrupt retry | deferred wake with queuedCommentInterrupt | authorized route persists interrupt marker; cancellation lifecycle and cleanup/eligibility transitions | original updatedAt + 30 seconds; cancelled legacy run naming exact queue bypasses initial grace as before; owner retains explicit interrupt and stop evidence |
+| Cancelled legacy queue | cancelled legacy run result names exact still-deferred queue | run lifecycle terminal/control commits; original interrupt intent | immediate restart reconstruction; original release/admission owner settles or leaves saved input behind its actual blocker |
+| Task chat receipt settlement | deferred task_retry with taskRetry.chatActionId | failed chat retry action settlement | immediate settlement-only pass folds linked action result; it cannot restage its own processed-but-deferred chat action. Startup/actual eligibility retains normal re-admission policy |
+| Native status wake intent | system/native-status-committer wake queued/claimed without run | status committer transaction; actual execution/issue/agent/company/budget eligibility | immediate queued intent; claimedAt + 60 seconds for crash recovery; error updatedAt + 30 seconds. Existing deterministic dispatcher identity/claims prevent duplicate delivery; original intent becomes linked/coalesced/skipped while a separate blocked dispatch receipt owns saved admission |
+
+Three scoped owners reconstruct only their actual durable receipt predicates, with keyset pages of 100. Normal signals contain normalized company/request identity. Instance scheduler settings alone may request instance-wide reconstruction. Repeated source changes coalesce reconstruction for 30 seconds; already-known stored deadlines preempt that cooldown. Pending checks read memory only. Shutdown uses the existing coordinator and physical execution tracking; task drain reconciles producer transactions and releases the settled intent hold while retaining work for resume. Warm standby and disabled registration do not execute domain work.
+
+The generic run-dispatch notifier and broad issue review-state notifier deliberately do not fan out deferred eligibility. Queue bookkeeping may write them while processing its own saved input. Separate eligibility commits and settlement-only chat notifications avoid a self-restaging loop.
+
+C2 remains separate and mandatory: active output silence deadlines, remaining orphan/process/controller/remote authority and resource debts, legacy shutdown settlements, stranded agent status, and final aggregate chain removal. Retained workspace and migrated native finalization workers remain their existing owners.
+
+Review controls cover queue edits that move an already loaded grace deadline. Each due check re-reads current stored grace or claim time before admission. Subtree hold release, subtree restore, and member archival publish eligibility inside their own transactions. Member archival preserves the cancelled Retry tombstone created by an earlier user reassignment.
+
+A resume that first records a new execution-wait reason arms its existing 30-second grace once. Writing the same closed-gate reason again does not publish another hint. A clock obligation, an eligibility change, and an infrastructure failure therefore retain separate exits: stored deadline, owning committed producer, and bounded failure backoff. Refreshing a diagnostic timestamp alone does not create permanent polling behind a deliberate pause.

doc/plans/2026-10-30-issue-recovery-sleep.md

@@ -0,0 +1,28 @@
+# Issue recovery without periodic scans
+
+The issue recovery owners reconstruct their current obligations once at startup. Ordinary comment writes invalidate only the affected issue for assignment and lock recovery. Lifecycle, graph, run, configuration and eligibility changes invalidate their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database.
+
+| Owner | Durable truth and producer boundary | Deadline or event | Exit |
+| --- | --- | --- | --- |
+| Stranded assignment | Visible assigned todo/in-progress/review issues; issue create/update/checkout/release; run lifecycle and execution control; agent/company/budget/configuration; interaction, approval and subtree hold changes | A committed eligibility change. A transient continuation uses the latest finish plus the existing exponential delay. | An execution path, durable wait, escalation, or current ineligibility. |
+| Orphan blocker assignment | Agent-created unassigned todo/blocked blockers with a nonterminal dependent; issue, relation, agent and company changes | Once per changed company, independent of assignment candidates. | Assigned, no longer blocking, or creator cannot execute until its next eligibility change. |
+| Active recovery actions | Active/escalated action records, including actions on completed sources; action, issue, run and wait changes | Once per changed company, independent of assignment candidates. | Resolved, restored path, explicit hold or a separately owned retry. |
+| Resolved dependency | Blocked assigned issues and company-scoped blocker relations; blocker status or relation changes through issue service and import; run/control and pause gate changes | The committing change. A failed enqueue retains that issue with error backoff. | Existing idempotent wake, live path, unresolved dependency or deliberate self-block. |
+| Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. |
+| Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. |
+
+Local comment hints do not broaden into an assignment check for every open task. A real PostgreSQL regression starts with1,000 open assigned issues, writes one comment through `issueService.addComment`, and observes exactly one assignment check; a subsequent company eligibility notification still checks all1,000. Restoring the previous comment hint makes that same regression check all1,000 after the comment. The test substitutes the expensive assignment decision only to count its invocations; it uses the real transaction, notification, coordinator and paged SQL. Separate producer acceptance cases exercise real recovery decisions and process execution.
+
+Parent/descendant disposition repair and shared execution references make status and run changes broader than a comment. Their existing notification boundaries deliberately retain company scope. Orphan blockers, active recovery actions, resolved dependencies and watchdogs also keep their company/subtree duties; an issue comment does not drop those independent obligations. The scope serializer selects known primitive fields and never serializes an entire insert or update object.
+
+Reconstruction uses keyset pages of at most100rows. Repeated company hints coalesce reconstruction for30seconds. Already-known entry deadlines preempt that spacing without an early probe on each comment. Scheduling suppression is read once per pass. A failed read keeps scoped backoff debt. The two subsidiary company repairs do not run inside the per-issue loop. Legacy direct aggregate API calls retain both repairs.
+
+The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry.
+
+Producer coverage uses the existing run-dispatch, execution-control, issue-review-state, native status reassessment and external conversation settlement boundaries. Direct gaps have explicit signals inside their owning transaction: document writes/import, attachment removal, blocker import, recovery-action adoption/mutation, conversation pause release, Telegram failure comment replacement, native artifact title/metadata updates and company deletion. Metadata-only comment classification that does not change timestamp, visibility or execution eligibility needs no additional watchdog activation. Approval links and decisions use their existing issue-review/continuation effects. Work-product create/update/delete/import uses its existing native status boundary.
+
+The new owners replace only these four startup and periodic calls. Deferred comments, silent active runs and residual orphan/resource recovery retain their existing backstop until the next slice. Every previously migrated native-owner skip remains unchanged.
+
+A controlled process test prints an execution proof but cannot apply a task disposition. The test operator completes that task through issue service. If the existing continuation policy already saved a disposition-repair retry, the test preserves it until its stored deadline. Only after all provider work has joined does it advance the clock; dispatch then revalidates the completed task and cancels the retry. The final quiet proof requires no registered deadlines, no tracked in-flight work and no owner SQL over twenty former polling minutes. It does not claim a saved future retry is already drained.
+
+Active action repair has no separate grace or lease countdown. Its delayed disposition repair first writes a scheduled-retry run, then mirrors the date into action wakePolicy.retryAt and timeoutAt. The dispatch owner owns that saved run deadline. Other skipped gates wait for their existing execution, wait, pause, budget, or finalizer producer; the company action worker does not add a polling fallback.

doc/plans/2026-10-30-run-dispatch-sleep.md

@@ -0,0 +1,34 @@
+# Run dispatch without idle retry scans
+
+The run-dispatch module owns retry admission, its durable `scheduled_retry_at`
+deadline, and atomic promotion. The heartbeat queue owns concurrency, current
+execution authority, and dispatch. The shared coordinator owns notification
+coalescing, nonoverlap, transaction uncertainty, drain/resume and shutdown.
+The heartbeat facade and `index.ts` only connect these owners.
+
+A scheduled retry is a future obligation. Startup reconstructs its next stored
+deadline through the run-dispatch adapter. Writes that create, move, cancel or
+invalidate retries notify the retry worker in their transaction. Promotion uses
+the existing gate and compare-and-set, and its committed write wakes ordinary
+queued dispatch. A queued run behind occupied capacity waits for the owning
+run's lifecycle transition. Configuration and authority changes also wake it.
+An idle queue has no recurring read or timer-driven dispatch attempt. Notifications batch for one second and carry the changed company to queued dispatch. Known retry deadlines still preempt batching. Cancellation-owned retries stay with their cancellation worker and do not retain a promotion deadline.
+
+The supported writer contract is service writes through the shared database
+connection scope, including caller-owned and nested transactions. Existing
+explicit domain notification boundaries cover lifecycle, issue review state,
+agent configuration/status, company lifecycle and scheduling settings. Direct
+SQL writers and independent replicas must provide an explicit ingress signal or
+restart reconstruction; a process-local signal does not observe their commits.
+This change adds no permanent database listener or write interception.
+
+The remaining periodic recovery chain still owns deferred comment receipts,
+execution-wait cancellation recovery, orphan/resource recovery, stranded issue
+assignments, dependency wake repair, task watchdogs, silent active runs and stale
+locks. Those duties retain their backstop until their separate producer/deadline
+migrations are qualified. This slice removes only the periodic scheduled-retry
+promotion call. Ordinary queued dispatch can still be reached by that backstop;
+the existing atomic claim and per-agent start lock prevent duplicate execution.
+
+These workers use internal process deadlines. They do not register Cloud wake
+alarms and do not relax the durable idle-sleep safety proof.

maps/agent-access-map/instance-operations.md

@@ -62,6 +62,14 @@ authority as company-agent API access.
 Gaps: Instance start/stop and task workspace start/stop are different capabilities.
 Local service control is not agent company authority.
 
+Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence
+reads authenticate the tenant host with its server capability; they expose no
+new board or agent endpoint. Their signed scope cannot authorize task recovery.
+[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts)
+cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts)
+cover signature, identity, freshness and separation from recovery authority.
+These checks do not establish a deployed sleep/wake result.
+
 ## `configuration`
 
 Capability ID: `instance-operations/configuration`.

maps/feature-map/instance-operations.md

@@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect
 before headers arrive. Confirm that neither leaves a permanent request hold. An
 unfinished application write must still prevent sleep after its client leaves.
 
+Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts)
+exercises the runtime inspection path for failed Cloud sandbox leases, including
+verification leases without a run. A signed cleanup-only absence observation
+can remove this blocker without changing failed outcomes or granting run
+continuation. An unresolved provider resource still prevents idle sleep.
+Manual: on an owned Cloud staging workspace, confirm the exact published build,
+reconcile failed test allocations, then verify actual provider suspension and
+normal hosted entry after wake. Local tests do not prove that hosted journey.
+
 ### `settings-backup`
 
 Cloud startup reads nonsecret application choices from a private, versioned

maps/feature-map/steering.md

@@ -60,6 +60,13 @@ inspect the resulting task and run in Paperclip.
 
 The explicit `stop-redirect` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) covers `steering/stop-and-pause` only for Stop and one new direction. It requires active work, native cancellation acknowledgement, resource cleanup, and completion without later original-turn activity. It does not qualify pause, interrupt, or all queued-message behavior. Source registration is not a live Cloud pass.
 
+The explicit `followup-while-running` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up)
+covers `steering/queue`: one follow-up is saved while a bound native command is
+active, its late requirement appears in one attributed final reply, the queue
+empties, and the result survives reload. One steered turn or one normal successor
+is allowed. Unit calibration and source registration are not live acceptance.
+
+
 Preconditions: follow the [baseline](./README.md#before-driving-a-journey).
 Use an agent that stays busy long enough to inspect the queue. Record the adapter,
 native/legacy mode, and advertised steering capability. Test supported and

packages/paperclip-runner/docs/durable-recovery.md

@@ -98,6 +98,35 @@ digest, and refuse the rest. A profile identity does not establish hosted
 harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc)
 summarizes these practices for contributors.
 
+Qualified ACPX native executables use Runner's persistent digest cache. The cache
+stores a computed SHA-256 digest under a key made from device, inode, size,
+nanosecond modification time, and nanosecond change time. Each check compares
+that digest with the independently selected build-owned pin. A changed file
+identity, missing entry, or unusable cache causes a full descriptor read and
+hash. A hit reuses the earlier digest; metadata identity is not a fresh content
+hash. Claude, Codex, and Grok use the same native executable admission path.
+
+Cache entries live in the operating-system temporary directory under
+`paperclip-runner-<uid>/executable-digests-v1`, outside task workspaces such as
+`/home/agent`. Neither `HOME` nor the task working directory selects this path.
+Node's `os.tmpdir()` selects the root and honors the platform's absolute
+`TMPDIR`, `TMP`, or `TEMP` override. Hosts must keep that root outside task
+workspaces; a relative or unusable root disables caching. Temporary-directory
+eviction removes the optimization until a later full hash fills it again. The
+directory and files are private to that user. Writes use a temporary file and
+atomic rename, so later Runner or sidecar processes can reuse complete entries.
+Cache errors fall back to hashing. The cache stores no credentials or file paths.
+
+Every admission still opens the executable without following its final symlink,
+checks its regular-file type, bounds, execute permission, and before/after file
+identity, and retains the descriptor for launch. Package qualification and the
+preparation-to-launch identity check remain. The cache deliberately trusts the
+operating-system user: a process with that user's access can plant a digest for
+a chosen fingerprint. It is not authenticated with a secret or signature.
+An administrator-owned read-only Runner can therefore use a user-writable cache,
+but this provides weaker protection against that same user than hashing every
+executable on each admission. Removing the cache restores full hashing.
+
 ## External controller checkpoints
 
 The optional controller persistence callback stores recovery control state, not

packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts

@@ -0,0 +1,172 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { EventEmitter } from "node:events";
+import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { PassThrough, Writable } from "node:stream";
+import { afterEach, describe, expect, it, vi } from "vitest";
+
+const setup = vi.hoisted(() => ({ verify: vi.fn(), spawn: vi.fn(), close: vi.fn(), retire: vi.fn() }));
+vi.mock("./antigravity-installation.js", () => ({ antigravityNativeDistributionSpec: () => ({ pinned: true }) }));
+vi.mock("./installation-integrity.js", () => ({ verifyNativeAcpxInstallation: setup.verify }));
+vi.mock("./antigravity-process.js", async importOriginal => ({
+  ...await importOriginal<typeof import("./antigravity-process.js")>(), retireAntigravityProcess: setup.retire,
+}));
+import { startAntigravityBrowserLogin } from "./antigravity-auth.js";
+
+const clientId = "1071006060591-tmhssin2h21lcre235vtolojh4g403ep.apps.googleusercontent.com";
+const scopes = ["https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/aicode"];
+const authorizationUrl = "https://accounts.google.com/o/oauth2/v2/auth?" + new URLSearchParams({ client_id: clientId, response_type: "code",
+  redirect_uri: "http://127.0.0.1:18973/", scope: scopes.join(" "), state: "fixture_state_0123456789", code_challenge: "a".repeat(43),
+  code_challenge_method: "S256", access_type: "offline", prompt: "consent" });
+const roots: string[] = [];
+const physicalChildren: ChildProcess[] = [];
+afterEach(async () => {
+  for (const child of physicalChildren.splice(0)) {
+    if (child.exitCode === null && child.signalCode === null) {
+      try { process.kill(-child.pid!, "SIGKILL"); } catch {}
+    }
+  }
+  vi.unstubAllGlobals(); vi.clearAllMocks();
+  await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true })));
+});
+
+type Mode = "url-first" | "result-first" | "wrong-url" | "duplicate-url" | "unexpected-prompt" | "retirement-failure" | "credential-failure" | "auth-error" | "late-wrong-url" | "late-duplicate-url";
+async function fixture(mode: Mode) {
+  const home = await realpath(await mkdtemp(join(tmpdir(), "paperclip-antigravity-stream-test-"))); roots.push(home);
+  await mkdir(join(home, "antigravity-acp"), { mode: 0o700 });
+  await writeFile(join(home, "antigravity-acp/acp_token.json"), JSON.stringify({ client_id: clientId, client_secret: "synthetic-client-secret",
+    refresh_token: "synthetic-refresh-token", token_uri: "https://oauth2.googleapis.com/token", scopes }), { mode: 0o600 });
+  const stdout = new PassThrough(), stderr = new PassThrough(), requests: string[] = [];
+  const child = Object.assign(new EventEmitter(), { stdout, stderr, stdin: new Writable({ write(chunk, _encoding, done) {
+    const request = JSON.parse(String(chunk)); requests.push(request.method);
+    queueMicrotask(() => {
+      if (request.method === "initialize") {
+        stdout.write(JSON.stringify({ jsonrpc: "2.0", id: request.id, result: { protocolVersion: 1,
+          agentInfo: { name: "antigravity-acp", version: "1.4.3" }, agentCapabilities: { _meta: { antigravity: { paperclipAdmission: 1 } } },
+          authMethods: [{ id: "oauth-personal" }] } }) + "\n");
+      } else if (request.method === "authenticate") {
+        const url = mode === "wrong-url" || mode === "late-wrong-url" ? "https://example.invalid/synthetic-private-marker" : authorizationUrl;
+        const emitUrl = () => stderr.write("Open the following link to authenticate the ACP server: " + url + "\n");
+        const resultFirst = ["result-first", "late-wrong-url", "late-duplicate-url"].includes(mode);
+        if (!resultFirst) emitUrl();
+        if (mode === "duplicate-url") emitUrl();
+        if (mode === "unexpected-prompt") stdout.write(JSON.stringify({ jsonrpc: "2.0", id: 9, method: "session/prompt", params: {} }) + "\n");
+        stdout.write(JSON.stringify(mode === "auth-error"
+          ? { jsonrpc: "2.0", id: request.id, error: { code: -32000, message: "synthetic-private-marker" } }
+          : { jsonrpc: "2.0", id: request.id, result: {} }) + "\n");
+        if (resultFirst) queueMicrotask(() => { emitUrl(); if (mode === "late-duplicate-url") emitUrl(); });
+      } else throw new Error("Unexpected session/model request");
+    });
+    done();
+  } }) });
+  setup.close.mockResolvedValue(undefined);
+  setup.retire.mockImplementation(async (_child, closed: Promise<void>) => {
+    await Promise.resolve(); child.emit("close", 0); await closed;
+  });
+  if (mode === "retirement-failure") setup.retire.mockImplementationOnce(async () => { throw new Error("synthetic-private-marker"); });
+  setup.spawn.mockReturnValue(child);
+  setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) });
+  const fetch = vi.fn(async (url: string) => {
+    if (mode === "credential-failure") return new Response(null, { status: 401 });
+    if (url === "https://oauth2.googleapis.com/token") return Response.json({ access_token: "synthetic-access-token", token_type: "Bearer" });
+    if (url === "https://www.googleapis.com/oauth2/v3/userinfo") return Response.json({ email: "person@example.test", email_verified: true });
+    throw new Error("Unexpected network request");
+  });
+  vi.stubGlobal("fetch", fetch);
+  return { home, requests, fetch };
+}
+
+describe("Antigravity login cross-stream completion", () => {
+  it.each(["url-first", "result-first"] as const)("accepts the same authenticated exchange with %s delivery", async mode => {
+    const { home, requests, fetch } = await fixture(mode);
+    const login = startAntigravityBrowserLogin(home);
+    await vi.waitFor(() => expect(login.outcome).toBeDefined());
+    expect(login.outcome).toBe("success");
+    expect(login.authorizationUrl).toBe(authorizationUrl);
+    expect(requests).toEqual(["initialize", "authenticate"]);
+    expect(setup.retire).toHaveBeenCalledOnce(); expect(setup.close).toHaveBeenCalledOnce();
+    expect(fetch).toHaveBeenCalledTimes(2);
+    await login.abort();
+  });
+
+  it.each(["wrong-url", "duplicate-url", "unexpected-prompt", "retirement-failure", "credential-failure", "auth-error", "late-wrong-url", "late-duplicate-url"] as const)("keeps %s failures closed and private", async mode => {
+    const { home, requests, fetch } = await fixture(mode);
+    const login = startAntigravityBrowserLogin(home);
+    await vi.waitFor(() => expect(login.outcome).toBeDefined());
+    expect(login.outcome).toBe("failure"); expect(login.error).toBe("Google sign-in did not complete. Start sign-in again.");
+    expect(login.error).not.toContain("synthetic-private-marker");
+    await login.abort(); expect(setup.close).toHaveBeenCalledOnce();
+    expect(requests).toEqual(["initialize", "authenticate"]);
+    expect(fetch).toHaveBeenCalledTimes(mode === "credential-failure" ? 1 : 0);
+    if (mode === "wrong-url" || mode === "late-wrong-url") expect(login.authorizationUrl).toBeUndefined();
+  });
+
+  it.each(["valid", "invalid", "duplicate"] as const)("handles a %s URL from a real child after authenticate completion and stdin EOF", async mode => {
+    const home = await realpath(await mkdtemp(join(tmpdir(), "paperclip-antigravity-real-stream-"))); roots.push(home);
+    const peer = join(home, "peer.cjs");
+    await writeFile(peer, `
+const fs = require('node:fs');
+const path = require('node:path');
+const readline = require('node:readline');
+const home = process.argv[2], mode = process.argv[3], requests = [], events = [];
+const reader = readline.createInterface({input:process.stdin});
+reader.on('line', line => {
+ const request = JSON.parse(line); requests.push(request.method);
+ fs.writeFileSync(path.join(home,'requests.json'),JSON.stringify(requests));
+ if(request.method==='initialize') {
+  process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{protocolVersion:1,
+   agentInfo:{name:'antigravity-acp',version:'1.4.3'},agentCapabilities:{_meta:{antigravity:{paperclipAdmission:1}}},
+   authMethods:[{id:'oauth-personal'}]}})+'\\n');
+ } else if(request.method==='authenticate') {
+  fs.mkdirSync(path.join(home,'antigravity-acp'),{mode:0o700});
+  fs.writeFileSync(path.join(home,'antigravity-acp/acp_token.json'),${JSON.stringify(JSON.stringify({ client_id: clientId,
+      client_secret: "synthetic-client-secret", refresh_token: "synthetic-refresh-token", token_uri: "https://oauth2.googleapis.com/token", scopes }))},{mode:0o600});
+  events.push('authenticate-result');
+  process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{}})+'\\n');
+ } else throw new Error('Unexpected session/model request');
+});
+reader.on('close',()=>{
+ events.push('stdin-eof');
+ const url = mode==='invalid' ? 'https://example.invalid/synthetic-private-marker' : ${JSON.stringify(authorizationUrl)};
+ const message = 'Open the following link to authenticate the ACP server: '+url+'\\n';
+ process.stderr.write(mode==='duplicate' ? message+message : message,()=>{
+  events.push('url-flushed');fs.writeFileSync(path.join(home,'events.json'),JSON.stringify(events));process.exit(0);
+ });
+});
+`);
+    setup.spawn.mockImplementation((_args, options) => {
+      const child = spawn(process.execPath, [peer, home, mode], { ...options, detached: true });
+      physicalChildren.push(child); return child;
+    });
+    setup.close.mockResolvedValue(undefined);
+    setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) });
+    const realProcess = await vi.importActual<typeof import("./antigravity-process.js")>("./antigravity-process.js");
+    setup.retire.mockImplementation(realProcess.retireAntigravityProcess);
+    const fetch = vi.fn(async (url: string) => {
+      if (url === "https://oauth2.googleapis.com/token") return Response.json({ access_token: "synthetic-access-token",
+        refresh_token: "synthetic-rotated-refresh", token_type: "Bearer" });
+      if (url === "https://www.googleapis.com/oauth2/v3/userinfo") return Response.json({ email: "person@example.test", email_verified: true });
+      throw new Error("Unexpected network request");
+    });
+    vi.stubGlobal("fetch", fetch);
+    const login = startAntigravityBrowserLogin(home);
+    await vi.waitFor(() => expect(login.outcome).toBeDefined(), { timeout: 5_000 });
+    expect(login.outcome).toBe(mode === "valid" ? "success" : "failure");
+    expect(JSON.parse(await readFile(join(home, "events.json"), "utf8"))).toEqual(["authenticate-result", "stdin-eof", "url-flushed"]);
+    expect(JSON.parse(await readFile(join(home, "requests.json"), "utf8"))).toEqual(["initialize", "authenticate"]);
+    expect(physicalChildren.at(-1)!.exitCode).toBe(0);
+    expect(setup.retire).toHaveBeenCalledOnce(); expect(setup.close).toHaveBeenCalledOnce();
+    expect(fetch).toHaveBeenCalledTimes(mode === "valid" ? 2 : 0);
+    if (mode === "valid") {
+      expect(login.authorizationUrl).toBe(authorizationUrl);
+      expect(JSON.parse(await readFile(join(home, "antigravity-acp", "acp_token.json"), "utf8")).refresh_token).toBe("synthetic-rotated-refresh");
+    } else {
+      expect(login.error).toBe("Google sign-in did not complete. Start sign-in again.");
+      expect(login.error).not.toContain("synthetic-private-marker");
+      if (mode === "invalid") expect(login.authorizationUrl).toBeUndefined();
+    }
+    await login.abort();
+    expect(setup.close).toHaveBeenCalledOnce();
+  }, 15_000);
+});

packages/paperclip-runner/src/drivers/acpx/antigravity-auth.ts

@@ -203,7 +203,7 @@ export function startAntigravityBrowserLogin(home: string): AntigravityBrowserLo
             if (!line.startsWith(AUTH_PREFIX)) continue;
             try {
               const url = validateAntigravityAuthorizationUrl(line.slice(AUTH_PREFIX.length));
-              if (phase !== 2 || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR);
+              if ((phase !== 2 && phase !== 3) || state.authorizationUrl !== undefined) throw new Error(LOGIN_ERROR);
               state.authorizationUrl = url;
             } catch { fail(); }
           }

packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts

@@ -0,0 +1,160 @@
+import { createHash } from "node:crypto";
+import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createExecutableDigestCache, executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js";
+import { openVerifiedRuntimeExecutable } from "./installation-integrity.js";
+
+const temporary: string[] = [];
+const identity: ExecutableFileIdentity = { device: "1", inode: "2", size: "3", modifiedNanoseconds: "4", changedNanoseconds: "5" };
+const digest = `sha256:${"a".repeat(64)}`;
+afterEach(async () => {
+  vi.restoreAllMocks();
+  await Promise.all(temporary.splice(0).map(path => rm(path, { recursive: true, force: true })));
+});
+
+async function fixture() {
+  const root = await mkdtemp(join(tmpdir(), "runner-executable-digest-cache-"));
+  temporary.push(root);
+  const directory = join(root, "cache");
+  const cache = createExecutableDigestCache(directory);
+  return { root, directory, cache };
+}
+
+describe("persistent executable digest cache", () => {
+  it("stores the actual digest in a private, versioned entry and survives a new cache instance", async () => {
+    const f = await fixture();
+    expect(await f.cache.read(identity)).toBeNull();
+    await f.cache.write(identity, digest);
+    expect(await createExecutableDigestCache(f.directory).read(identity)).toBe(digest);
+    const [name] = await readdir(f.directory);
+    expect(name).toMatch(/^v1-[a-f0-9]{64}\.json$/);
+    expect(JSON.parse(await readFile(join(f.directory, name), "utf8"))).toEqual({ version: 1, digest });
+    expect((await lstat(f.directory)).mode & 0o777).toBe(0o700);
+    expect((await lstat(join(f.directory, name))).mode & 0o777).toBe(0o600);
+  });
+
+  it.each(Object.keys(identity) as (keyof ExecutableFileIdentity)[])("invalidates a changed %s", async key => {
+    const f = await fixture();
+    await f.cache.write(identity, digest);
+    expect(await f.cache.read({ ...identity, [key]: "changed" })).toBeNull();
+  });
+
+  it.each(["malformed", "version", "digest", "oversized", "permissions", "symlink"] as const)(
+    "treats an unusable %s entry as a miss", async kind => {
+      const f = await fixture();
+      await f.cache.write(identity, digest);
+      const [name] = await readdir(f.directory), file = join(f.directory, name);
+      if (kind === "malformed") await writeFile(file, "{");
+      else if (kind === "version") await writeFile(file, JSON.stringify({ version: 2, digest }));
+      else if (kind === "digest") await writeFile(file, JSON.stringify({ version: 1, digest: "expected" }));
+      else if (kind === "oversized") await writeFile(file, "x".repeat(1025));
+      else if (kind === "permissions") await chmod(file, 0o644);
+      else { await rename(file, `${file}.target`); await symlink(`${file}.target`, file); }
+      expect(await f.cache.read(identity)).toBeNull();
+    },
+  );
+
+  it("falls back when the cache directory cannot be used and leaves existing permissions unchanged", async () => {
+    const f = await fixture();
+    await writeFile(f.directory, "not a directory");
+    await expect(f.cache.write(identity, digest)).resolves.toBeUndefined();
+    expect(await f.cache.read(identity)).toBeNull();
+    await rm(f.directory);
+    await mkdir(f.directory, { mode: 0o755 });
+    await f.cache.write(identity, digest);
+    expect(await f.cache.read(identity)).toBeNull();
+    expect((await lstat(f.directory)).mode & 0o777).toBe(0o755);
+  });
+
+  it("does not resolve a relative cache directory through the task working directory", async () => {
+    const cache = createExecutableDigestCache("relative-task-cache");
+    await cache.write(identity, digest);
+    expect(await cache.read(identity)).toBeNull();
+  });
+
+  it("publishes complete entries under concurrent writers without leftover temporary files", async () => {
+    const f = await fixture();
+    await Promise.all(Array.from({ length: 8 }, () => f.cache.write(identity, digest)));
+    expect(await f.cache.read(identity)).toBe(digest);
+    expect(await readdir(f.directory)).toHaveLength(1);
+  });
+});
+
+describe.skipIf(!["linux", "darwin"].includes(process.platform))("descriptor admission with cached digests", () => {
+  async function executable() {
+    const f = await fixture(), path = join(f.root, "native"), bytes = Buffer.from("qualified native fixture");
+    await writeFile(path, bytes, { mode: 0o700 });
+    vi.spyOn(executableDigestCache, "read").mockImplementation(f.cache.read);
+    vi.spyOn(executableDigestCache, "write").mockImplementation(f.cache.write);
+    return { ...f, path, digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` };
+  }
+
+  it("hashes a miss, reuses the actual digest, and still rejects a different independent pin", async () => {
+    const f = await executable();
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    expect(executableDigestCache.write).toHaveBeenCalledOnce();
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    expect(executableDigestCache.write).toHaveBeenCalledOnce();
+    await expect(openVerifiedRuntimeExecutable(f.path, digest, "fixture")).rejects.toThrow("digest mismatch");
+  });
+
+  it("rehashes changed contents and rejects corruption despite a previous valid entry", async () => {
+    const f = await executable();
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    await writeFile(f.path, "corrupted native fixture");
+    await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
+  });
+
+  it.each(["permissions", "symlink", "replacement"] as const)("retains admission checks after cached %s changes", async kind => {
+    const f = await executable();
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    if (kind === "permissions") {
+      await chmod(f.path, 0o600);
+      await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("bounded executable file");
+    } else if (kind === "symlink") {
+      await rename(f.path, `${f.path}.original`); await symlink(`${f.path}.original`, f.path);
+      await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("real regular file");
+    } else {
+      await rename(f.path, `${f.path}.original`); await writeFile(f.path, "new invalid executable", { mode: 0o700 });
+      await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
+    }
+  });
+
+  it("rejects mutation while a cache hit is being read", async () => {
+    const f = await executable();
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    vi.mocked(executableDigestCache.read).mockImplementationOnce(async () => {
+      await writeFile(f.path, "mutation during lookup");
+      return f.digest;
+    });
+    await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified");
+  });
+
+  it("checks executable identity after publishing a cold cache entry", async () => {
+    const f = await executable();
+    vi.mocked(executableDigestCache.write).mockImplementationOnce(async (identity, digest) => {
+      await f.cache.write(identity, digest);
+      await writeFile(f.path, "mutation during cache publication");
+    });
+    await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified");
+  });
+
+  it("falls back to hashing when the cache is unavailable", async () => {
+    const f = await executable();
+    await writeFile(f.directory, "unavailable cache");
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+    await writeFile(f.path, "invalid native fixture");
+    await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
+  });
+
+  it("trusts a digest planted by the same operating-system user for current metadata", async () => {
+    const f = await executable();
+    await writeFile(f.path, "content changed by the same user");
+    const metadata = await lstat(f.path, { bigint: true });
+    await f.cache.write({ device: metadata.dev.toString(), inode: metadata.ino.toString(), size: metadata.size.toString(),
+      modifiedNanoseconds: metadata.mtimeNs.toString(), changedNanoseconds: metadata.ctimeNs.toString() }, f.digest);
+    await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
+  });
+});

packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts

@@ -2,6 +2,7 @@ import { MAX_ACPX_RUNTIME_EXECUTABLE_BYTES, ACPX_PRIVATE_SNAPSHOT_ENV, createAcp
 import { createNativeAcpxDistributionSnapshot, NATIVE_ACPX_BOOTSTRAP_NAME, readNativeAcpxDistributionEntries, type NativeAcpxDistributionInput } from "./native-distribution-integrity.js";
 export type { NativeAcpxDistributionInput } from "./native-distribution-integrity.js";
 import { createHash } from "node:crypto";
+import { executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js";
 import {
   spawn as spawnChildProcess,
   type ChildProcess,
@@ -606,13 +607,7 @@ export async function awaitVerifiedAcpxProviderExit(
   await exitProof;
 }
 
-interface VerifiedAcpxCommandIdentity {
-  device: string;
-  inode: string;
-  size: string;
-  modifiedNanoseconds: string;
-  changedNanoseconds: string;
-}
+type VerifiedAcpxCommandIdentity = ExecutableFileIdentity;
 
 interface VerifiedAcpxRuntimeExecutable {
   path: string;
@@ -1204,7 +1199,7 @@ async function inspectCommand(
   }
 }
 
-async function openVerifiedRuntimeExecutable(
+export async function openVerifiedRuntimeExecutable(
   executablePath: string,
   expectedDigest: string,
   agent: string,
@@ -1246,23 +1241,31 @@ async function openVerifiedRuntimeExecutable(
         `ACPX ${agent} runtime executable must be a bounded executable file`,
       );
     }
-    const hash = createHash("sha256");
-    const buffer = Buffer.alloc(1024 * 1024);
+    const cachedDigest = await executableDigestCache.read(fileIdentity(before));
+    let digest = cachedDigest;
     let position = 0;
-    try {
-      while (position < Number(before.size)) {
-        const { bytesRead } = await handle.read(
-          buffer,
-          0,
-          Math.min(buffer.length, Number(before.size) - position),
-          position,
-        );
-        if (bytesRead === 0) break;
-        hash.update(buffer.subarray(0, bytesRead));
-        position += bytesRead;
+    if (digest === null) {
+      const hash = createHash("sha256");
+      const buffer = Buffer.alloc(1024 * 1024);
+      try {
+        while (position < Number(before.size)) {
+          const { bytesRead } = await handle.read(
+            buffer,
+            0,
+            Math.min(buffer.length, Number(before.size) - position),
+            position,
+          );
+          if (bytesRead === 0) break;
+          hash.update(buffer.subarray(0, bytesRead));
+          position += bytesRead;
+        }
+      } finally {
+        buffer.fill(0);
       }
-    } finally {
-      buffer.fill(0);
+      digest = `sha256:${hash.digest("hex")}`;
+    }
+    if (cachedDigest === null && position === Number(before.size) && digest === expectedDigest) {
+      await executableDigestCache.write(fileIdentity(before), digest);
     }
     const after = await handle.stat({ bigint: true });
     const lexicalAfter = await lstat(executablePath, { bigint: true }).catch(
@@ -1271,7 +1274,7 @@ async function openVerifiedRuntimeExecutable(
     const beforeIdentity = fileIdentity(before);
     const afterIdentity = fileIdentity(after);
     if (
-      position !== Number(before.size) ||
+      (cachedDigest === null && position !== Number(before.size)) ||
       lexicalAfter === null ||
       lexicalAfter.isSymbolicLink() ||
       !lexicalAfter.isFile() ||
@@ -1283,7 +1286,6 @@ async function openVerifiedRuntimeExecutable(
         `ACPX ${agent} runtime executable changed while it was verified`,
       );
     }
-    const digest = `sha256:${hash.digest("hex")}`;
     if (digest !== expectedDigest) {
       throw new Error(`ACPX ${agent} runtime executable digest mismatch`);
     }

packages/paperclip-runner/src/executable-digest-cache.ts

@@ -0,0 +1,70 @@
+import { createHash, randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, mkdir, open, rename, unlink } from "node:fs/promises";
+import { tmpdir, userInfo } from "node:os";
+import { isAbsolute, join } from "node:path";
+
+export interface ExecutableFileIdentity {
+  device: string;
+  inode: string;
+  size: string;
+  modifiedNanoseconds: string;
+  changedNanoseconds: string;
+}
+
+export function createExecutableDigestCache(directory?: string) {
+  async function location(identity: ExecutableFileIdentity, create: boolean): Promise<string> {
+    const user = userInfo();
+    const temporaryRoot = tmpdir();
+    if (!isAbsolute(directory ?? temporaryRoot)) throw new Error("Executable digest cache directory is unavailable");
+    const base = join(temporaryRoot, `paperclip-runner-${user.uid}`);
+    const directories = directory ? [directory] : [base, join(base, "executable-digests-v1")];
+    for (const [index, path] of directories.entries()) {
+      if (create) await mkdir(path, { mode: 0o700 }).catch(error => {
+        if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      });
+      const metadata = await lstat(path);
+      if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.uid !== user.uid ||
+          (metadata.mode & (index === directories.length - 1 ? 0o077 : 0o022)) !== 0) {
+        throw new Error("Executable digest cache directory is unavailable");
+      }
+    }
+    const key = createHash("sha256").update(JSON.stringify([1, identity.device, identity.inode, identity.size,
+      identity.modifiedNanoseconds, identity.changedNanoseconds])).digest("hex");
+    return join(directories.at(-1)!, `v1-${key}.json`);
+  }
+
+  return {
+    async read(identity: ExecutableFileIdentity): Promise<string | null> {
+      try {
+        const path = await location(identity, false);
+        const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+        try {
+          const metadata = await handle.stat();
+          if (!metadata.isFile() || metadata.uid !== userInfo().uid || (metadata.mode & 0o077) !== 0 ||
+              metadata.size < 1 || metadata.size > 1024) return null;
+          const entry: unknown = JSON.parse(await handle.readFile("utf8"));
+          if (typeof entry !== "object" || entry === null || !("version" in entry) || entry.version !== 1 ||
+              !("digest" in entry) || typeof entry.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(entry.digest)) return null;
+          return entry.digest;
+        } finally { await handle.close(); }
+      } catch { return null; }
+    },
+    async write(identity: ExecutableFileIdentity, digest: string): Promise<void> {
+      let temporary: string | undefined;
+      try {
+        if (!/^sha256:[a-f0-9]{64}$/.test(digest)) return;
+        const path = await location(identity, true);
+        temporary = `${path}.${randomUUID()}.tmp`;
+        const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+        try { await handle.writeFile(JSON.stringify({ version: 1, digest })); }
+        finally { await handle.close(); }
+        await rename(temporary, path);
+      } catch { } finally {
+        if (temporary) await unlink(temporary).catch(() => undefined);
+      }
+    },
+  };
+}
+
+export const executableDigestCache = createExecutableDigestCache();

scripts/reuse-canary-image.mjs

@@ -0,0 +1,34 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { appendFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+import { resolveStandardImageDigest } from "./standard-image-contract.mjs";
+
+export async function reuseCanaryImage(sha, {
+  resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }),
+  run = (command, args) => execFileSync(command, args, { stdio: "inherit" }),
+} = {}) {
+  assert.match(sha ?? "", /^[a-f0-9]{40}$/);
+  const digest = await resolve(sha);
+  if (digest === null) return false;
+  assert.match(digest, /^sha256:[a-f0-9]{64}$/);
+  const image = `ghcr.io/paperclipai/paperclip@${digest}`;
+  await run("gh", ["attestation", "verify", `oci://${image}`,
+    "--repo", "paperclipai/paperclip", "--bundle-from-oci",
+    "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml",
+    "--source-ref", "refs/heads/master", "--source-digest", sha,
+    "--signer-digest", sha, "--deny-self-hosted-runners"]);
+  await run("docker", ["buildx", "imagetools", "create", "-t",
+    `ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]);
+  return true;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const reused = await reuseCanaryImage(process.env.GITHUB_SHA);
+  if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`);
+  const summary = reused
+    ? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge."
+    : "No full-SHA master image exists. Building both platforms for this canary.";
+  console.log(summary);
+  if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`);
+}

scripts/reuse-canary-image.test.mjs

@@ -0,0 +1,75 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+import { reuseCanaryImage } from "./reuse-canary-image.mjs";
+
+const sha = "a".repeat(40);
+const digest = `sha256:${"b".repeat(64)}`;
+
+test("reuses only an exact-source, master-signed multi-platform image by digest", async () => {
+  const calls = [];
+  const reused = await reuseCanaryImage(sha, {
+    resolve: async source => { assert.equal(source, sha); return digest; },
+    run: async (...args) => calls.push(args),
+  });
+  assert.equal(reused, true);
+  assert.deepEqual(calls, [
+    ["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`,
+      "--repo", "paperclipai/paperclip", "--bundle-from-oci",
+      "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml",
+      "--source-ref", "refs/heads/master", "--source-digest", sha,
+      "--signer-digest", sha, "--deny-self-hosted-runners"]],
+    ["docker", ["buildx", "imagetools", "create", "-t",
+      "ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]],
+  ]);
+});
+
+test("a missing master image enables the fallback without retagging", async () => {
+  assert.equal(await reuseCanaryImage(sha, {
+    resolve: async () => null,
+    run: () => assert.fail("A missing image must not be promoted"),
+  }), false);
+});
+
+test("registry and provenance failures stop promotion rather than bypass verification", async () => {
+  await assert.rejects(reuseCanaryImage(sha, {
+    resolve: async () => { throw new Error("HTTP 403"); },
+    run: () => assert.fail("A registry failure must not be promoted"),
+  }), /HTTP 403/);
+  let commands = 0;
+  await assert.rejects(reuseCanaryImage(sha, {
+    resolve: async () => digest,
+    run: command => {
+      assert.equal(command, "gh");
+      commands++;
+      throw new Error("Untrusted source or signature");
+    },
+  }), /Untrusted source/);
+  assert.equal(commands, 1);
+});
+
+test("invalid source refs and digests cannot reach external commands", async () => {
+  for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) {
+    await assert.rejects(reuseCanaryImage(invalid, {
+      resolve: () => assert.fail("Invalid source accepted"),
+    }));
+  }
+  await assert.rejects(reuseCanaryImage(sha, {
+    resolve: async () => "mutable:tag",
+    run: () => assert.fail("Invalid digest accepted"),
+  }));
+});
+
+test("canary reuse gates both native builds and still runs convergent promotion", () => {
+  const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
+  const reuse = workflow.split("  reuse_canary_image:\n")[1].split("  build-and-push:\n")[0];
+  assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/);
+  assert.match(reuse, /node scripts\/reuse-canary-image.mjs/);
+  const build = workflow.split("  build-and-push:\n")[1].split("  merge-and-push:\n")[0];
+  assert.match(build, /needs: \[reuse_canary_image\]/);
+  assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/);
+  assert.match(build, /platform: linux\/amd64/);
+  assert.match(build, /platform: linux\/arm64/);
+  const promotion = workflow.split("  promote_canary_channel:\n")[1];
+  assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/);
+});

scripts/standard-image-contract.mjs

@@ -21,6 +21,7 @@ export function standardImageDigest(bytes) {
 /** Read registry bytes directly; CLI stdout formatting is not artifact content. */
 export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
   sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
+  missing = "error",
 } = {}) {
   assert.match(sha ?? "", /^[a-f0-9]{40}$/);
   const request = async (url, headers, consume, allowNotFound = false) => {
@@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
       }
       if (!response.ok) {
         await response.body?.cancel();
+        if (allowNotFound && response.status === 404 && missing === "return-null") return null;
         const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404);
         assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`);
         const retryAfter = Number(response.headers.get("retry-after"));
@@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
   };
   const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json());
   assert.ok(typeof token === "string" && token, "Missing public pull token");
-  const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, {
+  const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, {
     authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json",
   }, async response => {
     assert.ok(response.body, "Standard image index has no body");
@@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
     } finally { await reader.cancel(); }
     return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") };
   }, true);
+  if (manifest === null) return null;
+  const { raw, expected } = manifest;
   const digest = standardImageDigest(raw);
   assert.equal(digest, expected, "Registry digest does not match the index bytes");
   return digest;

scripts/standard-image-contract.test.mjs

@@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a
     assert.equal(calls, expectedCalls);
   }
 });
+
+test("optional lookup falls back only for a missing manifest", async () => {
+  for (const status of [401, 403, 404, 503]) {
+    let calls = 0;
+    const lookup = resolveStandardImageDigest("a".repeat(40), async url => {
+      if (url.includes("/token?")) return Response.json({ token: "public-fixture" });
+      calls++;
+      return new Response(null, { status });
+    }, { missing: "return-null", sleep: async () => {} });
+    if (status === 404) assert.equal(await lookup, null);
+    else await assert.rejects(lookup, /Registry lookup failed/);
+    assert.equal(calls, status === 503 ? 5 : 1);
+  }
+});

server/src/__tests__/budgets-history.test.ts

@@ -0,0 +1,109 @@
+import { randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
+import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
+import { budgetService } from "../services/budgets.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+
+vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() }));
+const support = await getEmbeddedPostgresTestSupport();
+
+describe.skipIf(!support.supported)("budget history for deleted scopes", () => {
+  let db: ReturnType<typeof createDb>;
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  beforeAll(async () => {
+    database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-");
+    db = createDb(database.connectionString);
+  }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+  afterEach(async () => {
+    await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents);
+    await db.delete(projects); await db.delete(agents); await db.delete(companies);
+  });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
+
+  async function fixture() {
+    const companyId = randomUUID(), agentId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" });
+    const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
+      metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning();
+    await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() });
+    return { companyId, agentId, policy: policy! };
+  }
+
+  it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => {
+    const { companyId, policy } = await fixture(), deletedId = randomUUID();
+    if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" });
+    else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" });
+    const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId,
+      metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning();
+    await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId,
+      metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
+      thresholdType: "hard", amountLimit: 200, amountObserved: 201 });
+    if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId));
+    else await db.delete(projects).where(eq(projects.id, deletedId));
+    const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents);
+
+    const overview = await budgetService(db).overview(companyId);
+    expect(overview.policies).toHaveLength(2);
+    expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 });
+    expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 });
+    expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]);
+    expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore);
+    expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore);
+    expect(await db.select().from(costEvents)).toEqual(costsBefore);
+  });
+  it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => {
+    const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db);
+    await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found");
+    await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found");
+    for (const scopeType of ["agent", "project"] as const) {
+      await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
+      const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId,
+        windowKind: "calendar_month_utc", amount: 100 }).returning();
+      const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning();
+      const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId,
+        metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
+        thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning();
+      const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
+        agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) };
+      await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board"))
+        .rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
+      expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
+        agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before);
+      const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board");
+      expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" });
+      const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!;
+      const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id));
+      expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) });
+      expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" });
+      expect(await db.select().from(budgetPolicies)).toEqual(before.policies);
+      expect(await db.select().from(agents)).toEqual(before.agents);
+      expect(await db.select().from(costEvents)).toEqual(before.costs);
+      expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false);
+    }
+  });
+
+  it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => {
+    const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID();
+    await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
+    await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" });
+    const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning();
+    const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId,
+      metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
+      thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning();
+    const service = budgetService(db);
+    await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found");
+    for (const scopeId of [foreignAgentId, randomUUID()]) {
+      await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id));
+      const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
+        agents: await db.select().from(agents), costs: await db.select().from(costEvents) };
+      for (const action of ["raise_budget_and_resume", "keep_paused"] as const) {
+        await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found");
+        expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
+          agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before);
+      }
+    }
+  });
+
+});

server/src/__tests__/company-portability.test.ts

@@ -4168,6 +4168,7 @@ describe("company portability", () => {
     const insertedRelationValues: Array<Record<string, unknown>> = [];
     const monitorUpdates: Array<Record<string, unknown>> = [];
     const db = {
+      transaction: async (write: (tx: unknown) => Promise<unknown>) => write(db),
       insert: () => ({
         values: (rows: Array<Record<string, unknown>>) => ({
           onConflictDoNothing: async () => {

server/src/__tests__/deferred-execution-receipt-work.test.ts

@@ -0,0 +1,427 @@
+import { setImmediate as immediate } from "node:timers/promises";
+import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
+import { accessService } from "../services/access.js";
+import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
+import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
+import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
+import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
+import { issueTreeControlService } from "../services/issue-tree-control.js";
+import { randomUUID } from "node:crypto";
+import { and, eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
+import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
+import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
+import { heartbeatService } from "../services/heartbeat.js";
+import { requestTaskRetry } from "../services/task-retry.js";
+import { issueService } from "../services/issues.js";
+import { idleWorkSnapshot } from "../services/task-admission.js";
+
+vi.setConfig({ testTimeout: 90_000 });
+describe("deferred receipt durable dispatch", () => {
+  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
+  let db: ReturnType<typeof createDb>;
+  let heartbeat: ReturnType<typeof heartbeatService>;
+  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
+  let drain = false;
+  const errors: unknown[] = [];
+  const passes = new Set<Promise<unknown>>();
+  beforeAll(async () => {
+    const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
+    if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+  });
+  beforeEach(async () => {
+    await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
+    heartbeat = heartbeatService(db);
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
+      onError: error => errors.push(error) });
+    const register = coordinator.register.bind(coordinator);
+    coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
+      const pass = Promise.resolve(task.run(signal)); passes.add(pass);
+      void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
+    } });
+  });
+  afterEach(async () => {
+    await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
+    vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
+  });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
+  async function fixture() {
+    const companyId = randomUUID(), actorId = "deferred-receipt-user";
+    await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
+      createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
+    await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
+      defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
+    await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
+    const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
+      adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
+      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
+    const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
+    const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
+      status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
+      contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
+    return { companyId, agent, issue, source, actorId };
+  }
+  async function quiet() {
+    await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
+    const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
+    const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    await vi.advanceTimersByTimeAsync(20 * 60_000);
+    expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
+  }
+  async function finish(issueId: string) {
+    await issueService(db).update(issueId, { status: "done" });
+    await vi.waitFor(async () => {
+      await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
+      expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
+    }, { timeout: 15_000 });
+    const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    if (saved.length) {
+      const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
+      while (Date.now() <= due) {
+        await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
+        await Promise.allSettled([...passes]); await immediate();
+      }
+    }
+    await vi.waitFor(async () => {
+      await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
+      await heartbeat.drainActiveRunExecutions();
+      expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
+    });
+  }
+  it("is quiet after empty reconstruction and terminal receipt history", async () => {
+    const f = await fixture();
+    await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
+  });
+  it("dispatches an ordinary committed task Retry into saved process output", async () => {
+    const f = await fixture();
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
+    const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
+      observedRunId: f.source.id, idempotencyKey: randomUUID() });
+    expect(receipt.accepted).toBe(true);
+    let runId: string | undefined;
+    await vi.waitFor(async () => {
+      const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
+      expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
+    }, { timeout: 15_000 });
+    await vi.waitFor(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
+      expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
+      expect(run.stdoutExcerpt).toContain("receipt-proof");
+    }, { timeout: 15_000 });
+    await finish(f.issue.id);
+    const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
+    expect(linked).toHaveLength(2);
+    expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
+    expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
+    expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
+    expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
+    await quiet();
+  });
+  it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
+    const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
+      payload: native ? {} : { executionWait: { reason: "recovery" } },
+      requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
+      await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
+    });
+    await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
+    expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
+    await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
+    await vi.advanceTimersByTimeAsync(1);
+    await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
+    await quiet();
+  });
+  it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
+    const f = await fixture(); drain = true;
+    const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
+    await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    await db.transaction(async tx => {
+      await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
+      await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
+        status: "deferred_issue_execution", reason: "task_retry" });
+    });
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
+    await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
+    drain = false; await vi.advanceTimersByTimeAsync(30_000);
+    await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
+  });
+  it.each([
+    { name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
+    { name: "interaction", payload: { mutation: "interaction" } },
+    { name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
+    { name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
+  ])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
+    const f = await fixture(), foreign = await fixture();
+    const resume = vi.fn(async () => {});
+    await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
+      source: "on_demand", status: "deferred_issue_execution", payload }).returning();
+    let requestId: string;
+    await db.transaction(async tx => {
+      await tx.transaction(async nested => {
+        await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
+        const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+          source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
+        requestId = row.id;
+      });
+      expect(resume).not.toHaveBeenCalled();
+    });
+    await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
+    expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
+    await quiet();
+  });
+  it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
+    const f = await fixture();
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "on_demand", status: "deferred_issue_execution" }).returning();
+    await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
+      .where(eq(heartbeatRuns.id, f.source.id));
+    const resume = vi.fn(async () => {});
+    await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
+    await quiet();
+  });
+  it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
+    const f = await fixture();
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
+    const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
+      uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
+    const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
+      provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
+    const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
+    await db.insert(agentWakeupRequests).values([
+      { id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
+        status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
+          taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
+            actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
+      { id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
+        payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
+    ]);
+    await db.transaction(async tx => {
+      await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
+      await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
+        providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
+        payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
+    });
+    await vi.waitFor(async () => {
+      const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
+      expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
+    }, { timeout: 5_000 });
+    expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
+    await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
+    await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
+      .toBe("issued"), { timeout: 5_000 });
+    await quiet();
+  });
+  it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
+    const f = await fixture(), initialAt = new Date();
+    const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
+      companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
+    }))).returning();
+    const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
+      _paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
+    await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
+      settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
+    await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
+    const movedAt = new Date();
+    const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
+      deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
+    await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
+      actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
+      async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
+        ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
+    await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
+    await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
+    expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
+    expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
+    await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
+    await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
+    await quiet();
+  });
+  it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
+    const f = await fixture(), tree = issueTreeControlService(db);
+    const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
+    const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
+    const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
+      observedRunId: f.source.id, idempotencyKey: randomUUID() });
+    expect(request.requestId).toBeTruthy();
+    await vi.waitFor(async () => {
+      const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
+      expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
+      expect(wake.runId).toBeNull();
+    }, { timeout: 5_000 });
+    await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
+    await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
+    let runId: string | null = null;
+    await vi.waitFor(async () => {
+      const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
+      expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
+    }, { timeout: 40_000 });
+    await vi.waitFor(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
+      expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
+      expect(run.stdoutExcerpt).toContain("receipt-proof");
+    }, { timeout: 15_000 });
+    expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
+    await finish(f.issue.id); await quiet();
+  });
+  it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
+    const f = await fixture(), initialAt = new Date();
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
+      requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    const dispatchNative = vi.fn(async () => {});
+    await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
+      dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
+    await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
+    const movedAt = new Date();
+    await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
+    await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
+    await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
+    expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
+    await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
+    await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
+  });
+  it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
+    const f = await fixture(); let reason = "operator_pause";
+    const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
+      await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
+    });
+    await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
+      source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    for (const nextReason of ["operator_pause", "budget_pause"]) {
+      reason = nextReason;
+      await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
+      const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
+      const due = current.updatedAt.getTime() + 30_000;
+      await vi.advanceTimersByTimeAsync(1_000);
+      await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
+      const checked = resume.mock.calls.length;
+      await vi.advanceTimersByTimeAsync(due - Date.now());
+      await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
+      await quiet();
+    }
+    expect(resume).toHaveBeenCalledTimes(2);
+  });
+  it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
+    const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
+    const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
+      observedRunId: f.source.id, idempotencyKey: randomUUID() });
+    const tree = issueTreeControlService(db);
+    const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
+    await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
+    const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
+    const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
+    await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
+    await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
+    await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
+    await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
+    expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
+    await commitEligibility();
+    let runId: string | null = null;
+    await vi.waitFor(async () => {
+      const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
+      expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
+    }, { timeout: 15_000 });
+    await vi.waitFor(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
+      expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
+      expect(run.stdoutExcerpt).toContain("receipt-proof");
+    }, { timeout: 15_000 });
+    expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
+    await finish(f.issue.id); await quiet();
+  });
+  it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
+    const f = await fixture(), departing = `departing-${randomUUID()}`;
+    const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
+      observedRunId: f.source.id, idempotencyKey: randomUUID() });
+    await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
+      createdAt: new Date(), updatedAt: new Date() });
+    const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
+      principalType: "user", status: "active", membershipRole: "member" }).returning();
+    await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
+    const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
+    expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
+    const deferred = vi.fn(), native = vi.fn();
+    const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
+      subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
+    try {
+      await expect(db.transaction(async tx => {
+        await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
+          { reassignment: { assigneeAgentId: f.agent.id } });
+        expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
+        throw new Error("roll back member archive");
+      })).rejects.toThrow("roll back member archive");
+      expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
+      expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
+      await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
+      await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
+      deferred.mockClear(); native.mockClear();
+      await db.transaction(async tx => {
+        await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
+          { reassignment: { assigneeAgentId: f.agent.id } });
+        expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
+      });
+      expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
+      expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
+        assigneeUserId: null, assigneeAgentId: f.agent.id,
+      });
+      expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
+      expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
+      expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
+    } finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
+    await quiet();
+  });
+  it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
+    const f = await fixture(); const dispatchNative = vi.fn(async () => {});
+    await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
+      schedulingSuppressed: async () => false }, () => !drain).ready;
+    await expect(db.transaction(async tx => {
+      await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
+      throw new Error("outer rollback");
+    })).rejects.toThrow("outer rollback");
+    await db.transaction(async tx => {
+      await expect(tx.transaction(async nested => {
+        await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
+      })).rejects.toThrow("nested rollback");
+    });
+    await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
+  });
+});

server/src/__tests__/heartbeat-execution-configuration.test.ts

@@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js";
 import { instanceSettingsService } from "../services/instance-settings.js";
 import { aiConnectionService } from "../services/ai-connections.js";
 import * as aiRuntime from "../services/ai-connection-runtime.js";
+import * as connectorRuntime from "../services/connector-runtime.js";
+import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
 import * as sentry from "../sentry.js";
 import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
 import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
 
+function deferred<T>() {
+  let resolve!: (value: T) => void;
+  const promise = new Promise<T>(complete => { resolve = complete; });
+  return { promise, resolve };
+}
+
 const skillFailures = vi.hoisted(() => new Map<string, string>());
+const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
 
 describe("Runner home mode during recovery preparation", () => {
   it.each([
@@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => {
     return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,
       options: Parameters<typeof service.withRuntimeSkillEntries>[1],
       consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {
+      await skillWaits.get(companyId)?.();
       const failure = skillFailures.get(companyId);
       if (failure) throw new Error(failure);
       return service.withRuntimeSkillEntries(companyId, options, consume);
@@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
       testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
     });
   }, 60_000);
-  afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
+  afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); });
   afterAll(async () => {
     await heartbeat?.drainActiveRunExecutions();
     unregisterServerAdapter(adapterType);
@@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
     const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
     expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } });
   }, 60_000);
+  it("overlaps skills with assignments and dependent connection instructions before composition", async () => {
+    const f = await seed();
+    const skillGate = deferred<void>();
+    skillWaits.set(f.companyId, () => skillGate.promise);
+    const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments");
+    const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun");
+    const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" });
+    const opening = configured.prepare();
+    try {
+      await vi.waitFor(() => {
+        expect(assignments).toHaveBeenCalledOnce();
+        expect(instructions).toHaveBeenCalledOnce();
+      });
+    } finally {
+      skillGate.resolve();
+    }
+    expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } });
+  });
+
+  it("settles concurrent configuration reads before exposing the first preparation failure", async () => {
+    const f = await seed();
+    const assignmentGate = deferred<Awaited<ReturnType<typeof connectorRuntime.resolveConnectorAssignments>>>();
+    const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise);
+    const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure"));
+    skillFailures.set(f.companyId, "first skill failure");
+    const configured = await configuration(f, { provider: "codex", homeMode: "isolated" });
+    let settled = false;
+    const opening = configured.prepare();
+    void opening.then(() => { settled = true; }, () => { settled = true; });
+    try {
+      await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce());
+      await Promise.resolve();
+      expect(settled).toBe(false);
+      expect(instructions).not.toHaveBeenCalled();
+    } finally {
+      assignmentGate.resolve([]);
+    }
+    await expect(opening).rejects.toThrow("first skill failure");
+  });
 
   it("does not install new environment declarations when recovering a saved skill selection", async () => {
     const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`;
@@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
       const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12);
       const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_")
         && span.attributes["paperclip.task.run.run_id"] === runIdHash);
-      expect(steps.map(span => span.name)).toEqual([
+      expect(steps.map(span => span.name).sort()).toEqual([
         "configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime",
         "configuration_skills", "configuration_connector_assignments", "configuration_connector_skills",
         "configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity",
-      ].map(step => `heartbeat.prepare.${step}`));
+      ].map(step => `heartbeat.prepare.${step}`).sort());
       for (const span of steps) {
         expect(span.status).toEqual({ code: 0 });
         expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
@@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
         && span.attributes["paperclip.task.run.run_id"] === runIdHash);
       expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0);
       expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 });
-      expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false);
+      expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true);
       for (const span of steps) {
         expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
         expect(span.events).toEqual([]);

server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts

@@ -0,0 +1,127 @@
+import { idleWorkSnapshot } from "../services/task-admission.js";
+import { PgDialect } from "drizzle-orm/pg-core";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
+import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js";
+
+const cleanup: Array<() => Promise<void>> = [];
+afterEach(async () => {
+  for (const stop of cleanup.splice(0)) await stop();
+  vi.useRealTimers();
+});
+function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) {
+  vi.useFakeTimers();
+  vi.setSystemTime(1_000_000);
+  const dialect = new PgDialect();
+  const execute = vi.fn(async (query: any) => {
+    if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
+    throw new Error("unexpected database query");
+  });
+  const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise<any>) => Promise<any> } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>): Promise<any> {
+    return callback({ execute, transaction: db.transaction });
+  } });
+  const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false,
+    draining: input.draining ?? false, standby: false, fail: false };
+  const promote = vi.fn(async () => {
+    if (state.fail) throw new Error("database unavailable");
+    if (state.due !== null && state.due <= Date.now()) state.due = null;
+  });
+  const next = vi.fn(async () => state.due);
+  const dispatch = vi.fn(async (_scope?: { companyId: string }) => {});
+  const coordinator = createDeliveryWorkCoordinator({ owner: db,
+    canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() });
+  const worker = registerRunDispatchWork(coordinator, {
+    promoteDueScheduledRetries: promote, nextScheduledRetryAt: next,
+    resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed,
+  }, () => !state.draining && !state.standby);
+  cleanup.push(() => coordinator.stop());
+  return { db, state, promote, next, dispatch, coordinator, worker,
+    notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)),
+    settle: () => vi.advanceTimersByTimeAsync(1_000) };
+}
+
+describe("run dispatch obligations", () => {
+  it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => {
+    const f = fixture({ suppressed }); await f.worker.ready;
+    const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length];
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+    await vi.advanceTimersByTimeAsync(20 * 60_000);
+    expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls);
+    expect(f.db.execute).not.toHaveBeenCalled();
+  });
+  it("restores an exact deadline and does no earlier recovery work", async () => {
+    const f = fixture({ due: 1_010_000 }); await f.worker.ready;
+    expect(f.coordinator.nextWakeAt()).toBe(1_010_000);
+    await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce();
+    await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2);
+    expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull();
+    expect(f.dispatch).toHaveBeenCalledOnce();
+  });
+  it("rearms earlier work and cancels a future deadline on a committed producer change", async () => {
+    const f = fixture({ due: 1_600_000 }); await f.worker.ready;
+    f.state.due = 1_005_000; await f.notify(); await f.settle();
+    expect(f.coordinator.nextWakeAt()).toBe(1_005_000);
+    f.state.due = null; await f.notify(); await f.settle();
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+    const before = f.next.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before);
+  });
+  it("batches repeated tenant writes and passes only their company to queued dispatch", async () => {
+    const f = fixture(); await f.worker.ready;
+    for (let index = 0; index < 100; index++) {
+      const source = { companyId: "company-a", issueId: `issue-${index % 2}`,
+        runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" };
+      await f.notify(source);
+      await vi.advanceTimersByTimeAsync(10);
+    }
+    expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2);
+    expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2);
+    await f.settle();
+    expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" });
+    for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" });
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+    const count = f.dispatch.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count);
+  });
+  it("remembers the first producer arriving during drain without owner SQL", async () => {
+    const f = fixture(); await f.worker.ready; f.state.draining = true;
+    await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000);
+    expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce();
+    expect(idleWorkSnapshot().active).toBe(0);
+    f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("releases a failed producer hold during drain and retains recovery for resume", async () => {
+    const f = fixture(); await f.worker.ready;
+    f.state.fail = true; await f.notify(); await f.settle();
+    expect(f.promote).toHaveBeenCalledTimes(2);
+    f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000);
+    expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce();
+    expect(idleWorkSnapshot().active).toBe(0);
+    f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("preserves a notification during a nonoverlapping pass", async () => {
+    const f = fixture(); await f.worker.ready;
+    let release!: () => void;
+    const held = new Promise<void>(resolve => { release = resolve; });
+    f.promote.mockImplementationOnce(() => held);
+    await f.notify(); await f.settle();
+    await f.notify(); await f.settle();
+    expect(f.promote).toHaveBeenCalledTimes(2);
+    release(); await f.settle();
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => {
+    const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready;
+    await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled();
+    f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull();
+    f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop();
+    await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2);
+  });
+});

server/src/__tests__/heartbeat-run-dispatch-work.test.ts

@@ -0,0 +1,238 @@
+import { idleWorkSnapshot } from "../services/task-admission.js";
+import { randomUUID } from "node:crypto";
+import { and, eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
+import { agentService } from "../services/agents.js";
+import { budgetService } from "../services/budgets.js";
+import { heartbeatService } from "../services/heartbeat.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
+import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
+
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL");
+const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
+
+describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => {
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
+  let db: ReturnType<typeof createDb>;
+  let heartbeat: ReturnType<typeof heartbeatService>;
+  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
+  let worker: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]> | undefined;
+  let reads: ReturnType<typeof vi.spyOn>;
+  let draining = false;
+  const errors: unknown[] = [];
+  beforeAll(async () => {
+    if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-");
+    db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+  }, 90_000);
+  beforeEach(async () => {
+    await db.execute(sql`TRUNCATE companies CASCADE`);
+    heartbeat = heartbeatService(db); draining = false; errors.length = 0;
+    reads = vi.spyOn(db, "select");
+  });
+  afterEach(async () => {
+    await worker?.stop(); await coordinator?.stop();
+    await heartbeat.drainActiveRunExecutions();
+    worker = undefined; coordinator = undefined;
+    vi.useRealTimers(); vi.restoreAllMocks();
+    expect(errors).toEqual([]);
+  });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
+
+  async function start() {
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
+      canReconcile: () => true, onError: error => errors.push(error) });
+    worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
+    await worker.ready;
+  }
+  async function fixture() {
+    const companyId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`,
+      requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" });
+    const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle",
+      adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() },
+      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
+    }).returning();
+    const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed",
+      responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(),
+      resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {},
+    }).returning();
+    return { companyId, agentId: agent.id, runId: run.id };
+  }
+  async function quiet() {
+    await heartbeat.drainActiveRunExecutions();
+    await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 });
+    const count = reads.mock.calls.length;
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    try {
+      await vi.advanceTimersByTimeAsync(20 * 60_000);
+      expect(reads).toHaveBeenCalledTimes(count);
+    } finally { vi.useRealTimers(); }
+  }
+  async function status(id: string) {
+    return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]);
+  }
+  async function completed(id: string) {
+    await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 });
+    await heartbeat.drainActiveRunExecutions();
+    const completed = await status(id);
+    expect(completed.status).toBe("succeeded");
+    expect(completed.stdoutExcerpt).toContain("dispatch-proof");
+  }
+  it("keeps fresh empty, configured idle and retained terminal history quiet", async () => {
+    await start(); await quiet();
+    await fixture(); await worker!.stop(); await coordinator!.stop();
+    await start(); await quiet();
+  });
+  it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => {
+    const f = await fixture(); await start(); await quiet();
+    const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 });
+    expect(result.outcome).toBe("scheduled");
+    if (result.outcome !== "scheduled") throw new Error("Expected retry creation");
+    await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 });
+    expect((await status(result.run.id)).status).toBe("scheduled_retry");
+    await new Promise(resolve => setTimeout(resolve, 100));
+    expect(Date.now()).toBeLessThan(result.dueAt.getTime());
+    expect((await status(result.run.id)).status).toBe("scheduled_retry");
+    await completed(result.run.id);
+    expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1);
+    await quiet();
+  });
+  it("reconstructs an overdue retry after process restart without a periodic tick", async () => {
+    const f = await fixture();
+    const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 });
+    if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
+    heartbeat = heartbeatService(db);
+    await start(); await completed(scheduled.run.id); await quiet();
+  });
+  it("cancels a future retry and removes its old deadline", async () => {
+    const f = await fixture(); await start();
+    const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 });
+    if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
+    await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 });
+    await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true });
+    await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled");
+  });
+  it("retains the first retry written during drain and catches it up after resume", async () => {
+    const f = await fixture(); await start(); await quiet(); draining = true;
+    const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 });
+    if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
+    await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
+    const count = reads.mock.calls.length;
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    try {
+      await vi.advanceTimersByTimeAsync(90_000);
+      expect(reads).toHaveBeenCalledTimes(count);
+      expect(idleWorkSnapshot().active).toBe(0);
+    } finally { vi.useRealTimers(); }
+    expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
+    draining = false; worker!.wake();
+    await completed(scheduled.run.id); await quiet();
+  });
+  it("uses a committed budget raise to resume a queued run in the company it paused", async () => {
+    const f = await fixture();
+    const budgets = budgetService(db);
+    await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100,
+      metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
+    const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId,
+      provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0,
+      costCents: 125, occurredAt: new Date() }).returning();
+    await budgets.evaluateCostEvent(cost);
+    expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused");
+    const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+      status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
+    await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued");
+    await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000,
+      metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
+    await completed(queued.id); await quiet();
+  });
+  it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => {
+    const f = await fixture(); await start(); await quiet();
+    await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId));
+    const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+      status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
+    let writing: ReturnType<ReturnType<typeof agentService>["clearError"]> | undefined;
+    try {
+      await db.transaction(async blocker => {
+        await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update");
+        writing = agentService(db).clearError(f.agentId);
+        void writing.catch(() => {});
+        await new Promise(resolve => setTimeout(resolve, 100));
+        expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false);
+        expect((await status(queued.id)).status).toBe("queued");
+        expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
+      });
+    } finally { await writing; }
+    await completed(queued.id); await quiet();
+  });
+  it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => {
+    const f = await fixture();
+    const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 });
+    if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
+    await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() },
+      cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id));
+    await start(); await quiet();
+    expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
+    expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] });
+    await quiet();
+  });
+  it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => {
+    const f = await fixture(); await start(); await quiet();
+    await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId));
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    try {
+      await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status");
+      expect(coordinator!.nextWakeAt()).toBeNull();
+      const count = reads.mock.calls.length;
+      await vi.advanceTimersByTimeAsync(60_000);
+      expect(reads).toHaveBeenCalledTimes(count);
+    } finally { vi.useRealTimers(); }
+    expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
+    await quiet();
+  });
+  it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => {
+    const selected = await fixture(), foreign = await fixture(); await start();
+    const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 });
+    const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 });
+    if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries");
+    const other = heartbeatService(db);
+    const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
+    const competingWorker = other.registerRunDispatchWork(competing, () => true);
+    try {
+      await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions();
+      expect((await status(later.run.id)).status).toBe("scheduled_retry");
+      expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt);
+      expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1);
+      await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true });
+      await quiet();
+    } finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); }
+  });
+  it("holds nested producer work until outer commit and rolls back without a successor", async () => {
+    const f = await fixture(); await start(); await quiet();
+    let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
+    let queuedId = "";
+    const writing = db.transaction(async tx => {
+      await tx.transaction(async nested => {
+        await notifyRunDispatch(nested);
+        const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+          responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning();
+        queuedId = row.id;
+      });
+      await held;
+    });
+    await vi.waitFor(() => expect(queuedId).not.toBe(""));
+    await new Promise(resolve => setTimeout(resolve, 100));
+    expect(await status(queuedId)).toBeUndefined();
+    release(); await writing; await completed(queuedId); await quiet();
+    await expect(db.transaction(async tx => {
+      await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); });
+      throw new Error("rollback outer");
+    })).rejects.toThrow("rollback outer");
+    await quiet();
+    expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
+  });
+});

server/src/__tests__/heartbeat-task-context.test.ts

@@ -1,10 +1,11 @@
 import { createHash, randomUUID } from "node:crypto";
 import { eq } from "drizzle-orm";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
 import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared";
 import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
 import { heartbeatService } from "../services/heartbeat.js";
+import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js";
 import { documentService } from "../services/documents.js";
 import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js";
 import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
@@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s
 describePostgres("heartbeat task context delivered to an adapter", () => {
   const adapterType = "task_context_test";
   const captured = new Map<string, AdapterExecutionContext>();
+  const inspectPreparation = new Map<string, () => void>();
   let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
   let db: ReturnType<typeof createDb>;
   let heartbeat: ReturnType<typeof heartbeatService>;
@@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
       type: adapterType,
       supportsLocalAgentJwt: false,
       execute: async (input) => {
+        inspectPreparation.get(input.agent.id)?.();
         captured.set(input.runId, input);
         return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} };
       },
@@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
     });
   });
 
+  it("reads selected comments once for both wake projections", async () => {
+    const f = await seed();
+    const commentId = randomUUID();
+    await db.insert(issueComments).values({
+      id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user",
+      body: "selected wake canary",
+    });
+    const select = vi.spyOn(db, "select");
+    inspectPreparation.set(f.agentId, () => {
+      const commentReads = select.mock.calls.filter(([projection]) => projection
+        && projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt);
+      expect(commentReads).toHaveLength(1);
+    });
+    try {
+      const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] });
+      expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" });
+      expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] });
+    } finally {
+      inspectPreparation.delete(f.agentId);
+      select.mockRestore();
+    }
+  });
+
+  it("cancels deleted-only queued comments before adapter dispatch", async () => {
+    const f = await seed(), commentId = randomUUID();
+    await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
+      authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() });
+    const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
+      payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } });
+    expect(run).not.toBeNull();
+    await heartbeat.drainActiveRunExecutions();
+    const finished = await heartbeat.getRun(run!.id);
+    expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" });
+    expect(captured.has(run!.id)).toBe(false);
+  });
+
+  it("preserves deleted comment metadata without delivering its body", async () => {
+    const f = await seed();
+    const commentId = randomUUID();
+    await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
+      authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(),
+      presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } });
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+      status: "running", issueId: f.issueId }).returning();
+    const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
+    const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
+    const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
+    await prepareHeartbeatTaskContext(db, {
+      run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
+      policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
+      services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
+    });
+    expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null });
+    expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] });
+    expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary");
+  });
+
+  it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => {
+    const f = await seed();
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+      status: "running", issueId: f.issueId }).returning();
+    const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
+    const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
+    const context: Record<string, unknown> = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } };
+    const pinned = vi.fn(async () => null);
+    const preparing = prepareHeartbeatTaskContext(db, {
+      run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
+      policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
+      services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned },
+    });
+    if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable");
+    else await preparing;
+    expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0);
+    expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) });
+  });
+
+  it("retains the context issue scope when no issue reference is available", async () => {
+    const f = await seed();
+    const otherIssueId = randomUUID(), commentId = randomUUID();
+    await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 });
+    await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId,
+      authorType: "user", authorUserId: "context-user", body: "other issue canary" });
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
+      status: "running", issueId: f.issueId }).returning();
+    const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
+    const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
+    await prepareHeartbeatTaskContext(db, {
+      run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null },
+      policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
+      services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
+    });
+    expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary");
+    expect(context.paperclipWakeComment).toBeUndefined();
+  });
+
   it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => {
     const f = await seed();
     const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId });

server/src/__tests__/historical-cleanup-verification.test.ts

@@ -1,8 +1,9 @@
-import { randomUUID } from "node:crypto";
+import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
 import { eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
 import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
 import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
+import { environmentRuntimeService } from "../services/environment-runtime.js";
 import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
 import { idleWorkSnapshot } from "../services/task-admission.js";
 import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
 (support.supported ? describe : describe.skip)("historical cleanup verification", () => {
   let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
   beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
-  afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
+  afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
   afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
   const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
   const state = async () => (await read()).backgroundWork;
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
       inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
     });
   }
+  it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
+    const f = await seed(adHoc);
+    const { privateKey, publicKey } = generateKeyPairSync("ed25519");
+    for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
+      PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
+      PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
+      PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
+    let absent = false;
+    const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
+      if (!absent) return Response.json({ absent: false });
+      const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
+      expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
+        leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
+      const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
+      const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
+        sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
+      return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
+    });
+    vi.stubGlobal("fetch", fetchImpl);
+    const runtime = environmentRuntimeService(db);
+    const service = historicalCleanupVerificationService(db, {
+      inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
+    });
+    expect(await state()).toBe("present");
+    expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
+    expect(await state()).toBe("present");
+    absent = true;
+    expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
+    expect(await state()).toBe("none");
+    const [current] = await db.select().from(environmentLeases);
+    expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
+    expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
+    expect(await service.sweep()).toMatchObject({ checked: 0 });
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+  });
+  it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
+    await seed(true);
+    for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
+      PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
+      PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
+    const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
+    const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
+      acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
+      inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
+    }] });
+    const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
+    expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
+    expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
+  });
   it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
     const f = await seed(adHoc); expect(await state()).toBe("present");
     const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });

server/src/__tests__/issue-recovery-deadlines.test.ts

@@ -0,0 +1,382 @@
+import { setImmediate as immediate } from "node:timers/promises";
+import { idleWorkSnapshot } from "../services/task-admission.js";
+import { randomUUID } from "node:crypto";
+import { and, eq, sql } from "drizzle-orm";
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
+import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { heartbeatService } from "../services/heartbeat.js";
+import { issueService } from "../services/issues.js";
+import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
+import { registerIssueRecoveryWork } from "../services/recovery/work.js";
+import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
+import { taskWatchdogService } from "../services/task-watchdogs.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+
+vi.setConfig({ testTimeout: 90_000 });
+
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL");
+const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
+describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => {
+  let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
+  let db: ReturnType<typeof createDb>;
+  let heartbeat: ReturnType<typeof heartbeatService>;
+  let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
+  let worker: ReturnType<ReturnType<typeof heartbeatService>["registerIssueStateRecoveryWork"]>;
+  let dispatch: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]>;
+  let cancellation: ReturnType<ReturnType<typeof heartbeatService>["registerScheduledRetryCancellationWork"]>;
+  let draining = false;
+  const tasks = new Map<string, { nextRunAt?: () => number | null; hasPending: () => Promise<boolean> }>();
+  const errors: unknown[] = [];
+  const passes = new Set<Promise<unknown>>();
+  const extraStops: Array<() => Promise<void>> = [];
+  const extraDeadlines: Array<() => number | null> = [];
+  beforeAll(async () => {
+    if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
+    db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
+    await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
+  }, 90_000);
+  beforeEach(async () => {
+    await db.execute(sql`TRUNCATE companies CASCADE`);
+    heartbeat = heartbeatService(db); draining = false; errors.length = 0;
+  });
+  afterEach(async () => {
+    for (const stop of extraStops.splice(0)) await stop();
+    extraDeadlines.length = 0;
+    await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop();
+    await heartbeat.drainActiveRunExecutions();
+    vi.useRealTimers(); vi.restoreAllMocks();
+    expect(errors).toEqual([]);
+  });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
+  async function start() {
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
+      onError: error => errors.push(error) });
+    tasks.clear();
+    const register = coordinator.register.bind(coordinator);
+    coordinator.register = (queue, task) => {
+      tasks.set(queue, task);
+      return register(queue, { ...task, run: signal => {
+        const work = task.run(signal); passes.add(work);
+        void work.then(() => passes.delete(work), () => passes.delete(work));
+        return work;
+      } });
+    };
+    worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining);
+    dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
+    cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining);
+    await Promise.all([worker.ready, dispatch.ready, cancellation.ready]);
+  }
+  async function compete() {
+    const other = heartbeatService(db);
+    const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
+    const work = other.registerIssueStateRecoveryWork(competing, () => true);
+    extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready;
+  }
+  async function fixture() {
+    const companyId = randomUUID();
+    await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
+    await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`,
+      requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" });
+    await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" });
+    const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle",
+      adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() },
+      runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
+    }).returning();
+    return { companyId, agentId: agent.id };
+  }
+  async function settled() {
+    await vi.waitFor(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const wake = coordinator.nextWakeAt();
+      const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))),
+        runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) });
+      expect(wake, detail).toBeNull();
+      expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null));
+      expect(idleWorkSnapshot().active).toBe(0);
+    }, { timeout: 40_000 });
+  }
+  async function quiet() {
+    await settled();
+    expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
+    const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute");
+    const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); }
+    finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); }
+  }
+  async function finish(issueId: string) {
+    await issueService(db).update(issueId, { status: "done" });
+    // Closing one blocker can legitimately start its dependent. Wait for this
+    // task's dispatch and execution; quiet() verifies the whole fixture later.
+    await vi.waitFor(async () => {
+      await Promise.allSettled([...passes]);
+      await heartbeat.drainActiveRunExecutions();
+      expect(await db.select().from(heartbeatRuns).where(and(
+        sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
+        sql`${heartbeatRuns.status} in ('queued', 'running')`,
+      ))).toHaveLength(0);
+    }, { timeout: 15_000 });
+    const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
+      sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    if (saved.length > 0) {
+      const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
+      // Keep PostgreSQL I/O and coordinator settlement ahead of virtual time.
+      // A single leap can otherwise occur before the retry deadline is armed.
+      while (Date.now() <= due) {
+        await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
+        await Promise.allSettled([...passes]);
+        await immediate();
+      }
+      await vi.waitFor(async () => {
+        await vi.advanceTimersByTimeAsync(500);
+        await Promise.allSettled([...passes]);
+        await heartbeat.drainActiveRunExecutions();
+        const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
+            sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
+        expect(remaining).toHaveLength(0);
+      });
+      for (const retry of saved) {
+        const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id));
+        expect(settledRetry.status).toBe("cancelled");
+      }
+    }
+  }
+  async function completed(issueId: string) {
+    let runId: string | undefined;
+    await vi.waitFor(async () => {
+      const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
+      const run = runs.find(row => row.status === "running" || row.status === "succeeded");
+      expect(run).toBeDefined(); runId = run!.id;
+    }, { timeout: 15_000 });
+    await vi.waitFor(async () => {
+      await heartbeat.drainActiveRunExecutions();
+      const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
+      expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded");
+    }, { timeout: 15_000 });
+    const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
+    expect(run.stdoutExcerpt).toContain("recovery-proof");
+    return run;
+  }
+  it("keeps empty, configured idle, and terminal history quiet", async () => {
+    await fixture(); await start(); await quiet();
+  });
+  it("creates a blocking child through the ordinary root service transaction", async () => {
+    const f = await fixture();
+    const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" });
+    const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true });
+    expect(child.parentBlockerAdded).toBe(true);
+    expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId),
+      eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
+  });
+  it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => {
+    const f = await fixture();
+    if (mode === "committed") { await start(); await quiet(); }
+    const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId });
+    const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] });
+    const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" });
+    const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id,
+      kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
+      fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
+    if (mode === "startup") await start();
+    await vi.waitFor(async () => {
+      expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId);
+      expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
+    }, { timeout: 40_000 });
+    await finish(blocker.id); await finish(dependent.id); await quiet();
+  });
+  it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => {
+    const f = await fixture(), foreign = await fixture();
+    await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId));
+    await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId,
+      title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId })));
+    const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning();
+    const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id,
+      kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
+      fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
+    const scans = { blockers: 0, actions: 0, loads: 0 };
+    const originalSelect = db.select.bind(db);
+    vi.spyOn(db, "select").mockImplementation(((fields?: Record<string, unknown>) => {
+      const query = originalSelect(fields as never); const from = query.from.bind(query);
+      query.from = ((table: unknown) => {
+        if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++;
+        if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++;
+        if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++;
+        return from(table as never);
+      }) as typeof query.from;
+      return query;
+    }) as typeof db.select);
+    await start(); await settled();
+    expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8);
+    expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
+    const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning();
+    const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id,
+      kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning();
+    const before = { ...scans };
+    for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
+    await new Promise(resolve => setTimeout(resolve, 100));
+    expect(scans.loads).toBe(before.loads);
+    await settled();
+    expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers);
+    expect(scans.loads).toBeLessThanOrEqual(before.loads + 4);
+    expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active");
+    expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet();
+  });
+  it("limits an ordinary comment to its issue among one thousand open assignments", async () => {
+    const f = await fixture();
+    const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({
+      companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId,
+    }))).returning({ id: issues.id });
+    const checked: string[] = [];
+    vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
+    const register = coordinator.register.bind(coordinator);
+    coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
+      const pass = task.run(signal); passes.add(pass);
+      void pass.then(() => passes.delete(pass), () => passes.delete(pass));
+      return pass;
+    } });
+    // Observe the actual paged reconstruction boundary without executing one
+    // thousand adapters. The producer, transaction, coordinator and SQL are real;
+    // the other acceptance cases exercise the resulting recovery decisions.
+    worker = registerIssueRecoveryWork(db, coordinator, {
+      stranded: async scope => { checked.push(scope.issueId); },
+      orphanBlockers: async () => {}, recoveryActions: async () => {},
+      dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {},
+      watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false,
+    }, () => true);
+    await worker.ready;
+    async function settleWork() {
+      for (let step = 0; step < 120; step++) {
+        await vi.advanceTimersByTimeAsync(1000);
+        await Promise.allSettled([...passes]); await immediate();
+        if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return;
+      }
+      throw new Error("Recovery did not settle its committed work within two virtual minutes");
+    }
+    await settleWork();
+    expect(new Set(checked).size).toBe(1000);
+    checked.length = 0;
+    await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" });
+    await settleWork();
+    expect(checked).toEqual([rows[0].id]);
+    // Company eligibility changes still reconstruct all assignments. Narrowing
+    // comment hints must not hide unrelated work after an operator resumes it.
+    checked.length = 0;
+    await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
+    await settleWork();
+    expect(new Set(checked).size).toBe(1000);
+    await quiet();
+  });
+  it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => {
+    const f = await fixture(); await start(); await quiet();
+    const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId });
+    await completed(issue.id);
+    await finish(issue.id); await quiet();
+  }, 90_000);
+  it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => {
+    const f = await fixture();
+    const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" });
+    const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId,
+      blockedByIssueIds: [blocker.id] });
+    await start(); await quiet();
+    await compete();
+    await issueService(db).update(blocker.id, { status: "done" });
+    const run = await completed(dependent.id);
+    expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved");
+    await finish(dependent.id); await quiet();
+    expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
+      sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1);
+  }, 90_000);
+  it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => {
+    const f = await fixture();
+    const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" });
+    await start(); await quiet();
+    await compete();
+    const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
+    let reviewId: string | undefined;
+    await vi.waitFor(async () => {
+      const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id));
+      expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!;
+    }, { timeout: 15_000 });
+    await completed(reviewId!);
+    expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
+      sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1);
+    await finish(reviewId!);
+    await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
+  }, 90_000);
+  it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => {
+    const selected = await fixture(); await start(); await quiet();
+    const foreign = await fixture();
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId,
+      status: "failed", finishedAt: new Date() }).returning();
+    const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done",
+      checkoutRunId: run.id }).returning();
+    await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet();
+    expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id);
+    await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop();
+    heartbeat = heartbeatService(db); await start(); await quiet();
+    expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull();
+  });
+  it("fences nested assignment writes until outer commit and ignores rollback", async () => {
+    const f = await fixture(); await start(); await quiet();
+    let committedId = "";
+    await db.transaction(async outer => {
+      await outer.transaction(async nested => {
+        const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested);
+        committedId = issue.id;
+      });
+      await new Promise(resolve => setTimeout(resolve, 100));
+      expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
+    });
+    await completed(committedId); await finish(committedId); await settled();
+    await expect(db.transaction(async outer => {
+      await outer.transaction(async nested => {
+        await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested);
+      });
+      throw new Error("rollback control");
+    })).rejects.toThrow("rollback control");
+    await quiet();
+    expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0);
+  }, 90_000);
+  it("retains the first committed assignment during drain and executes on resume", async () => {
+    const f = await fixture(); await start(); await quiet(); draining = true;
+    const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId });
+    await new Promise(resolve => setTimeout(resolve, 100));
+    const reads = vi.spyOn(db, "select");
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
+    try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); }
+    finally { vi.useRealTimers(); reads.mockRestore(); }
+    expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
+    draining = false; worker.wake(); dispatch.wake();
+    await completed(issue.id); await finish(issue.id); await quiet();
+  }, 90_000);
+  it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => {
+    const f = await fixture();
+    const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" });
+    await start(); await quiet();
+    await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
+    await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 });
+    expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0);
+    await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
+  });
+  it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => {
+    const f = await fixture();
+    const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" });
+    const lease = new Date(Date.now() + 5_000);
+    const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running",
+      runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease,
+      contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning();
+    await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id));
+    await start();
+    await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 });
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
+    await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 });
+    expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded");
+    if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
+  }, 90_000);
+});

server/src/__tests__/issue-recovery-work-lifecycle.test.ts

@@ -0,0 +1,149 @@
+import { PgDialect } from "drizzle-orm/pg-core";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db";
+import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
+import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
+import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
+import { registerIssueRecoveryWork } from "../services/recovery/work.js";
+import { idleWorkSnapshot } from "../services/task-admission.js";
+
+const stops: Array<() => Promise<void>> = [];
+afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); });
+function fixture() {
+  vi.useFakeTimers(); vi.setSystemTime(1_000_000);
+  const dialect = new PgDialect();
+  const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> };
+  const execute = vi.fn(async (query: any) => {
+    if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
+    throw new Error("unexpected SQL");
+  });
+  const load = vi.fn((table: unknown, distinct = false) => distinct
+    ? table === issueRecoveryActions ? rows.actions : rows.blockers
+    : table === issueWatchdogs ? rows.watchdogs : rows.issues);
+  const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) });
+  const db: any = installDatabaseWorkSignals({ execute,
+    select: () => query(), selectDistinct: () => query(true),
+    transaction: async (callback: (tx: any) => Promise<unknown>) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }),
+  });
+  const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false };
+  const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => {
+    if (state.fail) throw new Error("database unavailable");
+    if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline);
+  });
+  const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {});
+  const suppression = vi.fn(async () => state.suppressed);
+  const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; },
+    locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; },
+    schedulingSuppressed: suppression };
+  const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby,
+    canReconcile: () => !state.standby, onError: vi.fn() });
+  const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby);
+  stops.push(() => coordinator.stop());
+  return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression,
+    notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })),
+    settle: () => vi.advanceTimersByTimeAsync(1_000) };
+}
+describe("issue recovery obligation lifecycle", () => {
+  it("has no timer, SQL, or owner probe after empty reconstruction", async () => {
+    const f = fixture(); await f.worker.ready;
+    expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(20 * 60_000);
+    expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
+  });
+  it("reads scheduling suppression once per pass instead of once per issue", async () => {
+    const f = fixture();
+    f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" })));
+    await f.worker.ready;
+    expect(f.check).toHaveBeenCalledTimes(75);
+    expect(f.suppression).toHaveBeenCalledTimes(6);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => {
+    const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable"));
+    await f.worker.ready; const reads = f.suppression.mock.calls.length;
+    expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
+    await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads);
+    await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => {
+    const f = fixture(); await f.worker.ready;
+    const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions };
+    await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle();
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+    const loads = f.load.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads);
+  });
+  it("runs orphan blocker and action duties without assignment entries", async () => {
+    const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" });
+    f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready;
+    expect(f.check).not.toHaveBeenCalled();
+    expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
+    expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => {
+    const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
+    f.state.deadline = 1_015_000; await f.worker.ready;
+    await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length;
+    for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); }
+    expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked);
+    await vi.advanceTimersByTimeAsync(13_000);
+    expect(f.state.deadline).toBe(1_015_000);
+    expect(f.coordinator.nextWakeAt()).toBe(1_031_000);
+    await vi.advanceTimersByTimeAsync(16_000);
+    expect(f.load.mock.calls.length).toBeGreaterThan(loaded);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("retains exact grace and lease deadlines without early owner probes", async () => {
+    const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
+    f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000;
+    await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000);
+    const checked = f.check.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked);
+    await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("releases settled producer holds during domain drain while coordinator remains open", async () => {
+    const f = fixture(); await f.worker.ready; f.state.draining = true;
+    f.rows.issues.push({ id: "issue", companyId: "selected" });
+    await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length;
+    await vi.advanceTimersByTimeAsync(90_000);
+    expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
+    expect(idleWorkSnapshot().active).toBe(0);
+    f.state.draining = false; f.worker.wake(); await f.settle();
+    expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("retries concrete failed entries and then disarms without reloading the company", async () => {
+    const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true;
+    await f.worker.ready; const loaded = f.load.mock.calls.length;
+    expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
+    await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length;
+    f.state.fail = false; await vi.advanceTimersByTimeAsync(1);
+    expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("keeps an incomplete watchdog subtree as concrete retry debt", async () => {
+    const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true;
+    await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
+    const loaded = f.load.mock.calls.length; f.state.incomplete = false;
+    await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+  });
+  it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => {
+    const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready;
+    let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
+    f.check.mockImplementationOnce(() => held);
+    await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000);
+    expect(f.coordinator.nextWakeAt()).toBeNull();
+    expect(f.check.mock.calls.length).toBeGreaterThan(3);
+  });
+  it("retains startup work on standby and cancels future deadlines on shutdown", async () => {
+    const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" });
+    await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled();
+    f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle();
+    expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop();
+    await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce();
+  });
+});

server/src/__tests__/issue-tree-control-service.test.ts

@@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
   beforeAll(async () => {
     if (externalUrl) {
       await applyPendingMigrations(externalUrl);
-      db = createDb(externalUrl);
+      db = createDb(externalUrl, { maxConnections: 6 });
       return;
     }
     tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-");

server/src/__tests__/managed-skill-content.test.ts

@@ -1,3 +1,5 @@
+import { createServer } from "node:http";
+import { createS3StorageProvider } from "../storage/s3-provider.js";
 import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js";
 import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js";
 import { eq } from "drizzle-orm";
@@ -77,6 +79,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
       .toMatchObject({ content: "complete root guide" });
   });
 
+  it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => {
+    const companyId = await company();
+    const objects = new Map<string, Buffer>();
+    const server = createServer((request, response) => {
+      void (async () => {
+        const key = new URL(request.url!, "http://fixture").pathname;
+        if (request.method === "PUT") {
+          const chunks: Buffer[] = [];
+          for await (const chunk of request) chunks.push(Buffer.from(chunk));
+          let body = Buffer.concat(chunks);
+          if (request.headers["content-encoding"]?.includes("aws-chunked")) {
+            const decoded: Buffer[] = [];
+            let offset = 0;
+            while (offset < body.length) {
+              const end = body.indexOf("\r\n", offset);
+              const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16);
+              if (size === 0) break;
+              decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
+            }
+            body = Buffer.concat(decoded);
+          }
+          objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return;
+        }
+        const body = objects.get(key);
+        if (!body) { response.writeHead(404).end(); return; }
+        response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' });
+        response.end(request.method === "HEAD" ? undefined : body);
+      })().catch(() => { response.writeHead(500).end(); });
+    });
+    await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
+    const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address");
+    const endpoint = `http://127.0.0.1:${address.port}`;
+    const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" };
+    const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]]));
+    Object.assign(process.env, credentials);
+    vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({
+      bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true,
+    })));
+    const key = "cloud-runtime-config:v1";
+    try {
+      await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" });
+      const run = (phase: string) => promisify(execFile)(process.execPath, [
+        "--import", fileURLToPath(import.meta.resolve("tsx")),
+        fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)),
+        `--phase=${phase}`, `--company=${companyId}`, "--limit=50",
+      ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString,
+        PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture",
+        PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3",
+        PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1",
+        PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" },
+        timeout: 60_000, maxBuffer: 1024 * 1024 });
+      await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration");
+      expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0);
+      await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } });
+      const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!);
+      expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]);
+      const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const);
+      const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
+      expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
+      expect([...objects.entries()]).toEqual(before);
+      const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!;
+      const archive = objects.get(archiveKey)!;
+      const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1;
+      objects.set(archiveKey, corrupted);
+      const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
+      expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]);
+      objects.set(archiveKey, archive);
+      const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!;
+      const manifest = objects.get(manifestKey)!; objects.delete(manifestKey);
+      const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
+      expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]);
+      objects.set(manifestKey, manifest);
+      const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
+      expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
+      expect([...objects.entries()].sort()).toEqual([...before].sort());
+      expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key)))
+        .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]);
+    } finally {
+      await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key));
+      vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))));
+      for (const [key, value] of Object.entries(previousCredentials)) {
+        if (value === undefined) delete process.env[key]; else process.env[key] = value;
+      }
+      server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
+    }
+  }, 180_000);
+
   it.each([
     { storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" },
     { storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3",

server/src/__tests__/server-startup-feedback-export.test.ts

@@ -135,6 +135,9 @@ const {
     })),
     reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
     reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
+    registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
+    registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
+    registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
     registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })),
@@ -953,6 +956,21 @@ describe("startServer feedback export wiring", () => {
     if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true);
   });
 
+  it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => {
+    loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
+    await startServer(); await settleStartupHousekeeping();
+    expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
+    expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
+    expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
+    expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
+    if (enabled) {
+      const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
+      const app = await createAppMock.mock.results.at(-1)!.value;
+      expect(coordinator).toBe(app.locals.deliveryWork);
+      expect(admitted()).toBe(true);
+    }
+  });
+
   it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => {
     loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
     await startServer(); await settleStartupHousekeeping();
@@ -1351,13 +1369,14 @@ describe("startServer feedback export wiring", () => {
       await startServer();
       await settleStartupHousekeeping();
       expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
-      expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
+      expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
       expect(intervalCallback).not.toBeNull();
       intervalCallback?.();
       await new Promise<void>((resolve) => setImmediate(resolve));
-      expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
-      expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
+      expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
+      expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
       expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
+      expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
       expect(retiredDetector).not.toHaveBeenCalled();
     } finally {
       delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;

server/src/index.ts

@@ -109,7 +109,6 @@ import { createApp } from "./app.js";
 import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
 import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
 import { logger } from "./middleware/logger.js";
-import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
 import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
 import {
   StartupRefusalError,
@@ -972,9 +971,6 @@ async function startServerWithDatabaseTeardown(
     }
   };
   const pluginWorkerManager = createPluginWorkerManager();
-  const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter(
-    (details, message) => logger.warn(details, message),
-  );
   const heartbeat = config.heartbeatSchedulerEnabled
     ? heartbeatService(db as any, { pluginWorkerManager })
     : null;
@@ -1777,8 +1773,6 @@ async function startServerWithDatabaseTeardown(
           }
         }
 
-        const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries());
-        await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
         const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
         if (
           recoveredGoalActions.enqueued > 0 ||
@@ -1796,48 +1790,10 @@ async function startServerWithDatabaseTeardown(
             "startup session-goal recovery resumed durable agent goals",
           );
         }
-        const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
-        if (
-          promotion.promoted > 0 ||
-          reconciled.assignmentDispatched > 0 ||
-          reconciled.dispatchRequeued > 0 ||
-          reconciled.continuationRequeued > 0 ||
-          reconciled.successfulRunHandoffEscalated > 0 ||
-          reconciled.successfulRunHandoffRetried > 0 ||
-          reconciled.escalated > 0
-        ) {
-          logger.warn(
-            { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
-            "startup heartbeat recovery changed assigned issue state",
-          );
-        }
-
-        const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes();
-        if (dependencyWakesReconciled.healed > 0) {
-          logger.warn(
-            { ...dependencyWakesReconciled },
-            "startup dependency-wake reconciliation restored task execution paths",
-          );
-        }
-
-        const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs();
-        reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup");
-        if (taskWatchdogsReconciled.triggered > 0) {
-          logger.warn(
-            { ...taskWatchdogsReconciled },
-            "startup task-watchdog reconciliation triggered watchdog work",
-          );
-        }
-
         const scanned = await heartbeat.scanSilentActiveRuns();
         if (scanned.created > 0 || scanned.escalated > 0) {
           logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
         }
-
-        const swept = await heartbeat.sweepStaleIssueLocks();
-        if (swept.cleared > 0) {
-          logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks");
-        }
       })().catch((err) => {
         logger.error({ err }, "startup heartbeat recovery failed");
         throw err;
@@ -1846,6 +1802,12 @@ async function startServerWithDatabaseTeardown(
       await startupHeartbeatRecovery;
     }
 
+    if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
+      () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
+    if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
+      () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
+    if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork,
+      () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
     if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork,
       () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
 
@@ -2023,51 +1985,13 @@ async function startServerWithDatabaseTeardown(
           // Periodically reap orphaned runs (5-min staleness threshold) and make sure
           // persisted queued work is still being driven forward.
           runPeriodicRecovery(() => heartbeat
-            .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
-            .then(() => heartbeat.promoteDueScheduledRetries())
-            .then(async (promotion) => {
-              await heartbeat.resumeQueuedRuns();
-              const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true });
-              if (
-                promotion.promoted > 0 ||
-                reconciled.assignmentDispatched > 0 ||
-                reconciled.dispatchRequeued > 0 ||
-                reconciled.continuationRequeued > 0 ||
-                reconciled.successfulRunHandoffEscalated > 0 ||
-                reconciled.successfulRunHandoffRetried > 0 ||
-                reconciled.escalated > 0
-              ) {
-                logger.warn(
-                  { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
-                  "periodic heartbeat recovery changed assigned issue state",
-                );
-              }
-            })
-            .then(async () => {
-              const reconciled = await heartbeat.reconcileResolvedDependencyWakes();
-              if (reconciled.healed > 0) {
-                logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths");
-              }
-            })
-            .then(async () => {
-              const reconciled = await heartbeat.reconcileTaskWatchdogs();
-              reportIncompleteWatchdogScans(reconciled, "periodic");
-              if (reconciled.triggered > 0) {
-                logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work");
-              }
-            })
+            .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
             .then(async () => {
               const scanned = await heartbeat.scanSilentActiveRuns();
               if (scanned.created > 0 || scanned.escalated > 0) {
                 logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
               }
             })
-            .then(async () => {
-              const swept = await heartbeat.sweepStaleIssueLocks();
-              if (swept.cleared > 0) {
-                logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks");
-              }
-            })
             .catch((err) => {
               logger.error({ err }, "periodic heartbeat recovery failed");
             }));

server/src/modules/agent-lifecycle/adapters/records.ts

@@ -80,22 +80,20 @@ export function agentRecords(db: Db, effects: LifecycleEffects) {
         throw conflict("Only agents in error status can have their error cleared");
       }
 
-      const updated = await db
-        .update(agents)
-        .set({
-          status: "idle",
-          pauseReason: null,
-          pausedAt: null,
-          errorReason: null,
-          updatedAt: new Date(),
-        })
-        .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
-        .returning()
-        .then((rows) => rows[0] ?? null);
+      const updated = await effects.transaction(db, existing.companyId, async tx => {
+        const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState })
+          .from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update");
+        if (current?.status !== "error" || current.lifecycleState !== "ready") {
+          throw conflict("Only agents in error status can have their error cleared");
+        }
+        await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle");
+        const [saved] = await tx.update(agents).set({
+          status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(),
+        }).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning();
+        if (!saved) throw conflict("Only agents in error status can have their error cleared");
+        return saved;
+      });
 
-      if (!updated) {
-        throw conflict("Only agents in error status can have their error cleared");
-      }
       return getById(updated.id);
     },
 

server/src/modules/run-dispatch/adapters/postgres.ts

@@ -1,3 +1,4 @@
+import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
 import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
 import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
 import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
@@ -511,6 +512,15 @@ export function createPostgresRunDispatchAdapter(
     return decideScheduledRetryGate(factsResult.facts, input.now);
   }
 
+  async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null> {
+    const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns)
+      .where(and(eq(heartbeatRuns.status, "scheduled_retry"),
+        sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
+        input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined))
+      .orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1);
+    return row?.at?.getTime() ?? null;
+  }
+
   async function listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]> {
     const rows = await db
       .select()
@@ -518,6 +528,7 @@ export function createPostgresRunDispatchAdapter(
       .where(
         and(
           eq(heartbeatRuns.status, "scheduled_retry"),
+          sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
           lte(heartbeatRuns.scheduledRetryAt, input.now),
           input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined,
         ),
@@ -986,6 +997,7 @@ export function createPostgresRunDispatchAdapter(
         recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : [];
     }));
     if (commentIds.some(id => !recordedCommentIds.has(id))) return null;
+    await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id });
     const [deferred] = await tx.update(agentWakeupRequests).set({
       status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null,
       payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context,
@@ -1414,6 +1426,7 @@ export function createPostgresRunDispatchAdapter(
 
   return {
     evaluateScheduledRetryGate,
+    nextScheduledRetryAt,
     listDueRetries,
     cancelStaleQueuedRun,
     dispatchResolvedInteractionIfCurrent,

server/src/modules/run-dispatch/application/ports.ts

@@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = {
 
 /** Read-only scheduled-retry operations exposed to application use cases. */
 export interface ScheduledRetryReader {
+  nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null>;
   evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise<GateDecision>;
   listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]>;
 }

server/src/modules/run-dispatch/application/use-cases.test.ts

@@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu
   const evaluateCalls: unknown[] = [];
   return {
     evaluateCalls,
+    async nextScheduledRetryAt() { return null; },
     async evaluateScheduledRetryGate(input) {
       evaluateCalls.push(input);
       return { allowed: true };

server/src/modules/run-dispatch/index.ts

@@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) {
   });
 
   return {
+    nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input),
     evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }),
     promoteScheduledRetry,
     promoteDueScheduledRetries: createPromoteDueScheduledRetries({

server/src/modules/wake-queue/adapters/postgres.test.ts

@@ -1,3 +1,4 @@
+import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
 import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
 import { instanceSettingsService } from "../../../services/instance-settings.js";
 import { randomBytes, randomUUID } from "node:crypto";
@@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js";
 // a foreign-company row is invisible to a read, and a deferred-status
 // compare-and-set that affects no row leaves no other trace. The decision
 // branching itself is proven against plain facts in `domain/policy.test.ts`.
-const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL");
+const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
 const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
 
 if (!embeddedPostgresSupport.supported) {
@@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
   };
 
   beforeAll(async () => {
-    tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
-    db = createDb(tempDb.connectionString);
+    if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
+    db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
   }, 20_000);
 
   afterEach(async () => {
@@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
   });
 
   afterAll(async () => {
+    await db?.$client.end({ timeout: 1 });
     await tempDb?.cleanup();
   });
 

server/src/modules/wake-queue/adapters/postgres.ts

@@ -1,4 +1,6 @@
 import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
+import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
+import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
 import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js";
 import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
@@ -639,6 +641,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
         .returning()
         .then((rows) => rows[0]);
 
+      await notifyRunDispatch(tx, { companyId });
       const queuedRun = await tx
         .insert(heartbeatRuns)
         .values({
@@ -717,6 +720,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
         .returning()
         .then((rows) => rows[0]);
 
+      await notifyRunDispatch(tx, { companyId });
       const queuedRun = await tx
         .insert(heartbeatRuns)
         .values({
@@ -1009,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
       )).for("update");
       if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company");
       const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId);
+      const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload));
+      const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id));
+      const newInteraction = readNonEmptyString(input.mergedPayload.interactionId);
+      if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId))
+        await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id });
       const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId);
       const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) &&
         priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
@@ -1080,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
 
     async insertNewDeferredWake(scope, input) {
       const tx = requireAdmissionTx(scope, input.companyId);
+      await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
       await tx.insert(agentWakeupRequests).values({
         ...input.durableReceipt,
         companyId: input.companyId,

server/src/routes/issues.ts

@@ -1,3 +1,4 @@
+import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js";
 import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js";
 import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js";
@@ -16674,6 +16675,7 @@ export function issueRoutes(
         }
         // The click is durable fresh user intent, including when the message
         // predates a failed run's stop. Keep its content and original attribution.
+        await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id });
         await tx.update(agentWakeupRequests).set({
           payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: {
             actorId: actor.actorId, requestedAt: new Date().toISOString(),

server/src/services/access.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -744,6 +745,7 @@ export function accessService(db: Db) {
         sql`${issues.status} not in ('done', 'cancelled')`,
       );
       await notifyNativeStatusReassessment(tx, { companyId });
+      await notifyDeferredExecutionEligibility(tx, { companyId });
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
       await notifyStatusCardIssueChange(tx, companyId);

server/src/services/agent-configuration.ts

@@ -1,3 +1,5 @@
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
@@ -83,6 +85,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
       existing.adapterConfig,
     );
   }
+  await notifyRunDispatch(db, { companyId: existing.companyId });
+  await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
   if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
     await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
   }

server/src/services/agent-conversations.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
 import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
@@ -547,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
       eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
     )).for("update");
     if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
+    await notifyIssueRecovery(tx, { companyId: issue.companyId });
     const released = await tx.update(issueTreeHolds).set({
       status: "released", releasedAt: new Date(), updatedAt: new Date(),
       releasedByActorType: "user", releasedByUserId: comment.authorUserId,

server/src/services/agent-lifecycle.ts

@@ -1,3 +1,5 @@
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
 import { isAgentStatusInvokable } from "@paperclipai/shared";
 import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
@@ -38,7 +40,11 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
     },
     notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
     async beforeStatusChange(tx, companyId, agentId, before, after) {
-      if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
+      if (before !== after) {
+        await notifyRunDispatch(tx, { companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId });
+        await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
+      }
       if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
         await notifyNativeResponsePresentation(tx, { companyId, agentId });
         await notifyNativeReviewResponse(tx, { companyId, agentId });

server/src/services/budgets.ts

@@ -1,4 +1,6 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyApprovalWork } from "./approval-work-notifications.js";
 import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
 import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
@@ -164,6 +166,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
   };
 }
 
+async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
+  let scope: ScopeRecord;
+  try {
+    scope = await resolveScopeRecord(db, scopeType, scopeId);
+  } catch (error) {
+    if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
+    return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
+      paused: false, pauseReason: null, budgetHeld: false };
+  }
+  if (scope.companyId !== companyId) throw notFound("Budget scope not found");
+  return scope;
+}
+
 // Keep a large dashboard from filling the database pool with queued reads.
 async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
   const result: R[] = [];
@@ -288,6 +303,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
     }
 
     if (policy.scopeType === "project") {
+      await notifyRunDispatch(db, { companyId: policy.companyId });
+      await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
       await db
         .update(projects)
         .set({
@@ -301,6 +318,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
 
     const scope = await resolveScopeRecord(db, "company", policy.scopeId);
     if (scope.paused) return;
+    await notifyRunDispatch(db, { companyId: policy.companyId });
+    await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
     await notifyCompanyRetentionWork(db, policy.companyId);
     await notifyStatusCardIssueChange(db, policy.companyId);
     await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
@@ -348,6 +367,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
     }
 
     if (policy.scopeType === "project") {
+      await notifyRunDispatch(db, { companyId: policy.companyId });
+      await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
       await db
         .update(projects)
         .set({
@@ -362,6 +383,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
     await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
     const scope = await resolveScopeRecord(db, "company", policy.scopeId);
     if (scope.pauseReason !== "budget") return;
+    await notifyRunDispatch(db, { companyId: policy.companyId });
+    await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
     await notifyCompanyRetentionWork(db, policy.companyId);
     await notifyStatusCardIssueChange(db, policy.companyId);
     await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
@@ -398,7 +421,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
   }
 
   async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
-    const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
+    const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
     const observed = await computeObservedSpend(db, policy);
     const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
     const { windowStart: start, windowEnd: end } = observed;
@@ -563,7 +586,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
     const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
 
     return mapBudgetReads(rows, async (row) => {
-      const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
+      const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
       return {
         id: row.id,
         companyId: row.companyId,
@@ -928,6 +951,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
       if (incident.companyId !== companyId) throw notFound("Budget incident not found");
 
       const policy = await getPolicyRow(incident.policyId);
+      if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
+        throw notFound("Budget scope not found");
+      }
+      const scope = input.action === "keep_paused"
+        ? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
+        : await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
+      if (scope.companyId !== companyId) throw notFound("Budget scope not found");
       if (input.action === "raise_budget_and_resume") {
         const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
         const observed = await computeObservedSpend(db, policy);

server/src/services/chat-channels/failed-run-retry.ts

@@ -1,3 +1,4 @@
+import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
 import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
 import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
 import { createFailedRetryNotices } from "./failed-retry-notices.js";
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
     // Source authorization above checked every external receipt and its owner.
     // Reset only this certified unstarted batch under the task's admission lock.
     const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
+    await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
     await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
       payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
     }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
     const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
     const settle = async (status: string, result: Record<string, unknown>) => {
       const [current] = await db.transaction(async (tx) => {
+        if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
+          companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
+        });
         await notifyChatActionWork(tx, "failed_run_retry");
         if (status === "processed") await notifyNativeResponsePresentation(tx, {
           companyId: claimed.companyId, issueId: String(claimed.payload.issueId),

server/src/services/chat-channels/task-retry-input.ts

@@ -1,3 +1,4 @@
+import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
 import { z } from "zod";
 import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
 import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
     )).for("update");
     if (receipt?.runId) return;
     await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
+    await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
     await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
       payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
     }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),

server/src/services/chat-channels/telegram/attachment-failure.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "../../status-card-notifications.js";
+import { notifyIssueRecovery } from "../../issue-recovery-notifications.js";
 import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js";
 
 import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js";
@@ -79,6 +80,7 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm
           })(),
       );
     const effect = await db.transaction(async (tx) => {
+      await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId });
       // Replace the optimistic attachment placeholder with an explicit safe
       // Paperclip-visible failure. The message link stays authoritative for
       // exact provider redelivery, while no agent wakeup is enqueued for an

server/src/services/chat-run-milestone-notifications.ts

@@ -1,3 +1,4 @@
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyIssueReviewState } from "./issue-review-state-notifications.js";
 import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
 import { notifyChatResponseAvailability } from "./chat-response-availability.js";
@@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio
 export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string };
 
 export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) {
+  if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope);
   if (scope.issueId || scope.runId) {
     await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId,
       runId: scope.issueId ? undefined : scope.runId });

server/src/services/companies.ts

@@ -1,4 +1,6 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js";
 import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -333,6 +335,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
         const renamedPrefix = await resolveRenamedIssuePrefix(tx, id, companyPatch);
 
         if (companyPatch.status !== undefined && companyPatch.status !== existing.status) {
+          await notifyRunDispatch(tx, { companyId: id });
+          await notifyDeferredExecutionEligibility(tx, { companyId: id });
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);
@@ -490,6 +494,8 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) {
         const wasAlreadyArchived = existing.status === "archived";
 
         if (!wasAlreadyArchived) {
+          await notifyRunDispatch(tx, { companyId: id });
+          await notifyDeferredExecutionEligibility(tx, { companyId: id });
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules);

server/src/services/company-deletion.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
 import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js";
 import { notifyInstructionCollection } from "./instruction-collection-notifications.js";
@@ -72,6 +73,7 @@ export async function deleteCompany(db: Db, id: string) {
         const [existing] = await tx.select({ id: companies.id }).from(companies)
           .where(eq(companies.id, id)).for("no key update");
         if (!existing) return null;
+        await notifyIssueRecovery(tx, { companyId: id });
         await notifyChatCompanyConfiguration(tx, id);
         await chatCompanyDeletion.deleteCompanyData(tx, id);
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync);

server/src/services/company-portability.ts

@@ -1,3 +1,4 @@
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { createAgentLifecycle } from "./agent-lifecycle.js";
 import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared";
 import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
@@ -6465,7 +6466,9 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
           }
           if (relationRows.length > 0) {
             const relationCompanyId = targetCompany.id;
-            await db
+            await db.transaction(async tx => {
+              await notifyIssueRecovery(tx, { companyId: relationCompanyId });
+              await tx
               .insert(issueRelations)
               .values(relationRows.map((row) => ({
                 companyId: relationCompanyId,
@@ -6476,6 +6479,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) {
                 createdByUserId: actorUserId ?? null,
               })))
               .onConflictDoNothing();
+            });
           }
         }
 

server/src/services/deferred-execution-notifications.ts

@@ -0,0 +1,34 @@
+import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
+
+export type DeferredExecutionScope = { companyId: string; requestId?: string };
+export type DeferredReceiptRecoveryOptions = {
+  companyId?: string;
+  requestId?: string;
+  onError?: (error: unknown) => void;
+};
+
+function hint(scope: DeferredExecutionScope) {
+  return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId });
+}
+
+export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope));
+}
+
+export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope));
+}
+
+export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) {
+  await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope));
+}
+
+export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) {
+  if (scope) {
+    await notifyDeferredExecutionWork(tx, scope);
+    await notifyNativeStatusWakeWork(tx, scope);
+  } else {
+    await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*");
+    await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*");
+  }
+}

server/src/services/deferred-execution-wait.ts

@@ -0,0 +1,24 @@
+import { and, eq, sql } from "drizzle-orm";
+import { agentWakeupRequests, type Db } from "@paperclipai/db";
+import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js";
+
+export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string },
+  condition: Record<string, unknown>, options: { replace?: boolean } = {}) {
+  return db.transaction(async tx => {
+    const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests)
+      .where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
+        eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update");
+    if (!current) return;
+    const previous = current.payload?.executionWait;
+    const previousReason = previous && typeof previous === "object" && !Array.isArray(previous)
+      ? (previous as Record<string, unknown>).reason : undefined;
+    if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope);
+    await tx.update(agentWakeupRequests).set({
+      payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
+        ${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`}
+          || ${JSON.stringify(condition)}::jsonb)`,
+      updatedAt: new Date(),
+    }).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId),
+      eq(agentWakeupRequests.status, "deferred_issue_execution")));
+  });
+}

server/src/services/delivery-work-notifications.ts

@@ -2,6 +2,17 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db";
 
 /** Topics for existing durable queues; register intent before their writes. */
 export const DELIVERY_QUEUES = {
+  deferredExecutionReceipts: "deferred-execution-receipts",
+  taskRetryReceiptSettlement: "task-retry-receipt-settlement",
+  nativeStatusWakeIntents: "native-status-wake-intents",
+  orphanBlockers: "orphan-blockers",
+  activeRecoveryActions: "active-recovery-actions",
+  strandedAssignments: "stranded-assignments",
+  resolvedDependencies: "resolved-dependencies",
+  taskWatchdogs: "task-watchdogs",
+  staleIssueLocks: "stale-issue-locks",
+  scheduledRetryDispatch: "scheduled-retry-dispatch",
+  queuedRunDispatch: "queued-run-dispatch",
   statusCardEvaluation: "status-card-evaluation",
   runEvidenceMigration: "run-evidence-migration",
   providerTraceRetention: "provider-trace-retention",

server/src/services/documents.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { randomUUID } from "node:crypto";
 import { and, asc, desc, eq, sql } from "drizzle-orm";
 import type { Db } from "@paperclipai/db";
@@ -277,6 +278,7 @@ export function documentService(db: Db) {
                   .where(eq(issueDocuments.issueId, issue.id));
                 const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key));
 
+                await notifyIssueRecovery(tx, { companyId: issue.companyId });
                 const [document] = await tx
                   .insert(documents)
                   .values({
@@ -378,6 +380,7 @@ export function documentService(db: Db) {
               });
             }
 
+            await notifyIssueRecovery(tx, { companyId: issue.companyId });
             const nextRevisionNumber = existing.latestRevisionNumber + 1;
             const [revision] = await tx
               .insert(documentRevisions)
@@ -440,6 +443,7 @@ export function documentService(db: Db) {
             throw conflict("Document does not exist yet", { key });
           }
 
+          await notifyIssueRecovery(tx, { companyId: issue.companyId });
           const [document] = await tx
             .insert(documents)
             .values({
@@ -594,7 +598,10 @@ export function documentService(db: Db) {
         });
       }
       await db.transaction(async (tx) => {
-        for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId);
+        for (const companyId of new Set(rows.map(row => row.companyId))) {
+          await notifyStatusCardIssueChange(tx, companyId);
+          await notifyIssueRecovery(tx, { companyId });
+        }
         await insertRowsInChunks(tx, documents, documentRows);
         await insertRowsInChunks(tx, documentRevisions, revisionRows);
         await insertRowsInChunks(tx, issueDocuments, linkRows);
@@ -647,6 +654,7 @@ export function documentService(db: Db) {
           });
         }
 
+        await notifyIssueRecovery(tx, { companyId: existing.companyId });
         const now = new Date();
         const nextRevisionNumber = existing.latestRevisionNumber + 1;
         const [restoredRevision] = await tx
@@ -726,6 +734,7 @@ export function documentService(db: Db) {
           };
         }
 
+        await notifyIssueRecovery(tx, { companyId: existing.companyId });
         const now = new Date();
         await tx
           .update(documents)
@@ -773,6 +782,7 @@ export function documentService(db: Db) {
           };
         }
 
+        await notifyIssueRecovery(tx, { companyId: existing.companyId });
         const now = new Date();
         await tx
           .update(documents)
@@ -821,6 +831,7 @@ export function documentService(db: Db) {
           });
         }
 
+        await notifyIssueRecovery(tx, { companyId: existing.companyId });
         await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id));
         await notifyStatusCardIssueChange(tx, existing.companyId);
         await tx.delete(documents).where(eq(documents.id, existing.id));

server/src/services/environment-runtime.ts

@@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s
 import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk";
 import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js";
 import { remoteTerminationReceipt } from "./remote-execution-termination.js";
+import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js";
 import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js";
 import { createHash, randomUUID } from "node:crypto";
 import { and, eq, inArray, ne, or, sql } from "drizzle-orm";
@@ -4021,7 +4022,12 @@ export function environmentRuntimeService(
       const lease = await environmentsSvc.getLeaseById(leaseId);
       if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined;
       const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null;
-      return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease });
+      // Legacy Cloud leases can lack a credential version that permits a
+      // provider-local observation. Cloud's manager verifies the historical
+      // organization and signs only exact absence; no replacement key or
+      // cleanup label is treated as proof. Explicit provider answers win.
+      return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease })
+        ?? await inspectCloudSandboxCleanupAbsence(lease);
     },
 
     async retryPendingSandboxTeardown(input: {

server/src/services/execution-control-notifications.ts

@@ -1,11 +1,17 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import type { Db } from "@paperclipai/db";
 import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js";
 
 export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string };
 
 export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) {
-  if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
+  if (scope) {
+    await notifyStatusCardIssueChange(transaction, scope.companyId);
+    await notifyRunDispatch(transaction, scope);
+    await notifyDeferredExecutionEligibility(transaction, scope);
+  }
   for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization,
     DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) {
     await notifyDeliveryWork(transaction, queue);
@@ -20,6 +26,8 @@ export function writeExecutionControl<T>(
 ): Promise<T> {
   return db.transaction(async transaction => {
     if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId);
+    await notifyRunDispatch(transaction, scope);
+    if (scope) await notifyDeferredExecutionEligibility(transaction, scope);
     await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors);
     for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic);
     return write(transaction as unknown as Db);

server/src/services/external-conversation-settlement-notifications.ts

@@ -1,7 +1,9 @@
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js";
 
-export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
+export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) {
+  await notifyIssueRecovery(tx, scope);
   return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement,
     scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*");
 }

server/src/services/heartbeat.ts

@@ -1,3 +1,7 @@
+import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
+import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
+import { registerIssueRecoveryWork } from "./recovery/work.js";
+import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js";
 import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js";
 import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js";
 import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js";
@@ -999,6 +1003,7 @@ export function heartbeatService(
     finalizeAiConnectionBusyDeferral,
     finalizeWorkspaceBusyDeferral,
     promoteDueScheduledRetries,
+    nextScheduledRetryAt,
     retryScheduledRetryNow,
     scheduleBoundedRetry,
   } = createHeartbeatRetries(db, {
@@ -1333,8 +1338,8 @@ export function heartbeatService(
     return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args);
   }
 
-  function resumeQueuedRunsAfterDrain() {
-    const promise = heartbeatQueue.resumeQueuedRunsAfterDrain();
+  function resumeQueuedRunsAfterDrain(...args: Parameters<typeof heartbeatQueue.resumeQueuedRunsAfterDrain>) {
+    const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args);
     activeWakeupPromises.add(promise);
     void promise.then(
       () => { activeWakeupPromises.delete(promise); },
@@ -1609,15 +1614,15 @@ export function heartbeatService(
     void reportRunFailure(db, updated, failureReport);
   }
 
-  async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) {
+  async function reconcileStrandedAssignedIssues(opts?: Parameters<typeof recovery.reconcileStrandedAssignedIssues>[0]) {
     return recovery.reconcileStrandedAssignedIssues({
-      skipExternalConversationSettlement: opts?.skipExternalConversationSettlement,
+      ...opts,
       issueCreatedAtGte: await getWorktreeExecutionCutoff(),
     });
   }
 
-  async function sweepStaleIssueLocks() {
-    return recovery.sweepStaleIssueLocks();
+  async function sweepStaleIssueLocks(opts?: Parameters<typeof recovery.sweepStaleIssueLocks>[0]) {
+    return recovery.sweepStaleIssueLocks(opts);
   }
 
   function issueIdFromRunContext(contextSnapshot: unknown) {
@@ -1647,10 +1652,7 @@ export function heartbeatService(
     });
   }
 
-  async function reconcileTaskWatchdogs(opts?: {
-    companyId?: string | null;
-    runId?: string | null;
-  }) {
+  async function reconcileTaskWatchdogs(opts?: Parameters<typeof taskWatchdogs.reconcileTaskWatchdogs>[0]) {
     return taskWatchdogs.reconcileTaskWatchdogs({
       ...opts,
       issueCreatedAtGte: await getWorktreeExecutionCutoff(),
@@ -1675,10 +1677,7 @@ export function heartbeatService(
     return recovery.buildRunOutputSilence(run, now);
   }
 
-  async function reconcileResolvedDependencyWakes(opts?: {
-    runId?: string | null;
-    companyId?: string | null;
-  }) {
+  async function reconcileResolvedDependencyWakes(opts?: Parameters<typeof recovery.reconcileResolvedDependencyWakeBackstop>[0]) {
     return recovery.reconcileResolvedDependencyWakeBackstop(opts);
   }
 
@@ -2007,8 +2006,11 @@ export function heartbeatService(
       const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId);
       const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner
         && typeof run.contextSnapshot?.conversationSessionGeneration === "number";
-      if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId))
-        && !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) {
+      const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn
+        ? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId })
+          .from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null)
+        : null;
+      if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) {
         await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" });
         await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() });
         await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true });
@@ -2204,44 +2206,6 @@ export function heartbeatService(
         context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true;
       }
       const wakeCommentId = deriveCommentId(context, null);
-      const wakeCommentContext =
-        issueContext && wakeCommentId
-          ? await db
-              .select({
-                id: issueComments.id,
-                body: issueComments.body,
-                authorType: issueComments.authorType,
-                authorAgentId: issueComments.authorAgentId,
-                authorUserId: issueComments.authorUserId,
-                presentation: issueComments.presentation,
-                metadata: issueComments.metadata,
-                deletedAt: issueComments.deletedAt,
-                deletedByType: issueComments.deletedByType,
-                deletedByAgentId: issueComments.deletedByAgentId,
-                deletedByUserId: issueComments.deletedByUserId,
-                deletedByRunId: issueComments.deletedByRunId,
-                sourceTrust: issueComments.sourceTrust,
-              })
-              .from(issueComments)
-              .where(
-                and(
-                  eq(issueComments.id, wakeCommentId),
-                  eq(issueComments.issueId, issueContext.id),
-                  eq(issueComments.companyId, agent.companyId),
-                ),
-              )
-              .then((rows) => {
-                const row = rows[0] ?? null;
-                return row?.deletedAt
-                  ? {
-                      ...row,
-                      body: "",
-                      presentation: null,
-                      metadata: null,
-                    }
-                  : row;
-              })
-          : null;
       let issueAssigneeOverrides =
         issueContext && issueContext.assigneeAgentId === agent.id
           ? parseIssueAssigneeAdapterOverrides(
@@ -2610,7 +2574,7 @@ export function heartbeatService(
         continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest,
       } = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, {
         run, agent,
-        task: { issueRef, issueContext, wakeCommentContext, context, taskSession },
+        task: { issueRef, issueContext, context, taskSession },
         policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) },
         services: { issuesSvc, getPinnedSkillTestContext },
       }));
@@ -4246,6 +4210,30 @@ export function heartbeatService(
     computeTaskDrain,
     applyTaskDrain,
 
+    registerDeferredReceiptWork: (coordinator: Parameters<typeof registerDeferredReceiptWork>[1], canRun: () => boolean) =>
+      registerDeferredReceiptWork(db, coordinator, {
+        resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope),
+        settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }),
+        dispatchNative: dispatchPendingNativeStatusWakeups,
+        schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
+      }, () => canRun() && readTaskDrain(new Date()) === null),
+    registerIssueStateRecoveryWork: (coordinator: Parameters<typeof registerIssueRecoveryWork>[1], canRun: () => boolean) =>
+      registerIssueRecoveryWork(db, coordinator, {
+        stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }),
+        orphanBlockers: recovery.reconcileUnassignedBlockingIssues,
+        recoveryActions: recovery.reconcileActiveRecoveryActions,
+        dependencies: reconcileResolvedDependencyWakes,
+        locks: sweepStaleIssueLocks,
+        watchdogs: reconcileTaskWatchdogs,
+        schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
+      }, () => canRun() && readTaskDrain(new Date()) === null),
+    registerRunDispatchWork: (coordinator: Parameters<typeof registerRunDispatchWork>[0], canRun: () => boolean) =>
+      registerRunDispatchWork(coordinator, {
+        promoteDueScheduledRetries,
+        nextScheduledRetryAt,
+        resumeQueuedRunsAfterDrain,
+        schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed,
+      }, () => canRun() && readTaskDrain(new Date()) === null),
     promoteDueScheduledRetries,
     retryScheduledRetryNow,
 

server/src/services/heartbeat/deferred-receipt-work.ts

@@ -0,0 +1,118 @@
+import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
+import { agentWakeupRequests, type Db } from "@paperclipai/db";
+import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
+import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
+import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
+import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
+
+type Entry = { id: string; companyId: string; dueAt: number };
+type NativeIntentOptions = DeferredExecutionScope & {
+  onDeferredDeadline(at: number): void;
+  onError(error: unknown): void;
+};
+
+export function registerDeferredReceiptWork(
+  db: Db,
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  owner: {
+    resume(scope: DeferredReceiptRecoveryOptions): Promise<unknown>;
+    settleTaskChat(scope: DeferredExecutionScope): Promise<unknown>;
+    dispatchNative(scope: NativeIntentOptions): Promise<unknown>;
+    schedulingSuppressed(): Promise<boolean>;
+  },
+  canRun: () => boolean,
+) {
+  function register(kind: "deferred" | "task_chat" | "native") {
+    const cursors = new Map<string, string>();
+    let enabled = false;
+    const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts
+      : kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents;
+    return registerScopedDeadlineWork<Entry, DeferredExecutionScope>(coordinator, {
+      queue,
+      reconstructionIntervalMs: 30_000,
+      beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
+      decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
+      matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
+      load: async scope => {
+        const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
+        if (!enabled) { cursors.delete(key); return []; }
+        const cursor = cursors.get(key);
+        const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId,
+          status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt,
+          claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
+          delayed: sql<boolean>`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null
+            or ${agentWakeupRequests.payload}->'executionWait' is not null`,
+          interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
+            where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
+              and r.runtime_mode = 'legacy' and r.status = 'cancelled'
+              and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
+        }).from(agentWakeupRequests).where(and(
+          scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+          scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
+          cursor ? gt(agentWakeupRequests.id, cursor) : undefined,
+          kind === "native" ? and(
+            eq(agentWakeupRequests.requestedByActorType, "system"),
+            eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
+            inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId),
+          ) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
+            kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"),
+              sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or(
+              eq(agentWakeupRequests.reason, "task_retry"),
+              sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
+              sql`${agentWakeupRequests.payload}->'executionWait' is not null`,
+              sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array'
+                and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`,
+              sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`,
+              sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId}
+                and r.runtime_mode = 'legacy' and r.status = 'cancelled'
+                and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
+            )),
+        )).orderBy(asc(agentWakeupRequests.id)).limit(100);
+        const complete = rows.length < 100;
+        if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
+        return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
+          dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
+            : kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
+            : kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
+            : Date.now(),
+        })), complete };
+      },
+      check: async entry => {
+        if (!enabled) return null;
+        if (kind !== "task_chat") {
+          const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload,
+            updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error,
+            interruptedLegacy: sql<boolean>`exists (select 1 from heartbeat_runs r
+              where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId}
+                and r.runtime_mode = 'legacy' and r.status = 'cancelled'
+                and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`,
+          }).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId),
+            eq(agentWakeupRequests.id, entry.id)));
+          if (!current) return null;
+          const due = kind === "native" && current.status === "claimed" && current.claimedAt
+            ? current.claimedAt.getTime() + 60_000
+            : kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000
+              : kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
+                && (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
+                ? current.updatedAt.getTime() + 30_000 : null;
+          if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
+        }
+        const errors: unknown[] = [];
+        let next: number | null = null;
+        const scope = { companyId: entry.companyId, requestId: entry.id };
+        if (kind === "native") await owner.dispatchNative({ ...scope,
+          onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); },
+          onError: error => { errors.push(error); },
+        });
+        else if (kind === "task_chat") await owner.settleTaskChat(scope);
+        else await owner.resume({ ...scope, onError: error => { errors.push(error); } });
+        if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
+        return next === null ? null : { ...entry, dueAt: next };
+      },
+    }, canRun);
+  }
+  const workers = [register("deferred"), register("task_chat"), register("native")];
+  return { ready: Promise.all(workers.map(worker => worker.ready)),
+    wake() { for (const worker of workers) worker.invalidate(); },
+    async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
+}

server/src/services/heartbeat/dispatch-work.ts

@@ -0,0 +1,58 @@
+import type { RunDispatchScope } from "../run-dispatch-notifications.js";
+import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
+import { registerDeadlineWork } from "../deadline-work.js";
+import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
+
+export function registerRunDispatchWork(
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  owner: {
+    promoteDueScheduledRetries(): Promise<unknown>;
+    nextScheduledRetryAt(): Promise<number | null>;
+    resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise<unknown>;
+    schedulingSuppressed(): Promise<boolean>;
+  },
+  canRun: () => boolean,
+) {
+  let suppressed = false;
+  const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, {
+    retryMs: 30_000,
+    continuationMs: 1_000,
+    notificationCoalesceMs: 1_000,
+    canRun,
+    run: async signal => {
+      suppressed = await owner.schedulingSuppressed();
+      if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries();
+    },
+    next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(),
+  });
+  const scopes = new Set<string>(["*"]);
+  const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, {
+    retryMs: 30_000,
+    notificationCoalesceMs: 1_000,
+    canRun,
+    onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); },
+    run: async signal => {
+      const requested = scopes.has("*") ? ["*"] : [...scopes];
+      if (requested[0] === "*") scopes.clear();
+      else for (const hint of requested) scopes.delete(hint);
+      for (let index = 0; index < requested.length; index++) {
+        if (signal.aborted || !canRun()) {
+          for (const hint of requested.slice(index)) scopes.add(hint);
+          return;
+        }
+        try {
+          await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope);
+        } catch (error) {
+          for (const hint of requested.slice(index)) scopes.add(hint);
+          throw error;
+        }
+      }
+    },
+    next: async () => null,
+  });
+  return {
+    ready: Promise.all([retries.ready, queue.ready]),
+    wake() { scopes.add("*"); retries.wake(); queue.wake(); },
+    async stop() { await Promise.all([retries.stop(), queue.stop()]); },
+  };
+}

server/src/services/heartbeat/execution-configuration.ts

@@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
     stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig);
   const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId,
     runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
-  const runScopedMentionedSkillKeys =
-    preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
+  const [mentionedSkills, githubIdentity] = await Promise.allSettled([
+    preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({
       db,
       companyId: agent.companyId,
       issueId,
       commentIds: extractWakeCommentIds(context),
       actor: skillActor,
-    }));
+    })),
+    time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, {
+      agentId: agent.id, responsibleUserId, allowStandingDelegation: false,
+    })),
+  ]);
+  if (mentionedSkills.status === "rejected") throw mentionedSkills.reason;
+  if (githubIdentity.status === "rejected") throw githubIdentity.reason;
+  const runScopedMentionedSkillKeys = mentionedSkills.value;
   const runScopedSkillKeys =
     !pinnedSkillTest && acceptedPlanContinuationWake &&
     !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation
       ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY]
       : runScopedMentionedSkillKeys;
-  const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(
-    db,
-    agent.companyId,
-    {
-      agentId: agent.id,
-      responsibleUserId,
-      allowStandingDelegation: false,
-    },
-  ));
+  const githubSelection = githubIdentity.value;
   const useHostGitHub =
     !githubSelection.configured &&
     trustPreset.kind === "standard" &&
@@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
   );
   const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = [];
   const skillsPrepareStartedAtMs = Date.now();
+  let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs;
   let skillDeliveryConfig = effectiveResolvedConfig;
   const prepareSkills = async () => {
 
@@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear
       throw error;
     }
   };
-  const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills);
+  const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills))
+    .finally(() => { skillsPrepareEndedAtMs = Date.now(); });
+  const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
+  const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([
+    skillsPreparation,
+    assignmentsPreparation,
+    assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))),
+  ]);
+  if (preparedSkills.status === "rejected") throw preparedSkills.reason;
+  if (preparedAssignments.status === "rejected") throw preparedAssignments.reason;
+  if (preparedInstructions.status === "rejected") throw preparedInstructions.reason;
+  const runtimeSkillEntries = preparedSkills.value;
   nativeRunnerPreparationSpans.push({
     name: "skills.prepare",
     parentName: "task.prepare",
     startedAtMs: skillsPrepareStartedAtMs,
-    endedAtMs: Date.now(),
+    endedAtMs: skillsPrepareEndedAtMs,
   });
-  const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined }));
+  const connectorAssignments = preparedAssignments.value;
   const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments));
   // Both CLI adapters and native context materialization use the same resolved set.
   runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills);
   const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType));
   // Always replace this runtime-only field; caller wake data cannot supply skills.
   context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions };
-  let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({
-    resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }),
+  let runtimeConfig = await prepareConnectionInstructionDelivery({
+    resolve: async () => preparedInstructions.value,
     context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner",
-  }));
+  });
   const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined)
     ? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id))
     : undefined;

server/src/services/heartbeat/queue.ts

@@ -1,4 +1,8 @@
 import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
+import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
+import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
+import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
+import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js";
 import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
 import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -222,7 +226,7 @@ export interface HeartbeatQueueDependencies extends Pick<HeartbeatRetryDependenc
   publishRunLifecyclePluginEvent: (run: HeartbeatRun) => void;
   cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise<unknown>;
   cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise<unknown>;
-  resumeExecutionWaitComments: () => Promise<unknown>;
+  resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise<unknown>;
   resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise<unknown>;
   resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise<unknown>;
   formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string;
@@ -1852,13 +1856,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
     return cancelled;
   }
 
-  async function resumeQueuedRuns() {
+  async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) {
     if ((await getSchedulingSuppression()).suppressed) return;
-    await resumeExecutionWaitComments();
+    await resumeExecutionWaitComments(scope);
     const cutoff = await getWorktreeExecutionCutoff();
     const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
       .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId))
-      .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
+      .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+        scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
+        eq(agentWakeupRequests.status, "deferred_issue_execution"),
         eq(companies.status, "active"),
         sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`,
         lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)),
@@ -1869,6 +1875,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"),
       ));
       await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => {
+        scope.onError?.(err);
         logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue");
       });
     }
@@ -1882,7 +1889,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId),
           sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`)))
       .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
-      .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"),
+      .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+        scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
+        eq(agentWakeupRequests.status, "deferred_issue_execution"),
         isNull(issues.executionRunId),
         or(and(
           sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`,
@@ -1921,7 +1930,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue;
       if (wake.idempotencyKey?.startsWith("chat-inbound:")) {
         await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId,
-          issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => {
+          issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => {
+          scope.onError?.(err);
           logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery");
         });
         continue;
@@ -1933,6 +1943,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
           latest.errorCode === "execution_reconciliation_required";
         if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) {
           await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => {
+            scope.onError?.(err);
             logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped");
           });
         }
@@ -1940,6 +1951,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => {
         if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource);
       }).catch(err => {
+        scope.onError?.(err);
         logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments");
       });
     }
@@ -1957,6 +1969,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       ))
       .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId))
       .where(and(
+        scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+        scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
         eq(agentWakeupRequests.status, "deferred_issue_execution"),
         eq(heartbeatRuns.status, "cancelled"),
         eq(heartbeatRuns.runtimeMode, "legacy"),
@@ -1965,16 +1979,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       ));
     for (const run of interruptedQueues) {
       await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => {
+        scope.onError?.(err);
         logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue");
       });
     }
 
+  }
+
+  async function resumeQueuedRuns() {
+    await resumeDeferredExecutionReceipts();
     await resumeQueuedRunsAfterDrain();
   }
 
   // Release admission promptly without first scanning unrelated recovery work.
   // Every run still passes the ordinary per-agent claim and dispatch gates.
-  async function resumeQueuedRunsAfterDrain() {
+  async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) {
     if ((await getSchedulingSuppression()).suppressed) return;
     const cutoff = await getWorktreeExecutionCutoff();
     const queuedRuns = await db
@@ -1984,6 +2003,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       .where(
         and(
           eq(heartbeatRuns.status, "queued"),
+          scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined,
           eq(companies.status, "active"),
           cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined,
         ),
@@ -2506,17 +2526,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       // its one durable receipt ID on a skipped wake.
       if (caseRequest) return { created: false };
       if (executionWaitRequestId) {
-        await db.update(agentWakeupRequests).set({
-          payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
-            coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({
-              reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
-                ? "This task is paused. Resume it to send your saved message."
-                : "Waiting for task execution to be enabled. Your message is saved."),
-            })}::jsonb)`,
-          updatedAt: new Date(),
-        }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId),
-          eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId),
-          eq(agentWakeupRequests.status, "deferred_issue_execution")));
+        await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, {
+          reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active"
+            ? "This task is paused. Resume it to send your saved message."
+            : "Waiting for task execution to be enabled. Your message is saved."),
+        });
         return { created: false };
       }
       const request = {
@@ -3404,15 +3418,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
             opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause);
             const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait };
             if (executionWaitRequestId) {
-              await tx.update(agentWakeupRequests).set({
-                payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`,
-                updatedAt: new Date(),
-              }).where(eq(agentWakeupRequests.id, executionWaitRequestId));
+              await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId,
+                requestId: executionWaitRequestId }, condition, { replace: true });
               return { kind: "deferred" as const, admitted: true };
             }
             if (durableRequest || wakeCommentId ||
                 hasInteractionContinuationWakeContext(enrichedContextSnapshot) ||
                 readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) {
+              await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
               await tx.insert(agentWakeupRequests).values({
                 ...durableReceiptFields,
                 companyId: agent.companyId, agentId, source, triggerDetail, reason,
@@ -4031,6 +4044,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
                 }));
             }
             if (followUp) {
+              if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId });
               const assignmentId = followUp.assignmentId;
               const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake =>
                 wake.payload?.[followUpPayloadKey] === assignmentId &&
@@ -4635,6 +4649,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
     }
 
     const queueOutcome = await db.transaction(async (tx) => {
+      await notifyRunDispatch(tx, { companyId: agent.companyId });
       await tx.execute(
         sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`,
       );
@@ -4768,6 +4783,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
       companyId?: string;
       limit?: number;
       staleClaimMs?: number;
+      requestId?: string;
+      onDeferredDeadline?: (at: number) => void;
+      onError?: (error: unknown) => void;
     } = {},
   ) {
     const now = new Date();
@@ -4780,6 +4798,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
           input.companyId
             ? eq(agentWakeupRequests.companyId, input.companyId)
             : undefined,
+          input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined,
           eq(agentWakeupRequests.requestedByActorType, "system"),
           eq(agentWakeupRequests.requestedByActorId, "native-status-committer"),
           inArray(agentWakeupRequests.status, ["queued", "claimed"]),
@@ -4856,6 +4875,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         candidate.claimedAt &&
         now.getTime() - candidate.claimedAt.getTime() < staleClaimMs
       ) {
+        input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs);
         deferred += 1;
         continue;
       }
@@ -5030,6 +5050,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
         if (wakeRun) dispatched += 1;
         else deferred += 1;
       } catch (error) {
+        input.onError?.(error);
         await db
           .update(agentWakeupRequests)
           .set({
@@ -5073,6 +5094,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
     withChatControlRecoveryGate,
     trackWakeup,
     resumeQueuedRuns,
+    resumeDeferredExecutionReceipts,
     resumeQueuedRunsAfterDrain,
     parseHeartbeatPolicy,
     nextTimerHeartbeatAt,

server/src/services/heartbeat/recovery.ts

@@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
 
   let settledProcessesDiscovered = false;
 
-  async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
+  async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) {
     if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db);
     const staleThresholdMs = opts?.staleThresholdMs ?? 0;
     const now = new Date();
@@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
     });
     if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup();
     await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments);
-    await dispatchPendingNativeStatusWakeups().catch((error) => {
+    if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => {
       logger.warn(
         { err: error },
         "failed to dispatch persisted native status wake intents before orphan reaping",

server/src/services/heartbeat/retries.test.ts

@@ -35,6 +35,7 @@ function callbacks() {
     getWorktreeExecutionCutoff: vi.fn(async (): Promise<Date | null> => null),
     applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(),
     runDispatch: {
+      nextScheduledRetryAt: vi.fn(async () => null),
       evaluateScheduledRetryGate: vi.fn<HeartbeatRetryDependencies["runDispatch"]["evaluateScheduledRetryGate"]>(async () => ({ allowed: true })),
       promoteDueScheduledRetries: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteDueScheduledRetries"]>(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })),
       promoteScheduledRetry: vi.fn<HeartbeatRetryDependencies["runDispatch"]["promoteScheduledRetry"]>(async () => ({ outcome: "not_promoted" })),
@@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => {
   });
 });
 
-const support = await getEmbeddedPostgresTestSupport();
+import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
+const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL");
+const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
 const describePostgres = support.supported ? describe : describe.skip;
 if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`);
 
@@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => {
   const now = new Date("2026-10-08T12:00:00Z");
 
   beforeAll(async () => {
-    database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
-    db = createDb(database.connectionString);
+    if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-");
+    db = createDb(external ?? database.connectionString, { maxConnections: 6 });
   }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
 
-  afterAll(async () => { await database?.cleanup(); });
+  afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
 
   afterEach(async () => {
     await db.delete(issues);

server/src/services/heartbeat/retries.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "../status-card-notifications.js";
+import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js";
 import { infrastructureRetryDelayMs } from "../system-recovery-state.js";
 import {
@@ -78,7 +79,7 @@ type PlanApprovalResumeFailure = {
 export interface HeartbeatRetryDependencies {
   hasCaseHandoffForRun: (run: Pick<HeartbeatRun, "id" | "companyId">) => Promise<boolean>;
   runDispatch: Pick<ReturnType<typeof createRunDispatch>,
-    "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">;
+    "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">;
   getAgent: (agentId: string) => Promise<typeof agents.$inferSelect | null | undefined>;
   getRun: (
     runId: string,
@@ -998,6 +999,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
           retryContextSnapshot.previousRunId = continuation.previousRunId;
         }
 
+        await notifyRunDispatch(tx, { companyId: run.companyId });
         const wakeupRequest = await tx
           .insert(agentWakeupRequests)
           .values({
@@ -1645,6 +1647,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
     });
   }
 
+  async function nextScheduledRetryAt() {
+    return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() });
+  }
+
   async function promoteDueScheduledRetries(now = new Date()) {
     const cutoff = await getWorktreeExecutionCutoff();
     const result = await runDispatch.promoteDueScheduledRetries({
@@ -1752,6 +1758,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
     };
 
     const updated = await db.transaction(async (tx) => {
+      await notifyRunDispatch(tx, { companyId: scheduled.run.companyId });
       const row = await tx
         .update(heartbeatRuns)
         .set({
@@ -1903,6 +1910,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen
     finalizeAiConnectionBusyDeferral,
     finalizeWorkspaceBusyDeferral,
     promoteDueScheduledRetries,
+    nextScheduledRetryAt,
     retryScheduledRetryNow,
 
     scheduleBoundedRetry: async (

server/src/services/heartbeat/run-control.ts

@@ -1,3 +1,5 @@
+import { writeDeferredExecutionWait } from "../deferred-execution-wait.js";
+import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
 import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js";
 import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js";
 import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
@@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
           onBlocked: (reason, message) => { wait = { reason, message }; },
         });
         if (!admitted) {
-          await db.update(agentWakeupRequests).set({
-            payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}',
-              coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`,
-            updatedAt: new Date(),
-          }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId),
-            eq(agentWakeupRequests.status, "deferred_issue_execution")));
+          await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait);
           continue;
         }
       }
@@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
       idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id);
   }
 
-  async function resumeExecutionWaitComments() {
+  async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) {
     if ((await getSchedulingSuppression()).suppressed) return;
-    await reconcileTaskChatRetryReceipts(db);
+    await reconcileTaskChatRetryReceipts(db, scope);
     const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId })
       .from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId),
         sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
         eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
       .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
-      .where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
+      .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+        scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
+        eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
         sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
         notInArray(issues.status, ["done", "cancelled"])))
       .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
     for (const retry of retries) {
       await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id));
-      await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"));
+      await resumeTaskRetry(retry.companyId, retry.id).catch(err => {
+        scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred");
+      });
     }
     const waits = await db.select({ wake: agentWakeupRequests })
       .from(agentWakeupRequests)
@@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
         sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`,
         eq(issues.assigneeAgentId, agentWakeupRequests.agentId)))
       .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active")))
-      .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
+      .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined,
+        scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
+        exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and(
         eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id),
         executionBlockerPredicate(),
       ))), eq(agentWakeupRequests.status, "deferred_issue_execution"),
@@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
       if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue;
       if (canContinueCancelledRun(run)) {
         await resumeSavedComments(wake.companyId, wake.id).catch(err => {
+          scope.onError?.(err);
           logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation");
         });
         continue;
       }
       await resumeRemoteStopComments(run, wake.id).catch(err => {
+        scope.onError?.(err);
         logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message");
       });
     }

server/src/services/heartbeat/run-lifecycle.ts

@@ -1,3 +1,4 @@
+import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js";
 import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
 import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
@@ -2447,6 +2448,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl
         .returning()
         .then((rows) => rows[0]);
 
+      await notifyRunDispatch(tx, { companyId: run.companyId });
       const queuedRun = await tx
         .insert(heartbeatRuns)
         .values({

server/src/services/heartbeat/run-preparation.test.ts

@@ -14,6 +14,7 @@ import {
   ConfigurationIncompleteFailure,
   createHeartbeatRunPreparation,
   resolveExecutionRunAdapterConfig,
+  readHeartbeatWakeComments,
 } from "./run-preparation.js";
 
 // Existing integration suites exercise the real database and dispatch path.
@@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) {
   return { db: { select } as unknown as Db, select, scopes };
 }
 
+function deferred<T>() {
+  let resolve!: (value: T) => void;
+  const promise = new Promise<T>(complete => { resolve = complete; });
+  return { promise, resolve };
+}
+
 const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null };
 
 describe("heartbeat run preparation module", () => {
@@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => {
     expect(database.select).toHaveBeenCalledTimes(2);
   });
 
+  it("scopes one selected-comment snapshot for both wake consumers", async () => {
+    const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]);
+    expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"]))
+      .toEqual([{ id: "comment-one", body: "selected wake" }]);
+    expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"]));
+    expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]);
+    expect(database.select).toHaveBeenCalledOnce();
+  });
+
+  it("settles independent binding checks before surfacing a failure or resolving values", async () => {
+    const projectCheck = deferred<[]>();
+    const firstFailure = new Error("agent binding check failed");
+    const secretsSvc = {
+      resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(),
+      collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => {
+        if (context.consumerType === "agent") throw firstFailure;
+        return context.consumerType === "project" ? projectCheck.promise : [];
+      }),
+      collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []),
+    };
+    let settled = false;
+    const opening = resolveExecutionRunAdapterConfig({
+      companyId: "company-one", agentId: "agent-one", projectId: "project-one",
+      environmentId: "environment-one", environmentEnv: { SHARED: "environment" },
+      executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc,
+    });
+    void opening.then(() => { settled = true; }, () => { settled = true; });
+    await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3));
+    expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce();
+    expect(settled).toBe(false);
+    expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
+    expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled();
+    projectCheck.resolve([]);
+    await expect(opening).rejects.toBe(firstFailure);
+  });
+
   it("keeps missing scoped credentials as a pre-dispatch failure", async () => {
     const secretsSvc = {
       resolveAdapterConfigForRuntime: vi.fn(),

server/src/services/heartbeat/run-preparation.ts

@@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: {
   // no binding before resolving any secret value. Missing bindings short-circuit
   // to a configuration-incomplete blocker routed to a human owner instead of a
   // dispatched-then-failed run (which previously surfaced as opaque setup_failed).
-  if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") {
-    const missingBindings: MissingRuntimeBinding[] = [];
-    if (environmentEnv && input.environmentId) {
-      missingBindings.push(
-        ...(await input.secretsSvc.collectMissingRuntimeBindings(
-          input.companyId,
-          environmentEnv,
-          {
-            consumerType: "environment",
-            consumerId: input.environmentId,
-            responsibleUserId: input.responsibleUserId ?? null,
-          },
-        )),
-      );
-    }
+  const collectMissing = input.secretsSvc.collectMissingRuntimeBindings;
+  if (typeof collectMissing === "function") {
+    const checks: Array<() => Promise<MissingRuntimeBinding[]>> = [];
+    const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => {
+      checks.push(() => collectMissing(input.companyId, env, {
+        consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null,
+      }));
+    };
+    if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId);
     if (input.agentId) {
-      missingBindings.push(
-        ...(await input.secretsSvc.collectMissingRuntimeBindings(
-          input.companyId,
-          parseObject(executionRunConfig.env),
-          {
-            consumerType: "agent",
-            consumerId: input.agentId,
-            responsibleUserId: input.responsibleUserId ?? null,
-          },
-        )),
-      );
-      if (
-        typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings ===
-        "function"
-      ) {
-        missingBindings.push(
-          ...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings(
-            input.companyId,
-            executionRunConfig,
-            input.adapterType ?? null,
-            {
-              consumerType: "agent",
-              consumerId: input.agentId,
-              responsibleUserId: input.responsibleUserId ?? null,
-            },
-          )),
-        );
+      checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId);
+      const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings;
+      if (typeof collectAdapterMissing === "function") {
+        checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, {
+          consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null,
+        }));
       }
     }
-    if (projectEnv && input.projectId) {
-      missingBindings.push(
-        ...(await input.secretsSvc.collectMissingRuntimeBindings(
-          input.companyId,
-          projectEnv,
-          {
-            consumerType: "project",
-            consumerId: input.projectId,
-            responsibleUserId: input.responsibleUserId ?? null,
-          },
-        )),
-      );
-    }
-    if (routineEnv && input.routineId) {
-      missingBindings.push(
-        ...(await input.secretsSvc.collectMissingRuntimeBindings(
-          input.companyId,
-          routineEnv,
-          {
-            consumerType: "routine",
-            consumerId: input.routineId,
-            responsibleUserId: input.responsibleUserId ?? null,
-          },
-        )),
-      );
+    if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId);
+    if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId);
+    const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check)));
+    const missingBindings: MissingRuntimeBinding[] = [];
+    for (const result of results) {
+      if (result.status === "rejected") throw result.reason;
+      missingBindings.push(...result.value);
     }
     if (requiredScopedEnvBinding) {
       const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys);
@@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: {
   throw new Error("reviewed_chat_execution_binding_not_ready");
 }
 
+export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) {
+  if (commentIds.length === 0) return [];
+  return db
+    .select({
+      id: issueComments.id,
+      companyId: issueComments.companyId,
+      issueId: issueComments.issueId,
+      body: issueComments.body,
+      origin: issueComments.origin,
+      authorType: issueComments.authorType,
+      authorAgentId: issueComments.authorAgentId,
+      authorUserId: issueComments.authorUserId,
+      presentation: issueComments.presentation,
+      metadata: issueComments.metadata,
+      deletedAt: issueComments.deletedAt,
+      deletedByType: issueComments.deletedByType,
+      deletedByAgentId: issueComments.deletedByAgentId,
+      deletedByUserId: issueComments.deletedByUserId,
+      deletedByRunId: issueComments.deletedByRunId,
+      sourceTrust: issueComments.sourceTrust,
+      createdAt: issueComments.createdAt,
+    })
+    .from(issueComments)
+    .where(
+      and(
+        eq(issueComments.companyId, companyId),
+        issueId ? eq(issueComments.issueId, issueId) : undefined,
+        inArray(issueComments.id, commentIds),
+      ),
+    );
+}
+
 export async function buildPaperclipWakePayload(input: {
   db: Db;
   companyId: string;
   agentId?: string | null;
   runId?: string | null;
   contextSnapshot: Record<string, unknown>;
+  wakeCommentRows?: Awaited<ReturnType<typeof readHeartbeatWakeComments>>;
   continuationSummary?: {
     key: string;
     title: string | null;
@@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: {
   )
     return null;
 
-  const commentRows =
-    commentIds.length === 0
-      ? []
-      : await input.db
-          .select({
-            id: issueComments.id,
-            issueId: issueComments.issueId,
-            body: issueComments.body,
-            origin: issueComments.origin,
-            authorType: issueComments.authorType,
-            authorAgentId: issueComments.authorAgentId,
-            authorUserId: issueComments.authorUserId,
-            presentation: issueComments.presentation,
-            metadata: issueComments.metadata,
-            deletedAt: issueComments.deletedAt,
-            deletedByType: issueComments.deletedByType,
-            deletedByAgentId: issueComments.deletedByAgentId,
-            deletedByUserId: issueComments.deletedByUserId,
-            deletedByRunId: issueComments.deletedByRunId,
-            sourceTrust: issueComments.sourceTrust,
-            createdAt: issueComments.createdAt,
-          })
-          .from(issueComments)
-          .where(
-            and(
-              eq(issueComments.companyId, input.companyId),
-              issueId ? eq(issueComments.issueId, issueId) : undefined,
-              inArray(issueComments.id, commentIds),
-            ),
-          );
+  const commentRows = input.wakeCommentRows
+    ? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id))
+    : await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds);
 
   const commentsById = new Map(
     commentRows.map((comment) => [comment.id, comment]),

server/src/services/heartbeat/task-context.ts

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db";
+import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db";
 import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
 import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils";
 import { parseObject } from "../../adapters/utils.js";
@@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js";
 import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js";
 import type { TrustPresetResolution } from "../trust-preset-resolver.js";
 import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js";
-import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
+import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js";
 import { buildPaperclipTaskMarkdown } from "./task-markdown.js";
+import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js";
 import { workspacePreferenceService } from "../workspace-preferences.js";
 
 type RunPreparation = ReturnType<typeof createHeartbeatRunPreparation>;
@@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput {
       "id" | "identifier" | "title" | "status" | "priority" | "workMode" |
       "conversationAgentId" | "description" | "projectId"> | null;
     issueContext: IssueContext | null;
-    wakeCommentContext: Pick<typeof issueComments.$inferSelect,
-      "id" | "body" | "sourceTrust" | "presentation" | "metadata"> | null;
     context: Record<string, unknown>;
     taskSession: Pick<typeof agentTaskSessions.$inferSelect, "lastRunId"> | null;
   };
@@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null {
 /** Rebuilds task prompt fields on the run's context; acquires no execution resources. */
 export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) {
   const { run, agent } = input;
-  const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task;
+  const { issueRef, issueContext, context, taskSession } = input.task;
   const { trustPreset, experimentalInstanceSettings } = input.policy;
   const { issuesSvc, getPinnedSkillTestContext } = input.services;
-  const continuationSummary = issueRef && !isConversation(issueContext)
-    ? await getIssueContinuationSummaryDocument(db, issueRef.id)
-    : null;
-  const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
-  const safeContinuationSummary =
-    continuationSummary && !exposeLowTrustRaw
-      ? redactQuarantinedBodyForHigherTrust(continuationSummary)
-      : continuationSummary;
-  const safeWakeCommentContext =
-    wakeCommentContext && !exposeLowTrustRaw
-      ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
-      : wakeCommentContext;
-  const issueAncestors = issueRef
-    ? await issuesSvc.getAncestors(issueRef.id)
-    : [];
-  if (continuationSummary) {
-    context.paperclipContinuationSummary = {
-      key: safeContinuationSummary!.key,
-      title: safeContinuationSummary!.title,
-      body: safeContinuationSummary!.body,
-      sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
-      updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
-    };
-  } else {
-    delete context.paperclipContinuationSummary;
-  }
   const admittedScope = parseObject(context.paperclipAgentJwtScope);
   if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) {
     const savedIssue = parseObject(context.paperclipIssue);
@@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
       : { kind: "standard" };
   }
   const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test";
-  const pinnedSkillTestContext =
-    isSkillTest && issueRef
-      ? await getPinnedSkillTestContext(agent.companyId, issueRef.id)
-      : null;
+  const wakeCommentId = deriveCommentId(context, null);
+  const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])];
+  const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([
+    issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null),
+    issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]),
+    isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null),
+    readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds),
+  ]);
+  if (summary.status === "rejected") throw summary.reason;
+  if (ancestors.status === "rejected") throw ancestors.reason;
+  if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason;
+  if (wakeComments.status === "rejected") throw wakeComments.reason;
+  const wakeCommentRows = wakeComments.value;
+  const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null;
+  const wakeCommentContext = selectedWakeComment ? {
+    id: selectedWakeComment.id,
+    body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body,
+    authorType: selectedWakeComment.authorType,
+    authorAgentId: selectedWakeComment.authorAgentId,
+    authorUserId: selectedWakeComment.authorUserId,
+    presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation,
+    metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata,
+    deletedAt: selectedWakeComment.deletedAt,
+    deletedByType: selectedWakeComment.deletedByType,
+    deletedByAgentId: selectedWakeComment.deletedByAgentId,
+    deletedByUserId: selectedWakeComment.deletedByUserId,
+    deletedByRunId: selectedWakeComment.deletedByRunId,
+    sourceTrust: selectedWakeComment.sourceTrust,
+  } : null;
+  const continuationSummary = summary.value;
+  const issueAncestors = ancestors.value;
+  const pinnedSkillTestContext = pinnedSkillTestRead.value;
+  const exposeLowTrustRaw = trustPreset.kind === "low_trust_review";
+  const safeContinuationSummary =
+    continuationSummary && !exposeLowTrustRaw
+      ? redactQuarantinedBodyForHigherTrust(continuationSummary)
+      : continuationSummary;
+  const safeWakeCommentContext =
+    wakeCommentContext && !exposeLowTrustRaw
+      ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext)
+      : wakeCommentContext;
+  if (continuationSummary) {
+    context.paperclipContinuationSummary = {
+      key: safeContinuationSummary!.key,
+      title: safeContinuationSummary!.title,
+      body: safeContinuationSummary!.body,
+      sourceTrust: safeContinuationSummary!.sourceTrust ?? null,
+      updatedAt: safeContinuationSummary!.updatedAt.toISOString(),
+    };
+  } else {
+    delete context.paperclipContinuationSummary;
+  }
   if (isSkillTest && !pinnedSkillTestContext) {
     throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", {
       configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" },
@@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo
     agentId: agent.id,
     runId: run.id,
     contextSnapshot: context,
+    wakeCommentRows,
     continuationSummary,
     issueSummary: issueRef
       ? {

server/src/services/instance-settings.ts

@@ -1,4 +1,6 @@
 import { notifyStatusCardEvaluation } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js";
 import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js";
 import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js";
@@ -632,6 +634,8 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch);
         if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx);
         if (patch.enableWorktreeRunExecution !== undefined) {
+          await notifyRunDispatch(tx);
+          await notifyDeferredExecutionEligibility(tx);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           await notifyExternalAgentSettlement(tx);
           await notifyInstructionCommit(tx);

server/src/services/issue-comment-writes.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import type { PgInsertValue } from "drizzle-orm/pg-core";
 import { issueComments, type Db } from "@paperclipai/db";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
@@ -8,6 +9,8 @@ type WriteDb = Db | Parameters<Parameters<Db["transaction"]>[0]>[0];
 export function insertIssueComment(db: WriteDb, input: PgInsertValue<typeof issueComments> & { companyId: string }) {
   return db.transaction(async tx => {
     await notifyStatusCardIssueChange(tx, input.companyId);
+    await notifyIssueRecovery(tx, { companyId: input.companyId,
+      issueId: typeof input.issueId === "string" ? input.issueId : undefined });
     await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`);
     return tx.insert(issueComments).values(input).returning();
   });

server/src/services/issue-recovery-actions.ts

@@ -1,3 +1,4 @@
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
 import { writeExecutionControl } from "./execution-control-notifications.js";
@@ -322,6 +323,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
   ): Promise<IssueRecoveryAction> {
     try {
       const created = await db.transaction(async (tx) => {
+        await notifyIssueRecovery(tx, { companyId: input.companyId });
         await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition);
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation);
@@ -413,6 +415,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
           Math.min(nextAttemptCount, effectiveMaxAttempts),
         );
         const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
+          await notifyIssueRecovery(tx, { companyId: input.companyId });
           await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
           if ((!input.preserveExistingOwner && existing.cause !== input.cause)
             || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -474,6 +477,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
         return toReadModel(exhausted);
       }
       const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
+        await notifyIssueRecovery(tx, { companyId: input.companyId });
         await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
         if ((!input.preserveExistingOwner && existing.cause !== input.cause)
           || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) {
@@ -548,6 +552,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
 
     try {
       const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => {
+        await notifyIssueRecovery(tx, { companyId: input.companyId });
         await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
         await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
         return tx
@@ -595,6 +600,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) {
       const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions)
         .where(and(...predicates)).limit(1);
       if (!pending.length) return [];
+      await notifyIssueRecovery(tx, { companyId: input.companyId });
       await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
       await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId });
       return tx

server/src/services/issue-recovery-notifications.ts

@@ -0,0 +1,16 @@
+import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
+
+/** An issue hint is reserved for local facts such as comments. Lifecycle,
+ * graph, run and eligibility changes retain company scope because their
+ * consequences can include ancestors and shared execution references. */
+export type IssueRecoveryScope = { companyId: string; issueId?: string };
+
+export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) {
+  const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
+  const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*";
+  for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies,
+    DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) {
+    await notifyDeliveryWork(transaction, queue,
+      queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint);
+  }
+}

server/src/services/issue-review-state-notifications.ts

@@ -1,4 +1,5 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
 import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
@@ -6,6 +7,7 @@ import { notifyNativeStatusReassessment } from "./native-status-reassessment-not
 
 export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) {
   await notifyStatusCardIssueChange(tx, scope.companyId);
+  await notifyRunDispatch(tx, scope);
   await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
   await notifyExternalConversationSettlement(tx, scope);
   await notifyNativeReviewResponse(tx, scope);

server/src/services/issue-tree-control.ts

@@ -1,4 +1,6 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
 import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
@@ -759,6 +761,8 @@ export function issueTreeControlService(db: Db) {
       const releaseReason = input.reason ?? "Subtree resume applied.";
 
       const { hold: resumeHold } = await db.transaction(async (tx) => {
+        await notifyIssueRecovery(tx, { companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId });
         const [createdHold] = await tx
           .insert(issueTreeHolds)
           .values({
@@ -837,6 +841,8 @@ export function issueTreeControlService(db: Db) {
     }
 
     const { hold, members } = await db.transaction(async (tx) => {
+      await notifyIssueRecovery(tx, { companyId });
+      await notifyDeferredExecutionEligibility(tx, { companyId });
       const [createdHold] = await tx
         .insert(issueTreeHolds)
         .values({
@@ -1001,6 +1007,7 @@ export function issueTreeControlService(db: Db) {
     const now = new Date();
     const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id);
     const updatedIssues = await db.transaction(async (tx) => {
+      await notifyDeferredExecutionEligibility(tx, { companyId });
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
       await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
       const restored: TreeStatusUpdateResult["updatedIssues"] = [];
@@ -1179,7 +1186,10 @@ export function issueTreeControlService(db: Db) {
       throw conflict("Issue tree hold is already released");
     }
 
-    const [updated] = await db
+    const [updated] = await db.transaction(async tx => {
+      await notifyIssueRecovery(tx, { companyId });
+      await notifyDeferredExecutionEligibility(tx, { companyId });
+      return tx
       .update(issueTreeHolds)
       .set({
         status: "released",
@@ -1197,6 +1207,7 @@ export function issueTreeControlService(db: Db) {
       })
       .where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId)))
       .returning();
+    });
 
     const members = await db
       .select()

server/src/services/issues.ts

@@ -1,4 +1,7 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
 import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js";
 import { notifyNativeRunProjection } from "./native-run-projection-notifications.js";
@@ -7627,6 +7630,8 @@ export function issueService(db: Db) {
       await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx);
     }
 
+    await notifyIssueRecovery(dbOrTx, { companyId });
+    await notifyDeferredExecutionEligibility(dbOrTx, { companyId });
     await dbOrTx
       .delete(issueRelations)
       .where(
@@ -9677,27 +9682,30 @@ export function issueService(db: Db) {
       });
 
       if (blockParentUntilDone) {
-        const existingBlockers = await db
-          .select({ blockerIssueId: issueRelations.issueId })
-          .from(issueRelations)
-          .where(
-            and(
-              eq(issueRelations.companyId, parent.companyId),
-              eq(issueRelations.relatedIssueId, parent.id),
-              eq(issueRelations.type, "blocks"),
-            ),
+        await db.transaction(async tx => {
+          const existingBlockers = await tx
+            .select({ blockerIssueId: issueRelations.issueId })
+            .from(issueRelations)
+            .where(
+              and(
+                eq(issueRelations.companyId, parent.companyId),
+                eq(issueRelations.relatedIssueId, parent.id),
+                eq(issueRelations.type, "blocks"),
+              ),
+            );
+          await syncBlockedByIssueIds(
+            parent.id,
+            parent.companyId,
+            [
+              ...new Set([
+                ...existingBlockers.map((row) => row.blockerIssueId),
+                child.id,
+              ]),
+            ],
+            { agentId: actorAgentId ?? null, userId: actorUserId ?? null },
+            tx,
           );
-        await syncBlockedByIssueIds(
-          parent.id,
-          parent.companyId,
-          [
-            ...new Set([
-              ...existingBlockers.map((row) => row.blockerIssueId),
-              child.id,
-            ]),
-          ],
-          { agentId: actorAgentId ?? null, userId: actorUserId ?? null },
-        );
+        });
         [child] = await withIssueRelationSummaries(
           parent.companyId,
           [child],
@@ -10608,6 +10616,8 @@ export function issueService(db: Db) {
             : values.parentId ? `issue:${values.parentId}` : undefined);
         }
         if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+        await notifyRunDispatch(tx, { companyId: values.companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
         await notifyStatusCardIssueChange(tx, companyId);
         const [issue] = await tx.insert(issues).values(values).returning();
@@ -11499,7 +11509,11 @@ export function issueService(db: Db) {
           await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
         }
         if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined
-          || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) {
+          || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined
+          || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined
+          || blockedByIssueIds !== undefined) {
+          await notifyRunDispatch(tx, { companyId: receiptExisting.companyId });
+          await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId });
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
         }
         if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt &&
@@ -11968,6 +11982,8 @@ export function issueService(db: Db) {
           }
           await notifyDecisionTargetIssueRemoval(tx, id);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+          await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined);
+          if (owner) await notifyDeferredExecutionEligibility(tx, owner);
           await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
           removedIssue = await tx
             .delete(issues)
@@ -12097,6 +12113,8 @@ export function issueService(db: Db) {
       const updated = await db.transaction(async tx => {
         await notifyStatusCardIssueChange(tx, issueCompany.companyId);
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+        await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
         return tx
           .update(issues)
@@ -12230,6 +12248,8 @@ export function issueService(db: Db) {
           const adopted = await db.transaction(async tx => {
             await notifyStatusCardIssueChange(tx, issueCompany.companyId);
             await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+            await notifyRunDispatch(tx, { companyId: issueCompany.companyId });
+            await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId });
             await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
             return tx
               .update(issues)
@@ -12485,6 +12505,8 @@ export function issueService(db: Db) {
         await notifyTerminalWorkspaceWork(tx, `issue:${id}`);
         await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+        await notifyRunDispatch(tx, { companyId: existing.companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
         const updated = await tx
           .update(issues)
@@ -12546,6 +12568,8 @@ export function issueService(db: Db) {
           await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id });
         }
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
+        await notifyRunDispatch(tx, { companyId: existing.companyId });
+        await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId });
         await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
         const updated = await tx
           .update(issues)
@@ -13809,6 +13833,7 @@ export function issueService(db: Db) {
           .then((rows) => rows[0] ?? null);
         if (!existing) return null;
 
+        await notifyIssueRecovery(tx, { companyId: existing.companyId });
         await tx.delete(issueAttachments).where(eq(issueAttachments.id, id));
         await tx.delete(assets).where(eq(assets.id, existing.assetId));
         return existing;

server/src/services/legacy-controller-lease.ts

@@ -77,14 +77,18 @@ async function renewOnLivenessPool(
   });
 }
 
-export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
-  if (run.runtimeMode === "native" || !run.controllerBootId) return false;
-  const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and(
+export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise<Date | null> {
+  if (run.runtimeMode === "native" || !run.controllerBootId) return null;
+  const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and(
     eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId),
     eq(heartbeatRuns.status, "running"),
     gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`),
   ));
-  return Boolean(owner);
+  return owner?.expiresAt ?? null;
+}
+
+export async function hasLiveLegacyController(db: Db, run: Run): Promise<boolean> {
+  return (await liveLegacyControllerExpiresAt(db, run)) !== null;
 }
 
 export type RevokedLegacyControllerState = Pick<Run, "executionStage" | "processPid" | "processGroupId">;

server/src/services/native-runtime/chat-attachment-reuse.test.ts

@@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => {
   beforeAll(async () => {
     const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL");
     if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-");
-    db = createDb(external ?? temporary!.connectionString);
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
     const storageRoot = await mkdtemp(
       path.join(tmpdir(), "paperclip-chat-reuse-"),
     );

server/src/services/native-runtime/chat-attachment-reuse.ts

@@ -1,3 +1,4 @@
+import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
 import { createHash } from "node:crypto";
 
 import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm";
@@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: {
     if (!workProduct) {
       throw new Error("paperclip_runner_chat_attachment_origin_not_persisted");
     }
-    await input.db
+    await input.db.transaction(async tx => {
+      await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
+      await tx
       .update(issueWorkProducts)
       .set({
         title: input.title,
@@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: {
           eq(issueWorkProducts.createdByRunId, input.binding.runId),
         ),
       );
+    });
     const comment = await issueService(input.db).addComment(
       input.binding.issueId,
       `Prepared ${input.title} for this response.`,

server/src/services/native-runtime/native-provider-capacity-retry.ts

@@ -1,3 +1,4 @@
+import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { randomUUID } from "node:crypto";
 import { and, eq } from "drizzle-orm";
 import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db";
@@ -49,6 +50,7 @@ export async function materializeNativeProviderCapacityRetry(input: {
     scheduledRetryAt: dueAt.toISOString(),
     executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt),
   };
+  await notifyRunDispatch(input.tx, { companyId: run.companyId });
   const [wake] = await input.tx.insert(agentWakeupRequests).values({
     companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system",
     reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId,

server/src/services/native-runtime/native-runner-file-handoff.test.ts

@@ -1,3 +1,4 @@
+import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
 import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js";
 import { nativeCompletionFeedback } from "./native-completion-feedback.js";
 import { createHash, randomUUID } from "node:crypto";
@@ -61,10 +62,9 @@ describe("native runner file handoff", () => {
   const runId = "00000000-0000-4000-8000-000000009104";
 
   beforeAll(async () => {
-    temporary = await startEmbeddedPostgresTestDatabase(
-      "native-runner-file-handoff-",
-    );
-    db = createDb(temporary.connectionString);
+    const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL");
+    if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-");
+    db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
     temporaryRoot = await mkdtemp(
       path.join(tmpdir(), "paperclip-native-file-handoff-"),
     );

server/src/services/native-runtime/native-runner-file-handoff.ts

@@ -1,3 +1,4 @@
+import { notifyIssueRecovery } from "../issue-recovery-notifications.js";
 import { createHash, randomUUID } from "node:crypto";
 import { execFile } from "node:child_process";
 import { constants } from "node:fs";
@@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: {
     ) {
       throw new Error("paperclip_runner_file_handoff_origin_not_persisted");
     }
-    await input.db
+    await input.db.transaction(async tx => {
+      await notifyIssueRecovery(tx, { companyId: input.binding.companyId });
+      await tx
       .update(issueWorkProducts)
       .set({ title: verified.title, updatedAt: new Date() })
       .where(
@@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: {
           eq(issueWorkProducts.createdByRunId, input.binding.runId),
         ),
       );
+    });
     const comment = await issueService(input.db).addComment(
       input.binding.issueId,
       `Prepared ${verified.title} for this response.`,

server/src/services/native-runtime/native-workspace-finalizer.test.ts

@@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => {
     const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL");
     if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-");
     connectionString = external ?? temporary!.connectionString;
-    db = createDb(connectionString);
+    db = createDb(connectionString, { maxConnections: 6 });
 
     await db.insert(companies).values([
       { id: companyId, name: "Native workspace owner", issuePrefix: "NWO" },
@@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => {
     const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" });
     const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => {
       await ownership.assertHeld();
-      expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull();
+      expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull();
       expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]);
       return "copied";
     });
@@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => {
     try {
       await entered;
       await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`);
-      const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run");
+      const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run");
       expect(second).toEqual({ acquired: false, reason: "busy" });
       const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId));
       expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy();
@@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => {
 
   it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => {
     const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" });
-    const interruptedDb = createDb(connectionString);
+    const interruptedDb = createDb(connectionString, { maxConnections: 6 });
     let joined = false;
     await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => {
       await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work");
@@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => {
         .toEqual({ acquired: false, reason: "owner_unverified" });
     }
     await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId));
-    expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed"))
+    expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed"))
       .toEqual({ acquired: true, value: "resumed" });
   });
 

server/src/services/native-runtime/runner-api.integration.test.ts

@@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}});
     const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId));
     const context = structuredClone(run.contextSnapshot ?? {});
     await prepareHeartbeatTaskContext(server.db, { run, agent,
-      task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null },
+      task: { issueRef: issue, issueContext: null, context, taskSession: null },
       policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
       services: { issuesSvc: { getAncestors: async () => [] },
         getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext },

server/src/services/native-runtime/status-decision-committer.ts

@@ -1,3 +1,4 @@
+import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js";
 import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
 import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js";
 import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js";
@@ -478,6 +479,7 @@ async function enqueueWake(input: {
     .limit(1)
     .then((rows) => rows[0] ?? null);
   if (existing) return existing.id;
+  await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId });
   const inserted = await input.tx
     .insert(agentWakeupRequests)
     .values({

server/src/services/native-status-reassessment-notifications.ts

@@ -1,10 +1,12 @@
 import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 
 export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string };
 
 export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) {
   await notifyStatusCardIssueChange(tx, scope.companyId);
+  await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined);
   return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment,
     JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId }));
 }

server/src/services/recovery/service.ts

@@ -1,3 +1,4 @@
+import { notifyRunDispatch } from "../run-dispatch-notifications.js";
 import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js";
 import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js";
 import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js";
@@ -26,7 +27,7 @@ import {
   decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint,
   LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode,
 } from "./legacy-continuation.js";
-import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js";
+import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js";
 import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js";
 import { instanceSettingsService } from "../instance-settings.js";
 import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js";
@@ -244,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = {
   runId?: string | null;
   companyId?: string | null;
   blockerIssueId?: string | null;
+  issueId?: string;
   source?: ResolvedDependencyWakeBackstopSource;
 };
 
@@ -2201,7 +2203,7 @@ export function recoveryService(
     return Boolean(budgetBlock);
   }
 
-  async function reconcileUnassignedBlockingIssues() {
+  async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) {
     const candidates = await db
       .select({
         id: issues.id,
@@ -2215,6 +2217,8 @@ export function recoveryService(
       .where(
         and(
           eq(issueRelations.type, "blocks"),
+          eq(issueRelations.companyId, issues.companyId),
+          opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
           inArray(issues.status, ["todo", "blocked"]),
           isNull(issues.assigneeAgentId),
           isNull(issues.assigneeUserId),
@@ -2716,6 +2720,7 @@ export function recoveryService(
     const now = new Date();
     const retryAt = readProviderQuotaRetryAt(input.latestRun, now);
     return database.transaction(async (tx) => {
+      await notifyRunDispatch(tx, { companyId: input.issue.companyId });
       const wakeup = await tx
         .insert(agentWakeupRequests)
         .values({
@@ -3316,6 +3321,7 @@ export function recoveryService(
           created = Boolean(enqueuedRun);
         } else {
           scheduledRun = await db.transaction(async (tx) => {
+            await notifyRunDispatch(tx, { companyId: input.issue.companyId });
             const wakeup = await tx
               .insert(agentWakeupRequests)
               .values({
@@ -3497,7 +3503,7 @@ export function recoveryService(
     return Boolean(run || wake);
   }
 
-  async function reconcileActiveRecoveryActions() {
+  async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) {
     const rows = await db
       .select({
         action: issueRecoveryActions,
@@ -3511,7 +3517,8 @@ export function recoveryService(
           eq(issues.companyId, issueRecoveryActions.companyId),
         ),
       )
-      .where(inArray(issueRecoveryActions.status, ["active", "escalated"]));
+      .where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
+        opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined));
 
     const result = {
       requeued: 0,
@@ -4523,14 +4530,20 @@ export function recoveryService(
   }
 
   async function reconcileStrandedAssignedIssues(opts?: {
+    companyId?: string;
+    issueId?: string;
+    onDeferredDeadline?: (at: number) => void;
     issueCreatedAtGte?: Date | null;
     skipExternalConversationSettlement?: boolean;
+    skipRelatedRecovery?: boolean;
   }) {
     const candidates = await db
       .select(recoveryIssueFields)
       .from(issues)
       .where(
         and(
+          opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
+          opts?.issueId ? eq(issues.id, opts.issueId) : undefined,
           isNull(issues.assigneeUserId),
           inArray(issues.status, ["todo", "in_progress", "in_review"]),
           or(
@@ -5711,6 +5724,7 @@ export function recoveryService(
               classification.baseBackoffMs *
               Math.pow(2, Math.max(0, consecutive - 1));
             if (elapsed < requiredDelay) {
+              opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay);
               result.skipped += 1;
               continue;
             }
@@ -5770,16 +5784,18 @@ export function recoveryService(
       }
     }
 
-    const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues();
-    result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
-    result.skipped += orphanBlockerRecovery.skipped;
-    result.issueIds.push(...orphanBlockerRecovery.issueIds);
+    if (!opts?.skipRelatedRecovery) {
+      const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts);
+      result.orphanBlockersAssigned = orphanBlockerRecovery.assigned;
+      result.skipped += orphanBlockerRecovery.skipped;
+      result.issueIds.push(...orphanBlockerRecovery.issueIds);
 
-    const activeRecovery = await reconcileActiveRecoveryActions();
-    result.continuationRequeued += activeRecovery.requeued;
-    result.escalated += activeRecovery.escalated;
-    result.skipped += activeRecovery.skipped;
-    result.issueIds.push(...activeRecovery.issueIds);
+      const activeRecovery = await reconcileActiveRecoveryActions(opts);
+      result.continuationRequeued += activeRecovery.requeued;
+      result.escalated += activeRecovery.escalated;
+      result.skipped += activeRecovery.skipped;
+      result.issueIds.push(...activeRecovery.issueIds);
+    }
     result.issueIds = [...new Set(result.issueIds)];
 
     return result;
@@ -5812,7 +5828,7 @@ export function recoveryService(
       source === "workspace.finalize"
         ? "workspace_finalize_reconciliation"
         : "issue_graph_liveness_reconciliation";
-    const useCursor = !opts?.blockerIssueId;
+    const useCursor = !opts?.blockerIssueId && !opts?.issueId;
 
     const queryCandidates = (afterIssueId: string | null) => {
       const filters = [
@@ -5822,6 +5838,7 @@ export function recoveryService(
         sql`${issues.assigneeAgentId} is not null`,
       ];
       if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId));
+      if (opts?.issueId) filters.push(eq(issues.id, opts.issueId));
       if (afterIssueId) filters.push(gt(issues.id, afterIssueId));
 
       if (opts?.blockerIssueId) {
@@ -6112,6 +6129,7 @@ export function recoveryService(
       // fallback below. Without it, a terminal issue named in the run context
       // snapshot would still terminalize the shared run and defeat the guard.
       runReferencedByActiveIssue?: boolean;
+      onDeferredDeadline?: (at: number) => void;
     },
   ): Promise<{ terminalized: boolean; status: string }> {
     // Act only on a run in "running" status. A "queued" run has no process yet,
@@ -6127,8 +6145,11 @@ export function recoveryService(
     // Another controller may own a sandbox run whose PID has no meaning on
     // this host. Its live lease owns both execution and finalization, even if
     // the issue is already terminal or this process has no in-memory handle.
-    if (await hasLiveLegacyController(db, run))
+    const controllerDeadline = await liveLegacyControllerExpiresAt(db, run);
+    if (controllerDeadline) {
+      options?.onDeferredDeadline?.(controllerDeadline.getTime());
       return { terminalized: false, status: run.status };
+    }
 
     const pid = run.processPid ?? null;
     const processGroupId = run.processGroupId ?? null;
@@ -6349,7 +6370,7 @@ export function recoveryService(
   // still claims to be live but can no longer reach a terminal status on its
   // own, so a stuck "running" run can no longer block the sweep. Idempotent and
   // safe: clears at most one row's worth of lock columns per candidate.
-  async function sweepStaleIssueLocks() {
+  async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) {
     const result = {
       cleared: 0,
       issueIds: [] as string[],
@@ -6366,7 +6387,9 @@ export function recoveryService(
       })
       .from(issues)
       .where(
-        sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
+        and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`,
+          opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined,
+          opts?.issueId ? eq(issues.id, opts.issueId) : undefined),
       );
 
     const referencedRunIds = [
@@ -6381,7 +6404,8 @@ export function recoveryService(
         ? await db
             .select()
             .from(heartbeatRuns)
-            .where(inArray(heartbeatRuns.id, referencedRunIds))
+            .where(and(inArray(heartbeatRuns.id, referencedRunIds),
+              opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined))
         : [];
     const runStatusById = new Map<string, string>();
     for (const row of runRows) runStatusById.set(row.id, row.status);
@@ -6392,8 +6416,15 @@ export function recoveryService(
     // the same run id in a stale lock column. The terminal reference alone must
     // not terminalize a run that an active issue still owns, so exclude these
     // runs from the issue-terminal authority below.
+    const ownershipReferences = opts?.issueId && referencedRunIds.length > 0
+      ? await db.select({ companyId: issues.companyId, status: issues.status,
+          checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and(
+          opts.companyId ? eq(issues.companyId, opts.companyId) : undefined,
+          or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)),
+        ))
+      : candidates;
     const runIdsReferencedByActiveIssue = new Set<string>();
-    for (const issue of candidates) {
+    for (const issue of ownershipReferences) {
       if (issue.status === "done" || issue.status === "cancelled") continue;
       for (const runId of [issue.checkoutRunId, issue.executionRunId]) {
         if (runId) runIdsReferencedByActiveIssue.add(runId);
@@ -6434,6 +6465,7 @@ export function recoveryService(
         referencingIssueTerminalStatus:
           issueTerminalStatusByRunId.get(row.id) ?? null,
         runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id),
+        onDeferredDeadline: opts?.onDeferredDeadline,
       });
       runStatusById.set(row.id, outcome.status);
       if (outcome.terminalized) result.terminalizedRunIds.push(row.id);
@@ -6537,6 +6569,8 @@ export function recoveryService(
     recordWatchdogDecision,
     scanSilentActiveRuns,
     reconcileStrandedAssignedIssues,
+    reconcileUnassignedBlockingIssues,
+    reconcileActiveRecoveryActions,
     reconcileLegacyContinuation,
     legacyRepairDispatchBlock,
     sweepStaleIssueLocks,

server/src/services/recovery/work.ts

@@ -0,0 +1,113 @@
+import { registerTaskWatchdogWork } from "../task-watchdog-work.js";
+import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm";
+import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db";
+import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js";
+import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js";
+import type { IssueRecoveryScope } from "../issue-recovery-notifications.js";
+import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
+
+type Entry = { id: string; companyId: string; dueAt: number };
+type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void };
+const PAGE_SIZE = 100;
+
+export function registerIssueRecoveryWork(
+  db: Db,
+  coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
+  owner: {
+    stranded(scope: Scope): Promise<unknown>;
+    orphanBlockers(scope: { companyId: string }): Promise<unknown>;
+    recoveryActions(scope: { companyId: string }): Promise<unknown>;
+    dependencies(scope: Scope): Promise<{ enqueueFailed: number }>;
+    locks(scope: Scope): Promise<unknown>;
+    watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>;
+    schedulingSuppressed(): Promise<boolean>;
+  },
+  canRun: () => boolean,
+) {
+  function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise<unknown>) {
+    const cursors = new Map<string, string>();
+    let enabled = false;
+    return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
+      queue,
+      reconstructionIntervalMs: 30_000,
+      beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
+      decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
+      matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)),
+      load: async scope => {
+        const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*";
+        if (!enabled) { cursors.delete(key); return []; }
+        const cursor = cursors.get(key);
+        const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and(
+          scope ? eq(issues.companyId, scope.companyId) : undefined,
+          scope?.issueId ? eq(issues.id, scope.issueId) : undefined,
+          cursor ? gt(issues.id, cursor) : undefined,
+          statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)),
+          statuses ? isNull(issues.hiddenAt) : undefined,
+          statuses ? isNull(issues.assigneeUserId) : undefined,
+          statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined,
+        )).orderBy(asc(issues.id)).limit(PAGE_SIZE);
+        const complete = rows.length < PAGE_SIZE;
+        if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
+        return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
+      },
+      check: async entry => {
+        if (!enabled) return null;
+        let next: number | null = null;
+        await check({ companyId: entry.companyId, issueId: entry.id,
+          onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } });
+        return next === null ? null : { ...entry, dueAt: next };
+      },
+    }, canRun);
+  }
+  function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise<unknown>) {
+    const cursors = new Map<string, string>();
+    let enabled = false;
+    return registerScopedDeadlineWork<Entry, IssueRecoveryScope>(coordinator, {
+      queue,
+      reconstructionIntervalMs: 30_000,
+      beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); },
+      decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope,
+      matches: (entry, scope) => !scope || entry.companyId === scope.companyId,
+      load: async scope => {
+        const key = scope?.companyId ?? "*";
+        if (!enabled) { cursors.delete(key); return []; }
+        const cursor = cursors.get(key);
+        const rows = kind === "actions"
+          ? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId })
+            .from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]),
+              scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined,
+              cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined))
+            .orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE)
+          : await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and(
+              inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId),
+              isNotNull(issues.createdByAgentId),
+              scope ? eq(issues.companyId, scope.companyId) : undefined,
+              cursor ? gt(issues.companyId, cursor) : undefined,
+              sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent
+                on dependent.id = r.related_issue_id and dependent.company_id = r.company_id
+                where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id}
+                and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`,
+            )).orderBy(asc(issues.companyId)).limit(PAGE_SIZE);
+        const complete = rows.length < PAGE_SIZE;
+        if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
+        return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete };
+      },
+      check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; },
+    }, canRun);
+  }
+  const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded);
+  const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers);
+  const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions);
+  const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => {
+    const result = await owner.dependencies(scope);
+    if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered");
+  });
+  const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks);
+  const watchdogs = registerTaskWatchdogWork(db, coordinator, {
+    reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()),
+  }, canRun);
+  const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs];
+  return { ready: Promise.all(workers.map(worker => worker.ready)),
+    wake() { for (const worker of workers) worker.invalidate(); },
+    async stop() { await Promise.all(workers.map(worker => worker.stop())); } };
+}

server/src/services/run-dispatch-notifications.ts

@@ -0,0 +1,11 @@
+import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
+import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
+
+export type RunDispatchScope = { companyId: string };
+
+export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) {
+  await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined);
+  const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*";
+  await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint);
+  await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint);
+}

server/src/services/sandbox-absence-proof.test.ts

@@ -1,6 +1,7 @@
 import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto";
 import { describe, expect, it, vi } from "vitest";
-import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js";
+import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt,
+  verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js";
 
 const { privateKey, publicKey } = generateKeyPairSync("ed25519");
 const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(),
@@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI
   PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
   PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) };
 const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000;
-function receipt(overrides: Record<string, unknown> = {}) {
-  const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url");
-  const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack",
-    ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
+function receipt(overrides: Record<string, unknown> = {}, purpose = "sandbox-absence", source: Record<string, unknown> = binding) {
+  const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url");
+  const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack",
+    ...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url");
   return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`;
 }
 describe("sandbox absence proof", () => {
@@ -45,3 +46,44 @@ describe("sandbox absence proof", () => {
     expect(fetchImpl).not.toHaveBeenCalled();
   });
 });
+
+describe("cleanup-only sandbox absence", () => {
+  const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(),
+    leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId };
+  const lease = { id: cleanup.leaseId, ...cleanup };
+  const signed = (overrides: Record<string, unknown> = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup);
+  it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => {
+    const source = { ...cleanup, heartbeatRunId };
+    const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) }));
+    expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl }))
+      .toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" });
+    const [url, init] = fetchImpl.mock.calls[0];
+    expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence");
+    expect(JSON.parse(init.body)).toEqual(source);
+    expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN);
+    expect(init.redirect).toBe("error");
+  });
+  it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => {
+    expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup);
+    for (const overrides of [
+      ...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })),
+      { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" },
+      { iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 },
+    ]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull();
+    expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull();
+    expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull();
+    expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull();
+  });
+  it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => {
+    for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }])
+      expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined();
+    for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease,
+      { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined();
+    expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined();
+    const fetchImpl = vi.fn();
+    expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined();
+    for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }])
+      expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined();
+    expect(fetchImpl).not.toHaveBeenCalled();
+  });
+});

server/src/services/sandbox-absence-proof.ts

@@ -7,8 +7,18 @@ const bindingSchema = z.object({
   companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(),
   leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"),
   providerLeaseId: z.string().uuid(),
-});
+}).strict();
+const cleanupBindingSchema = z.object({
+  purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(),
+  leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(),
+  provider: z.literal("daytona"), providerLeaseId: z.string().uuid(),
+}).strict();
 export type SandboxAbsenceBinding = z.infer<typeof bindingSchema>;
+export type SandboxCleanupAbsenceBinding = z.infer<typeof cleanupBindingSchema>;
+export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & {
+  v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string;
+  absent: true; iat: number; exp: number;
+};
 export type SandboxAbsenceProof = SandboxAbsenceBinding & {
   v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string;
   absent: true; iat: number; exp: number;
@@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & {
 
 export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding,
   env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null {
+  const parsed = bindingSchema.safeParse(binding);
+  return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null;
+}
+
+export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding,
+  env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null {
+  const parsed = cleanupBindingSchema.safeParse(binding);
+  return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null;
+}
+
+function verifyReceipt(receipt: string, binding: Record<string, unknown>, purpose: string,
+  env: NodeJS.ProcessEnv, now: Date): Record<string, unknown> | null {
   try {
-    const parsed = bindingSchema.safeParse(binding);
-    if (receipt.length > 16_000 || !parsed.success) return null;
+    if (receipt.length > 16_000) return null;
     const parts = receipt.split(".");
     if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null;
     const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8"));
-    if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null;
+    if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null;
     if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null;
-    const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof;
+    const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8"));
     const seconds = Math.floor(now.getTime() / 1000);
-    if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1"
+    if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1`
       || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true
-      || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding])
+      || Object.keys(binding).some(key => proof[key] !== binding[key])
       || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30
       || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null;
     return proof;
@@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs
 export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: {
   env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date;
 } = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> {
+  const parsed = bindingSchema.safeParse(binding);
+  if (!parsed.success) return null;
+  const receipt = await requestReceipt(parsed.data, options);
+  if (!receipt) return null;
+  const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now);
+  return proof ? { proof, receipt } : null;
+}
+
+/** Cleanup-only observation: never grants continuation authority to a run. */
+export async function inspectCloudSandboxCleanupAbsence(lease: {
+  id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null;
+  provider: string | null; providerLeaseId: string | null;
+}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) {
+  const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId,
+    environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId,
+    provider: lease.provider, providerLeaseId: lease.providerLeaseId });
+  if (!parsed.success) return undefined;
+  const receipt = await requestReceipt(parsed.data, options);
+  if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined;
+  return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const };
+}
+
+async function requestReceipt(binding: Record<string, unknown>, options: {
+  env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch;
+}): Promise<string | null> {
   const env = options.env ?? process.env;
   const context = getCloudStackContext(env);
   const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET;
-  const parsed = bindingSchema.safeParse(binding);
-  if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null;
+  if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null;
   try {
     const origin = new URL(context.cloudOrigin);
     const host = new URL(`https://${context.primaryHost}`);
@@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding,
     const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, {
       method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000),
       headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" },
-      body: JSON.stringify(parsed.data),
+      body: JSON.stringify(binding),
     });
     if (!response.ok) return null;
     const payload = await response.json() as { receipt?: unknown };
     if (typeof payload.receipt !== "string") return null;
-    const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now);
-    return proof ? { proof, receipt: payload.receipt } : null;
+    return payload.receipt;
   } catch { return null; }
 }

server/src/services/scheduled-retry-cancellation-notifications.ts

@@ -1,7 +1,9 @@
+import { notifyRunDispatch } from "./run-dispatch-notifications.js";
 import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
 
 export type ScheduledRetryCancellationScope = { companyId: string; runId?: string };
 
 export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) {
+  await notifyRunDispatch(transaction, scope);
   await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*");
 }