diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 7799ead39ed..a90cd328622 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -681,3 +681,46 @@ and are atomically deleted before code exchange. The `code_verifier` column hold this non-secret binding for this namespace; Slack bot installation does not use PKCE. Removal and manual recovery invalidate outstanding attempts under the same credential-mutation lease used by configuration. + + +## Transaction-aware delivery work signals + +Five existing delivery queues use process-local work signals to avoid empty +polling. Register work **before** writing a queue row, on that row's transaction: + +```ts +await db.transaction(async tx => { + await signalDatabaseWork(tx, "queue-topic"); + await tx.insert(existingQueueTable).values(row); +}); +``` + +`createDb` instruments transaction callbacks and nested savepoints. Subscribers +receive intent immediately and settlement only after the outer transaction +finishes. Caller-owned transactions therefore need no extra post-commit wrapper. +Dedicated clients from `withDedicatedDbConnection` share the owner's signal +scope. No schema changes, triggers, dedicated listener connection, or periodic +queries are installed by this API. + +The first registered write in a transaction fetches `pg_current_xact_id()`. +Awaiting registration is required: a failure at that point must prevent the +queue write. Unrelated transactions add no queries. If the transaction rejects, +the coordinator probes `pg_xact_status(xid)` through the root pool before scanning +the queue. An in-progress transaction remains unresolved even when the queue +currently looks empty. Committed/aborted results permit reconciliation; NULL +means PostgreSQL has discarded an old, no-longer-active transaction's outcome. +The existing durable queue supplies the work in either case. Probe failures +retain the idle hold and schedule another attempt. Work signals never replay +queries or convert a failed database operation into success. + +This uses PostgreSQL's transaction-information functions (this path requires +PostgreSQL 14+; embedded PostgreSQL uses 18). See the +[PostgreSQL transaction information documentation](https://www.postgresql.org/docs/18/functions-info.html#FUNCTIONS-PG-SNAPSHOT). +The process-local intent survives coordinator replacement, but not process exit. +Startup scans recover committed queue rows. Independent DB clients, other +processes, late transactions from an old process, and database failover need an +explicit ownership/wake protocol; these signals are not cross-process messaging. +Use the same root client for in-process writers. New queue insertion paths must +register before writing, or they can remain unseen until the next startup or +another notification. Tests should verify both the producer and its outer +transaction boundary. diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index f6fcad77477..35fcb16bc1a 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1290,6 +1290,16 @@ reporting and forwarding callbacks preserve construction order for the other heartbeat modules. Existing public helpers remain re-exported by `heartbeat.ts`. Keep lifecycle policy changes separate from this extraction and adapter execution. +Run cancellation and cleanup are in `server/src/services/heartbeat/run-control.ts`. +It owns Stop, agent and budget cancellation, environment lease release, and +resumption of saved comments after execution stops. `createHeartbeatRunControl` +binds the database, lifecycle and admission callbacks, and shared executor +barriers without starting work. The service retains the process-wide execution +and cancellation maps so scheduler and route instances observe the same owners. +Forwarding callbacks preserve construction order for budget, retry, recovery, +and queue services. Existing public helpers remain re-exported by `heartbeat.ts`. +Keep cancellation policy changes separate from this extraction and execution. + ## Wake Context Delivery Built-in adapters deliver wake context through the run prompt, including structured diff --git a/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md new file mode 100644 index 00000000000..cc7df120edd --- /dev/null +++ b/doc/plans/2026-10-08-instance-sleep-and-scheduler-inventory.md @@ -0,0 +1,867 @@ +# Instance sleep and scheduler inventory + +Date: 2026-10-08 + +Status: Inventory with first delivery-coordinator implementation; full idle sleep remains open + +Code baseline: `0ac194450` + +Paperclip should release idle database connections and allow inactive customer +instances to sleep without losing scheduled work, incoming events, accounting, +or recovery. Thousands of individually hosted instances make empty polling a +material cost even when each query is cheap. + +**Architectural direction: triage each loop before choosing its replacement.** +Work caused by a committed change or incoming event should run from that event. +Work caused by the passage of time belongs in one in-process, instance-wide +scheduler. A later external-waker adapter publishes only the stack's next wake +time. Routine definitions and execution decisions remain inside the stack. + +Immediate delivery does not have to pass through the scheduler. Its delayed +retry does. The same service can contain both categories; classify its work +paths rather than treating its current polling loop as one indivisible job. + +The database already closes idle pooled connections after 60 seconds and opens +new connections on demand. The primary work is eliminating unnecessary queries +and coordinating background work, connections, and hosting sleep. + +## First slice: transaction-aware delivery notifications and shared scheduling + +Updated: 2026-10-09. This implementation covers five existing durable queues: + +- [x] JOB-04: feedback exports register intent before the export write. +- [x] JOB-22: completion receipts register intent inside their transaction, including caller-owned and nested transactions. No extra post-commit action is needed from callers. +- [x] JOB-23: connection continuations register intent before enqueue. +- [x] JOB-25: question answers register intent before enqueue. +- [x] JOB-24: tool receipts and review resolution register intent before enqueue. +- [x] `createDb` centrally defers delivery wakes until the outer transaction settles. Dedicated child pools share the owning client's signal scope. +- [x] One scheduler owns pending-work and failure retries. Empty, settled queues have no recurring deadline or database scan. +- [x] Unresolved local writes prevent idle admission. Recovery probes PostgreSQL's transaction status before scanning the existing queue and clears the hold once both are settled. +- [x] Tests cover nested commits, rollback, late commit/abort after a lost client response, concurrent writes during a sweep, worker replacement, restart recovery, drain/standby, and shutdown. +- [ ] Prove the complete instance writer-admission and idle/resume boundary across all other work. +- [ ] Migrate other actual scheduled work and expose the whole instance's earliest deadline to an external waker. +- [ ] Verify actual app and database sleep on the hosting providers. + +Existing delivery rows remain authoritative; no tables or migrations are added. +A producer adds one awaited `notifyDeliveryWork(tx, queue)` before its write. +`createDb().transaction()` and nested savepoints are instrumented centrally. The +first opted-in write fetches the outer transaction's XID; unrelated transactions +add no SQL. A rejected transaction with a known XID is reconciled with +`pg_xact_status` before the queue scan. An empty scan cannot discharge an +in-progress transaction. Committed, aborted, or aged-out XIDs can be discharged +after scanning. A failed status query retains the hold and a retry deadline. + +The scheduler retains startup sweeps, retry deadlines while durable work exists, +and retries after failures. Idle drain permits transaction and queue checks but +does not start deliveries; warm standby suppresses both. There is no +unconditional 60-second recovery pass. +Feedback flushes remain serialized and cancellable, with a 30-second upload +limit. Vote routes return after saving rather than waiting for upload. Tool +receipts awaiting review or execution still retain their retry cadence. + +**Scope:** notifications and unresolved-XID tracking are process-local to a +`createDb` owner and its dedicated child pools. They do not observe direct SQL, +an independently created client, or another process. Such writers need an +explicit wake/reconciliation integration before this mechanism can support +them. Startup scans recover already committed durable rows; they do not prove +that transactions still running in an old process have finished. Cross-process +handoff/fencing is separate work, as is PostgreSQL failover during uncertainty. + +This removes these five empty polling loops; it does not establish whole-stack +sleep. `nextWakeAt` covers only registered queues. Database connection teardown, +other background loops, full idle reconciliation, and external wake registration +remain separate work. + +Implementation: `packages/db/src/work-signals.ts`, +`server/src/services/work-scheduler.ts`, `delivery-work-coordinator.ts`, and +`delivery-work-notifications.ts`. + +## Using this checklist + +Items have stable IDs so they can become separate issues or PRs. All boxes start +unchecked: an existing mechanism or a recommendation is not proof that the +sleep journey has passed. Mark an item complete only after its completion +criteria have evidence, and append the issue/PR and verification result to it. +Record deferrals with a reason and the capability that must remain awake. + +Source links identify files in the audited revision. Function names and stated +cadences are navigation aids; recheck them when implementing a later revision. +Cadences describe ordinary claimed-instance operation. Warm-standby, idle-drain, +configuration, or feature gates can suppress individual paths. This inventory +does not establish the actual deployed fleet configuration. + +## Triage before implementation + +Use this question for every operation: **If no new request or event arrives, +does this operation still need to happen at a particular future time?** + +- **Yes: clock-driven.** Register that deadline with the internal scheduler. + Examples are a routine at 09:00, a retry after backoff, and a session expiry. +- **No: event-driven or on demand.** Invoke it from the owning committed mutation, + verified external event, or user request. An empty queue is not a scheduled + task. Examples are delivering a newly committed answer and refreshing a UI + after an issue changes. + +Many existing sweeps combine both. Separate their immediate dispatch from their +retry/timeout/recovery paths. Persistent transports and active-operation +keepalives are lifecycle requirements, not idle business schedules; their +ownership and shutdown need separate treatment. + +The tables below are a first-pass mechanism triage grounded in the inventory. +They select where to investigate first, not permission to delete a recovery +path. Detailed checklist items remain the implementation and verification record. + +### Work that genuinely needs a clock + +| Checklist items | Reason time matters | What belongs in the scheduler | +| --- | --- | --- | +| JOB-08 Agent timer heartbeats and issue monitors | The user explicitly requested future checks. | The next enabled heartbeat/monitor deadline; none when unconfigured. | +| JOB-09 Routine cron triggers | A routine must run even without new incoming activity. | Existing `nextRunAt`, retaining catch-up and timezone semantics. | +| JOB-07 Plugin scheduled jobs | A plugin may declare real recurring work. | Declared job deadlines; plugin load/configuration events update registration. | +| JOB-29 Backups | A backup guarantee has a time requirement. | The next backup if this instance owns it; otherwise the explicit external backup owner. | +| JOB-18, JOB-19 Login and setup expiry | An abandoned session/resource must expire even if nobody returns. | Actual expiry and failed-cleanup retry times, only while relevant sessions exist. | +| JOB-11, JOB-21 Decision and secret/token expiry | Decision deadlines and some deletion/retention obligations are time-based. | Required expiry/scrub deadlines. Check validity synchronously on use regardless of whether physical cleanup ran. | +| JOB-10 Decision retention | An unchanged item can become old enough to archive. | The next actual retention deadline; immediate change handling and notification delivery are event-driven. | +| JOB-12, JOB-13 External objects and status cards | Some configured freshness/update policies require future work. | Only time-based refresh/evaluation deadlines; reactive updates use events. | +| JOB-27, JOB-28 Retries and watchdogs | A retry delay or absence of expected progress can itself require action. | Concrete retry, lease-expiry, monitor, or watchdog deadlines for existing work. | + +### Work whose normal path should stop polling + +| Checklist items | Preferred normal trigger | Clock-driven remainder or qualification | +| --- | --- | --- | +| JOB-04 Feedback exports | Committed export enqueue. | Backoff after delivery failure and durable recovery of outstanding exports. No empty five-second scan. | +| JOB-22 Chat completion delivery | Committed task completion/update. An existing postcommit fast path is already present. | Delivery retries and recovery; the current fast path is best-effort, so prove replacement reliability before removing its sweep. | +| JOB-23 Connection continuation delivery | Committed connection availability/change and pending continuation enqueue. | Failed-delivery retry/claim recovery. | +| JOB-25 Question response delivery | Committed user answer. | Retry/claim expiry while delivery is outstanding. | +| JOB-24 Tool action reviews and deliveries | Committed approval/rejection or action enqueue. | Review expiry and delivery retries. | +| JOB-26 Cost accounting | Arrival or durable capture of a usage receipt/result. | Recovery of unresolved receipts/reservations and failed writes. Accounting debt cannot be dropped. | +| JOB-05 MCP and Dot events | Committed source event or subscription change. | Delivery retries and required expiry cleanup. | +| JOB-01 Chat reconciliation | Incoming message, committed publication, or endpoint configuration change. | Split its lanes: delivery retry and missed-event recovery are temporal; socket ownership is NET work. | +| JOB-02 Email reconciliation | Verified webhook, outgoing send, or endpoint configuration change. | Retry and provider catch-up guarantees; socket reception and catch-up are NET-01/NET-02. | +| JOB-03 Browser-use cleanup | Browser/session lifecycle change. | Register actual idle/expiry/reconciliation deadlines while sessions exist; active leases remain protected. | +| JOB-06 Execution-control reconciliation | Execution state transitions and committed control requests. | Timeout, lease-expiry, abandoned-operation, and failed-delivery recovery. Split the lanes before changing cadence. | +| JOB-14 GitHub events and continuity | Verified webhook and connection lifecycle change where supported. | Credential deadlines; bounded polling only for providers/events without adequate push or replay guarantees. | +| JOB-15 Merge confirmations | Verified merge event for an outstanding candidate. | Bounded fallback only while candidates exist and webhook/replay coverage is insufficient. | +| JOB-16 Terminal workspaces | Committed terminal task/tree transition. | Cooldown, failed-cleanup retry, and recovery of outstanding cleanup. | +| JOB-17 Sandbox cleanup | Resource release or failed acquire/release. | Retry/backoff and provider/fleet orphan recovery. | +| JOB-20 Connection health | Use of the connection or configuration change. | Proactive checks only where a defined freshness guarantee requires them. | +| JOB-10, JOB-11 Decision effects and notifications | Committed decision/relevant source mutation. | Expiry/retention stays in the clock-driven category. | +| JOB-12, JOB-13 Reactive external objects/cards | Verified provider event or relevant local mutation. | Configured time-based refresh stays in the clock-driven category. | +| JOB-27, JOB-28 Queued runs, assignments, and dependencies | Committed enqueue, assignment, terminal result, or dependency transition. | Time-based recovery/watchdogs stay in the clock-driven category. | +| AUX-01, AUX-03 Telemetry and plugin log flushing | First buffered entry activates a bounded flush. | Batch delay and failed-delivery retry while nonempty; no empty heartbeat. | +| AUX-04 Import spools and runtime status expiry | Spool creation/completion, startup, or access to an expiring in-memory value. | Schedule only required file-retention/cleanup deadlines. Lazy expiration can handle noncontractual memory cleanup. | +| UI-01, UI-02 Stable UI data | Existing live events, initial read, foreground return, and reconnect reconciliation. | Preserve bounded fallback during a genuine realtime outage or active operation. Reconcile subscription gaps because live events are not replayed. | +| UI-03, UI-04, UI-06 Recovery, app updates, and setup | Foreground/online/user activity and bounded setup state. | Client-side retry/update deadlines where needed; these do not become tenant server jobs. | + +JOB-30 is removal of the aggregate polling wrapper after its constituents are +handled, not a new scheduled job. DB-01–DB-03 concern the connection pool; +DB-04/DB-05 concern probes. NET-01–NET-07, UI-05, and AUX-05 concern transport or +plugin lifetime. AUX-02/AUX-06 require checking actual exporter/SDK traffic. +SCH, WAKE, SAFE, HOST, and TEST items are infrastructure, policy, or verification +work rather than business jobs. This keeps every inventory item accounted for +without making the scheduler responsible for every source of activity. + +### Qualification before removing a poll + +- [ ] **TRI-01 — Identify the real trigger and every producer.** For each + event-driven candidate, list the committed mutations/external events that + create work, including alternate API paths and recovery writers. **Complete + when:** every supported producer has an owned dispatch path and the service + needs no scan to discover normal new work. + +- [ ] **TRI-02 — Separate first delivery from delayed recovery.** Record which + paths are immediate, which have a concrete due time, and which depend on an + external provider's push capabilities. **Complete when:** each migrated lane + has an explicit trigger and its remaining deadlines are enumerated. + +- [ ] **TRI-03 — Prove durable delivery before retiring its backstop.** A + best-effort callback after commit is not a durable queue consumer. Cover + commit-before-signal crashes, lost signals while the process stays alive, + handler failures, concurrent producers, and restart. **Complete when:** an + outstanding record cannot become invisible indefinitely. Retain a bounded, + explicitly justified reconciliation path until this is proven; document its + wake cost rather than assuming startup recovery covers every failure. + +- [ ] **TRI-04 — Order work by avoidable idle traffic and migration risk.** + Record expected idle SQL/network reduction and the smallest focused proof for + the next item. **Complete when:** the next PR has one clear boundary and does + not require the entire scheduler or all connector lifecycles to be finished. + +### Initial implementation shortlist + +1. **DB-04/DB-05: health and probe separation.** A self-contained source of + avoidable demand, independent of building a scheduler. Verify actual callers + so the new endpoint is used. +2. **JOB-04: event-driven feedback exports.** A contained first dispatch + migration. Triage its enqueue, failed-export behavior, and recovery first; + put only actual retry/batch deadlines in scheduling infrastructure. +3. **JOB-22/JOB-25: completion and answer delivery.** Reuse existing postcommit + paths where possible, after proving that durable delivery can replace the + sweep rather than merely supplement it. +4. **JOB-01/JOB-02/JOB-03: frequent empty chat, email, and browser scans.** Large + baseline benefit, but split the component lanes and preserve configured + transports, expiry, and recovery. Resource absence must be invalidated when + a new resource is committed. +5. **SCH with JOB-09: routine scheduling as the first true clock-driven slice.** + Existing `nextRunAt` makes the real scheduler requirement concrete. Add other + temporal paths and then the external-waker adapter. + +This order supersedes starting by routing the feedback polling loop through a +generic scheduler. Event-driven improvements can ship before that scheduler is +complete. The resulting service still needs an owned way to retry failures. + +## Provider behavior and cost model + +| Layer | Documented behavior | Consequence | +| --- | --- | --- | +| Neon | Normally suspends after five minutes without active queries. Idle-in-transaction connections count as active; ordinary idle connections do not necessarily block suspension. New connection requests can reset inactivity. | Stop empty SQL polling and connection churn; do not force-close active transactions. | +| Railway | Detailed Serverless documentation specifies five minutes without outbound packets, sampled so actual sleep is roughly 5–10 minutes. Private-network traffic and responses count. Enabling Serverless requires a deployment to affect the container. | Every recurring source of packets matters. Even a DB-free endpoint can keep the app awake when repeatedly called externally. | +| Unikraft | `on` waits for connections/HTTP requests to finish; `idle` can suspend inactive TCP connections. Active HTTP requests prevent suspension. Stateful mode resumes a snapshot. | Quiesce connections deliberately and explicitly protect background work from premature suspension. | + +Provider sources: [Neon compute lifecycle](https://neon.com/docs/introduction/compute-lifecycle), +[Neon compute management](https://neon.com/docs/manage/endpoints/), +[Railway Serverless](https://docs.railway.com/deployments/serverless), +[Unikraft scale to zero](https://unikraft.com/docs/features/scale-to-zero). + +Neon usage is reported in CU-seconds. The transcript's five-minute concern is +primarily an idle-timeout cost, not a universal billing quantum. With a +five-minute timeout, a trivial hourly check creates roughly an 8.3% active-time +floor before useful work; checking every five minutes can leave compute +continuously active. This illustration assumes the default timeout and excludes +other traffic, processing time, and storage charges. Verify actual endpoint +settings and the applicable plan. See [Neon usage calculations](https://neon.com/docs/introduction/usage-calculations). + +## 1. Internal scheduler for true deadlines + +- [ ] **SCH-01 — Establish the instance-wide scheduler boundary.** Route + application-owned delayed jobs, recurring jobs, retry deadlines, and required + expiry/cleanup/maintenance deadlines through one scheduler shared across all + companies. Immediate event/queue dispatch remains in its owning service; + running-work tracking is shared with sleep admission. Keep domain handlers and + their authorization/atomic claims in their existing services. **Complete + when:** one scheduling interface supports true temporal obligations, company + scope remains explicit, and self-hosted operation requires no external service. + +- [ ] **SCH-02 — Model work and deadlines explicitly.** Track a stable job/lane + identity, company scope where applicable, due time, pending/running state, + retry policy, and whether running work blocks sleep. Accept running-work + information from event-driven handlers without forcing their dispatch through + the scheduler. Distinguish scheduled + work, immediate queued work, waiting for an external event, and passive + configuration/history. **Complete when:** an empty lane has no recurring SQL + check, and the scheduler can explain outstanding work and the earliest due + time without querying the database on every timer tick. + +- [ ] **SCH-03 — Reconstruct state and handle committed changes.** Seed the + scheduler from existing durable domain records on startup; use committed + mutations to register, reschedule, cancel, or activate work. In-memory hints + cannot be the sole record of an obligation. Define how all supported writers + invalidate hints; initially retain the existing single-process boundary. + **Complete when:** restart and a crash between commit and notification recover + work, and creating an earlier deadline updates the scheduler immediately. + +- [ ] **SCH-04 — Implement local scheduling and catch-up.** Arm the nearest + necessary local timer and run due handlers with bounded concurrency and + per-lane overlap protection. Preserve each domain's timezone, missed-run, + coalescing, priority, and retry semantics. **Complete when:** delayed callbacks, + clock changes, restart, and duplicate activation cannot silently lose work or + bypass existing execution claims. + +- [ ] **SCH-05 — Expose an aggregate next wake time.** Compute the earliest + required deadline across the whole instance, including retries, cleanup, + accounting, and maintenance. Represent no scheduled work as no alarm; represent + unknown obligations as a sleep blocker. **Complete when:** adding, moving, + removing, or completing the earliest item correctly changes the aggregate. + +- [ ] **SCH-06 — Integrate running-work ownership and drain.** Reuse + [task admission](../../server/src/services/task-admission.ts), + [idle admission](../../server/src/middleware/idle-admission.ts), and the + [existing sleep protocol](../idle-sleep-safety.md). Track handler completion + even after an HTTP response or client disconnect. **Complete when:** draining + prevents new scheduled admissions and waits for accepted work without losing + fire-and-forget work or mutations racing the final check. + +- [ ] **SCH-07 — Classify transport and SDK timer exceptions.** Business + scheduling belongs to the scheduler. Protocol pings, socket deadlines, and + active-operation lease renewals can remain local to their transports, but + must have explicit start/stop ownership and participate in sleep blocking. + Browser timers use a separate foreground/activity policy. **Complete when:** + exceptions are inventoried and owned; no hidden independent timer can renew + an idle connector indefinitely after its scheduler lane is stopped. + +- [ ] **SCH-08 — Make scheduler migration incremental and observable.** Migrate + each inventory item below independently. Report pending work, next deadline, + active holds, and unknown/unmigrated blockers through local diagnostics. + Avoid periodic SQL or remote reporting merely to observe inactivity. + **Complete when:** an operator can explain why a stack is awake and a migrated + lane can be rolled back without duplicate processing. Preserve the telemetry + review rules if any first-party event contract changes. + +## 2. External waker adapter later + +The minimal external model is `(stack, nextWakeTime)`. Operational delivery may +also carry a generation/idempotency value and delivery status. It must not +require copying every routine, agent configuration, or permission into Cloud. +The internal scheduler remains authoritative about what to execute. + +```mermaid +flowchart LR + D[Temporal obligations and retry deadlines] --> S[Instance scheduler] + S --> L[Local timer while awake] + S -->|stack, nextWakeTime, generation| W[External waker adapter] + W --> G[Wake-aware ingress or provider wake] + U[User request or webhook] --> G + G -->|Reconcile due deadlines on wake| S + G -->|Verified request or event| H + L --> H[Existing domain handlers] + S --> H +``` + +- [ ] **WAKE-01 — Define the adapter and local-only behavior.** Support + setting/replacing and cancelling the next alarm. No external adapter is + required to use the internal scheduler. **Complete when:** timed work remains + local and the instance remains awake if no durable external wake is available; + introducing the internal scheduler alone never authorizes sleeping through a + deadline. + +- [ ] **WAKE-02 — Make alarm publication durable and acknowledge it before sleep.** + Persist publication intent/generation or use an equivalent recoverable + handshake. Recheck that the acknowledged alarm still covers the earliest + deadline before sleeping. **Complete when:** failure, lost responses, and a + concurrently moved-earlier deadline keep the instance awake until covered; + local scheduling continues during external-waker outages. + +- [ ] **WAKE-03 — Handle duplicate, stale, delayed, and missed wake delivery.** + Fence stale generations, retry failed delivery, reconcile due work after + wake, and retain domain-level atomic claims. **Complete when:** a repeated + doorbell does not duplicate work, and missed deadlines follow each domain's + explicit catch-up policy. Do not claim exactly-once network delivery. + +- [ ] **WAKE-04 — Serialize ingress, sleep, deploy, and wake.** Buffer or retry + incoming requests while draining; authenticate external control and preserve + webhook verification and acknowledgement deadlines. The host must bind its + final stop decision to the correct process/provider generation. **Complete + when:** requests arriving during sleep entry are neither lost nor admitted + into a process being stopped. Multi-replica support requires shared fencing. + +- [ ] **WAKE-05 — Implement and qualify provider adapters.** Evaluate Unikraft + scheduled `start` operations and a shared durable delayed-delivery mechanism + for Railway. Verify that provider resume actually invokes due-work processing, + including stateful resume without process startup. **Complete when:** an + isolated instance wakes for its earliest deadline and no waker polls sleeping + tenant databases. See [Unikraft scheduled wake-ups](https://unikraft.com/docs/features/cron-jobs). + +- [ ] **WAKE-06 — Protect active background execution on the host.** Connect + active-work holds to the provider's sleep controls. Unikraft exposes a + reference-counted disable mechanism; account for its signal delay. **Complete + when:** startup, transactions, agent control, finalization, and cleanup cannot + be suspended halfway through accepted work. See [Unikraft application sleep controls](https://unikraft.com/docs/tutorials/scale-to-zero-triggers). + +- [ ] **WAKE-07 — Validate wake concentration and cost.** Capacity-test concurrent + scheduled wakes, apply bounded dispatch, and use jitter only where semantics + permit. **Complete when:** scheduler backlog/delivery delay is observable and + a midnight burst does not silently skip customer jobs. Avoid periodically + waking all tenants just to discover whether any work exists. + +## 3. Database and health + +- [ ] **DB-01 — Qualify the existing idle pool behavior.** + [client.ts](../../packages/db/src/client.ts) already defaults + `DATABASE_IDLE_TIMEOUT_SECONDS` to 60; `0` disables reaping. Keep the existing + Drizzle handle and lazy reconnection; evaluate 30 seconds after pollers become + quiet. **Complete when:** idle connections disappear and a subsequent read, + write, and transaction work after actual provider suspension without per-route + lifecycle code. Existing tests are in + [client.test.ts](../../packages/db/src/client.test.ts). + +- [ ] **DB-02 — Preserve disconnect and transaction safety.** `.end()` permanently + shuts down the pool and must not become an idle operation. Reconnection does + not authorize replaying an interrupted write. **Complete when:** existing + transaction isolation after disconnect and explicitly idempotent retry rules + still hold; ambiguous commits are not blindly replayed. See + [database retry behavior](../DATABASE.md#connection-loss-and-retries). + +- [ ] **DB-03 — Cover every client and socket.** Better Auth shares the main DB; + the optional plugin migration pool uses the same idle configuration. Utility, + backup, and dedicated advisory-lock connections are bounded work. The audited + postgres.js version has a 60-second TCP keepalive; keepalive is not SQL. + **Complete when:** all pools can reach zero when idle, reserved/active work is + protected, and no new persistent LISTEN or health-ping connection is introduced + as a substitute for polling. Sources: + [DB client](../../packages/db/src/client.ts), + [auth adapter](../../server/src/auth/better-auth.ts), + [backup library](../../packages/db/src/backup-lib.ts). + +- [ ] **DB-04 — Split process liveness from DB readiness.** Claimed-instance + [health](../../server/src/routes/health.ts) executes `SELECT 1`; auth and deep + checks may perform additional queries. Install cheap liveness before session + and actor resolution. **Complete when:** normal liveness produces no SQL, + explicit deep readiness still tests DB health, and signed claim/bootstrap + mutations retain their existing validation and tracking. + +- [ ] **DB-05 — Audit external probes and configured endpoints.** Inspect the + fleet gateway, uptime monitors, deployment checks, DB pool settings, and Neon + suspend configuration. The [ECS example](../../docker/ecs-task-definition.json) + probes every 30 seconds; it is not evidence of Railway fleet behavior. + **Complete when:** sleeping is an expected state, repeated checks do not wake + tenants or reconnect to Neon, and readiness errors remain distinguishable + from intentional sleep. Railway's built-in health checks are deployment-only: + [provider documentation](https://docs.railway.com/deployments/healthchecks). + +## 4. Server scheduling migration inventory + +Apply the triage above to each operation within these services. Remove polling +from event-driven normal paths and route only real deadlines through the internal +scheduler. Preserve durable delivery, startup recovery, and domain-level claims. +Longer polling intervals alone do not complete an item. + +- [ ] **JOB-01 — Chat reconciliation.** Current: every **1s**, with multiple + runtime/delivery/publication lanes and SQL even without connectors. Change: + disarm empty lanes and activate on committed events or deadlines. **Complete + when:** a tenant with no chat obligations issues no chat SQL and new inbound, + publication, and recovery work still runs. Sources: + [app.ts](../../server/src/app.ts), + [chat-channels.ts](../../server/src/services/chat-channels.ts). + +- [ ] **JOB-02 — Email reconciliation.** Current: every **1s**, querying active + endpoints even when none exist. Change: initialize resource/pending state and + update on endpoint changes, webhook receipt, sends, and retries. **Complete + when:** empty mail state produces no polling SQL and new work activates the + worker without restart. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **JOB-03 — Browser-use cleanup.** Current: every **3s** with due-session + queries even when empty. Change: register actual session reconciliation/expiry + deadlines. **Complete when:** no sessions means no queries, while active leases, + expiry, and failed cleanup remain recoverable. Sources: + [app.ts](../../server/src/app.ts), + [browser-use.ts](../../server/src/services/browser-use.ts). + +- [ ] **JOB-04 — Feedback exports.** Current: every **5s**, querying the queue + even without a configured share client. Change: committed enqueue signals and + durable retries. **Complete when:** empty exports are quiet and queued exports + survive restart and temporary delivery failure. Sources: + [app.ts](../../server/src/app.ts), + [feedback.ts](../../server/src/services/feedback.ts). + +- [ ] **JOB-05 — Public MCP and Dot event dispatch.** Current: **2s** per started + dispatcher, including SQL-backed enablement, expiry, and delivery scans. + Change: cache capability state and schedule delivery/retry/expiry work. + **Complete when:** disabled/empty dispatchers are DB-free and pending events + retain subscription scope, expiry, and replay safety. Source: + [events.ts](../../server/src/services/public-mcp/events.ts). + +- [ ] **JOB-06 — Execution-control reconciliation.** Current: independent + **15s** recovery/delivery lanes, including when the main heartbeat is disabled. + Change: activate lanes from durable pending work and deadlines. **Complete + when:** empty lanes are quiet and abandoned finalizations, replacements, + continuations, status/disposition delivery, and login expiry still recover. + Sources: [index.ts](../../server/src/index.ts), + [execution-control-deadline.ts](../../server/src/services/execution-control-deadline.ts). + +- [ ] **JOB-07 — Plugin job scheduling.** Current: **30s** SQL even without jobs. + Change: register nearest `nextRunAt` and signal registration/update/removal. + **Complete when:** empty plugins/jobs produce no scheduler SQL and due jobs + retain atomic ownership. The plugin job coordinator itself is event-driven, + not another periodic loop. Sources: + [plugin-job-scheduler.ts](../../server/src/services/plugin-job-scheduler.ts), + [plugin-job-coordinator.ts](../../server/src/services/plugin-job-coordinator.ts). + +- [ ] **JOB-08 — Agent timer heartbeats and issue monitors.** Current: **30s** + agent/monitor scans; agent-level disabled heartbeats still require discovery. + Change: register each relevant deadline and aggregate locally. **Complete + when:** no timer work means no queries, and due work preserves pause, budget, + authorization, and atomic wake admission. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `tickTimers`. + +- [ ] **JOB-09 — Routine cron triggers.** Current: **30s** due-trigger query. + Change: use existing `nextRunAt` in the internal scheduler. **Complete when:** + timezone, catch-up/coalescing, concurrency, project pause, edits, and deletion + behave correctly; external waking can be added without relocating routines. + Source: [routines.ts](../../server/src/services/routines.ts), `tickScheduledTriggers`. + +- [ ] **JOB-10 — Decision retention and notifications.** Current: every **30s**, + builds the complete attention feed for each active company before archiving + and delivering notifications. Change: relevant mutation signals, actual + retention deadlines, and batched maintenance. **Complete when:** unchanged + idle companies do not rebuild feeds and required archive/notification behavior + remains timely. Source: [index.ts](../../server/src/index.ts), `runRetentionSweep`. + +- [ ] **JOB-11 — Decision expiry and recovery.** Current: **30s** scans for stuck + effects, continuations, and TTLs. Change: schedule their actual deadlines and + signal committed effects. **Complete when:** expiration remains authoritative + at action time and interrupted effects recover without perpetual empty scans. + Source: [decisions.ts](../../server/src/services/decisions.ts), `sweepExpired`. + +- [ ] **JOB-12 — External-object refresh.** Current: **30s** scans, including a + DB feature check when disabled. Change: invalidate cached capability on settings + changes; register `nextRefreshAt`; use webhook signals where supported. + **Complete when:** disabled/empty state is quiet and promised refresh freshness + has an owned deadline. Sources: + [external-objects.ts](../../server/src/services/external-objects.ts), + [instance-settings.ts](../../server/src/services/instance-settings.ts). + +- [ ] **JOB-13 — Status-card evaluation.** Current: **30s**, including a DB + feature check when disabled. Change: register `nextEvalAt` and relevant event + invalidations. **Complete when:** no capability/due cards means no queries, + and evaluation preserves existing assignment and cancellation behavior. + Source: [status-cards.ts](../../server/src/services/status-cards.ts). + +- [ ] **JOB-14 — GitHub events and connection continuity.** Current: **30s** + connection scans; provider calls depend on configured/due work. Change: + webhook-first activation plus explicit poll fallback, credential, and retry + deadlines. **Complete when:** empty state is quiet and configured connections + retain their delivery and continuity guarantees. Source: + [index.ts](../../server/src/index.ts), `scheduleGitHubConnectionEventPoll` and + `scheduleGitHubConnectionContinuitySweep`. + +- [ ] **JOB-15 — Merged pull-request confirmations.** Current: **30s** pending + confirmation scan, followed by GitHub checks for matching candidates. Change: + activate on merge events with bounded polling fallback only while candidates + exist. **Complete when:** confirmation resolution remains authorized and + restart-safe, with no empty scan. Source: + [issue-thread-interactions.ts](../../server/src/services/issue-thread-interactions.ts). + +- [ ] **JOB-16 — Terminal workspace cleanup.** Current: **30s** candidate scans. + Change: enqueue from terminal transitions and cooldown deadlines. **Complete + when:** empty state is quiet, and cleanup still checks current task state, + artifacts, deliveries, and ownership before deleting resources. Source: + [execution-workspaces.ts](../../server/src/services/execution-workspaces.ts). + +- [ ] **JOB-17 — Sandbox cleanup and orphan spools.** Current: **30s** scans even + when attempts have longer staleness/backoff. Change: durable cleanup queue with + next retry, immediate failed-release activation, and provider/fleet backstop. + **Complete when:** idle scans stop without leaking paid sandboxes or ignoring + malformed/unpersisted spool entries. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts), `sweepPendingCleanupLeases`. + +- [ ] **JOB-18 — Adapter login and setup-token reapers.** Current: **30s** + durable-session/lease scans, alongside local one-shot timers. Change: register + actual expiry and cleanup retry times. **Complete when:** abandoned logins and + interrupted teardown still release sandboxes across restart, without scans + when no obligations exist. Source: + [index.ts](../../server/src/index.ts), adapter and setup-token reaper wiring. + +- [ ] **JOB-19 — Custom-image setup expiry.** Current: **30s** active-session + expiry scans. Change: register `expiresAt` and cleanup attempts. **Complete + when:** no sessions means no SQL and expired provider resources are released. + Source: [environment-custom-images.ts](../../server/src/services/environment-custom-images.ts). + +- [ ] **JOB-20 — Tool connection health.** Current: **30s** DB inventory; + provider probes are conditional. Change: check on use/configuration change, + with explicit deadlines for any promised proactive health checks. **Complete + when:** quiet connections stop creating empty scans and failures remain + visible within the agreed freshness contract. Source: + [index.ts](../../server/src/index.ts), `tools.sweepConnectionHealth`. + +- [ ] **JOB-21 — Secret proposals and expired action tokens.** Current: **30s** + expiry/cleanup queries. Change: enforce expiry synchronously on use and + register required scrub/cleanup deadlines. **Complete when:** idle cleanup + does not extend authorization or secret-retention periods. Sources: + [secret-proposals.ts](../../server/src/services/secret-proposals.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-22 — Chat completion delivery.** Current: **30s** outbox scan plus + an existing postcommit fast path. Change: extend that signal and register + retries only while pending. **Complete when:** committed completions survive + restart, duplicate activation, and provider failure without empty polling. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-23 — Connection continuation delivery.** Current: **30s** outbox + scans. Change: committed signals, retry deadlines, and startup reconciliation. + **Complete when:** newly available connections resume eligible work once, + preserve execution gates, and have no empty SQL loop. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **JOB-24 — Tool action reviews and deliveries.** Current: **30s** + review/session/outbox scans. Change: register review expiry and delivery + deadlines; activate after committed decisions. **Complete when:** approval, + rejection, expiry, and delivery retain authorization and replay protection. + Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-25 — Question response delivery.** Current: **30s** outbox scans. + Change: committed-answer activation, durable retry deadlines, and startup + recovery. **Complete when:** responses preserve attempt-generation fencing + and recover from delivery failure without polling empty queues. Source: + [question-response-delivery.ts](../../server/src/services/question-response-delivery.ts). + +- [ ] **JOB-26 — Cost accounting and decision-model recovery.** Current: + **30s** reconciliation even when empty. Change: pending-receipt signals, + spool awareness, and durable retries. **Complete when:** no empty queries, + but unresolved receipts/reservations remain tracked and cannot be discarded + to permit sleep. Source: [heartbeat.ts](../../server/src/services/heartbeat.ts), + `reconcileCostAccounting`. + +- [ ] **JOB-27 — Run recovery, scheduled retries, and queued work.** Current: + **30s** orphan/retry/queue/stranded-assignment scans. Change: remain responsive + while work exists, register retry/control deadlines, and reconcile on startup + or wake. **Complete when:** quiet state has no scans and interrupted runs do + not duplicate provider work or lose continuation. Source: + [index.ts](../../server/src/index.ts), periodic heartbeat recovery chain. + +- [ ] **JOB-28 — Dependencies, task watchdogs, silent runs, and stale locks.** + Current: additional **30s** scans. Change: dependency/run transition signals, + active-run-only monitoring, actual timeout deadlines, and startup backstops. + **Complete when:** recovery remains scoped and fenced, and a completed task + still receives any owed watchdog review. Source: + [heartbeat.ts](../../server/src/services/heartbeat.ts). + +- [ ] **JOB-29 — Automatic database backups.** Current: enabled by default, + typically **hourly**, with separate dump connections. Change: register with + the internal scheduler; later assign durable waking or explicitly transfer + backup responsibility to managed infrastructure. **Complete when:** backup + and restore guarantees are demonstrated, active backup blocks sleep, and a + merely configured future backup no longer blocks indefinitely once covered + by an external alarm. Source: [index.ts](../../server/src/index.ts). + +- [ ] **JOB-30 — Replace the main tick and its disabled branch.** Current: + default **30s** main tick; setting `HEARTBEAT_SCHEDULER_ENABLED=false` still + leaves accounting, external-object, cleanup, and GitHub work, plus independent + workers. Change: migrate all lanes before retiring aggregate polling. + **Complete when:** disabled scheduling semantics remain documented and no + legacy fallback quietly reintroduces recurring SQL. Sources: + [index.ts](../../server/src/index.ts), [config.ts](../../server/src/config.ts). + +## 5. Connectors and transport lifetime + +- [ ] **NET-01 — AgentMail socket ownership.** Current: outbound WebSocket and + **20s** DB lease renewal; its renewal bypasses the top-level background gate. + Change: prefer webhook mode for hosted stacks; otherwise retain an awake + receiver or use shared durable ingress. **Complete when:** explicit suspend + closes all associated timers and message reception/replay still works. + Source: [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-02 — AgentMail catch-up.** Current: roughly **one-minute** provider + reconciliation even in webhook mode. Change: catch up on wake/connect and at + an owned recovery deadline. **Complete when:** checkpoints and deduplication + survive lost webhooks, with a defined maximum recovery delay. Source: + [email-channels.ts](../../server/src/services/email-channels.ts). + +- [ ] **NET-03 — Discord Gateway.** Current: persistent connection and **5s** DB + ownership renewal. Change: keep unsupported stacks awake initially; evaluate + a shared ingress worker owning the connection, lease, and durable delivery. + **Complete when:** idle tenant sleep does not silently stop timely reception + or weaken ownership fencing. Sources: + [chat-channels.ts](../../server/src/services/chat-channels.ts), + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-04 — Photon iMessage.** Current: several gRPC subscriptions, **5s** + ownership renewal, and **15s** catch-up. Change: explicit transport lifecycle + and, where required, shared durable receiving. **Complete when:** resume + preserves ordered checkpoints and detects replay/history gaps; stopping the + top-level chat loop also retires dependent work safely. Source: + [receiver.ts](../../server/src/services/photon/receiver.ts). + +- [ ] **NET-05 — Slack webhook ingress.** Current transport already uses webhook + mode. Change: qualify it through wake-aware ingress rather than introducing + a persistent receiver. **Complete when:** cold start meets provider timing + through durable acknowledgement/retry where necessary, with signature + verification, deduplication, and company scope preserved. Source: + [chat-sdk-runtime.ts](../../server/src/services/chat-sdk-runtime.ts). + +- [ ] **NET-06 — HTTP idle connections.** Current server keepalive is **185s** + to outlive common proxy defaults. Change: explicitly close idle sockets during + drain and qualify provider-specific connection policies. **Complete when:** + quiet sockets do not prevent intended sleep, and normal proxy reuse does not + gain intermittent 502/ECONNRESET failures. Source: + [index.ts](../../server/src/index.ts). + +- [ ] **NET-07 — Active execution transports and lease renewals.** Inventory + runner/PRP connections, HTTP/2 bridge pings, native/legacy controller leases, + terminal SSH keepalives, browser viewport ownership, and per-operation claims. + Retain their required cadence during work. **Complete when:** active operations + hold the instance awake and unused transports retire; no arbitrary heartbeat + disabling is used to manufacture sleep. Sources: + [native-session-executor.ts](../../server/src/services/native-runtime/native-session-executor.ts), + [legacy-controller-lease.ts](../../server/src/services/legacy-controller-lease.ts), + [HTTP/2 bridge](../../packages/adapter-utils/src/http2-bridge-server.ts), + [terminal WebSocket](../../server/src/realtime/environment-custom-image-terminal-ws.ts). + +## 6. Browser demand + +These client-side schedules do not run in the server scheduler. Consolidate +their activity policy so a parked browser does not create artificial tenant +demand. Preserve real-time behavior while a person is actively using the app. + +- [ ] **UI-01 — Live WebSocket and fallback polling.** Hidden tabs already close + the main socket; visible tabs receive **30s** pings and use **15s** refetch + fallback when disconnected. Change: distinguish intentional sleep from + outage and decide whether unattended visible tabs may become inactive. + **Complete when:** reconnection does not create a self-wake loop and returning + users reconcile missed state. Source: + [LiveUpdatesProvider.tsx](../../ui/src/context/LiveUpdatesProvider.tsx). + +- [ ] **UI-02 — Stable page queries.** Current issue views poll at **1–3s**, + browser/artifact views at **3–5s**, and agents/costs/reviews/settings at + **5–60s**. Change: extend event invalidation and active-work-only fallback; + reuse [polling.ts](../../ui/src/lib/polling.ts) and + [useSharedPolling.ts](../../ui/src/hooks/useSharedPolling.ts). + **Complete when:** idle views stop artificial demand without hiding updates; + multiple tabs do not duplicate avoidable work. Starting sources: + [IssueDetail.tsx](../../ui/src/pages/IssueDetail.tsx), + [Costs.tsx](../../ui/src/pages/Costs.tsx), + [useTaskBrowsers.ts](../../ui/src/hooks/useTaskBrowsers.ts). + +- [ ] **UI-03 — Cloud-access error recovery.** Current transient-error retries + run every **5s**, explicitly in background tabs; bootstrap has faster polling. + Change: visibility/online gating and wake-aware recovery, with bounded + bootstrap behavior. **Complete when:** a hidden tab that last received 502/503 + cannot keep waking its tenant indefinitely. Source: + [CloudAccessGate.tsx](../../ui/src/components/CloudAccessGate.tsx). + +- [ ] **UI-04 — Service-worker update checks.** Current update check is + **hourly**, regardless of visibility. Change: check on foreground activity or + serve shell/update assets from shared static hosting. **Complete when:** a + parked hidden tab does not periodically wake its tenant to fetch `/sw.js`. + Source: [service-worker-updates.ts](../../ui/src/lib/service-worker-updates.ts). + +- [ ] **UI-05 — Plugin SSE streams.** Current EventSource lacks visibility + gating and holds an HTTP response open. Change: explicit visibility/idle + lifetime with reconciliation or replay on reconnect. **Complete when:** + plugin state remains correct and abandoned streams do not block sleep. + Sources: [bridge.ts](../../ui/src/plugins/bridge.ts), + [plugin routes](../../server/src/routes/plugins.ts). + +- [ ] **UI-06 — Setup and enrollment polling.** OAuth/chat setup often polls at + **1.5–3s**, email setup at **10s**, commonly while pending. Change: keep these + responsive during the bounded workflow; stop on terminal state, close, or + inactivity with a deliberate resume path. **Complete when:** abandoned setup + creates neither permanent demand nor lost completion evidence. Sources: + [ConnectionSetupFlow.tsx](../../ui/src/features/connections/ConnectionSetupFlow.tsx), + [chat setup](../../ui/src/pages/apps/chat/ChatEndpointSetup.tsx), + [email setup](../../ui/src/pages/apps/chat/EmailEndpointSetup.tsx). + +## 7. Lower traffic housekeeping and plugins + +- [ ] **AUX-01 — First-party telemetry.** The **60s** flush returns without + sending when its queue is empty. Change: register flush/retry work only when + needed and drain or durably retain pending batches. **Complete when:** empty + telemetry is network-quiet and delivery semantics remain explicit. Preserve + generated-contract, README, and privacy-review requirements for event changes. + Sources: [telemetry.ts](../../server/src/telemetry.ts), + [telemetry client](../../packages/shared/src/telemetry/client.ts). + +- [ ] **AUX-02 — Optional OpenTelemetry.** No configured OTLP endpoint means a + no-op tracer; this is separate from first-party telemetry and the local run + log. Change: inspect exporter batching/connection behavior only when enabled; + avoid a perpetual stream of idle-maintenance spans. **Complete when:** the + endpoint gate remains intact and configured exporters do not defeat intended + sleep after their pending work settles. Source: + [instrumentation.ts](../../server/src/instrumentation.ts). + +- [ ] **AUX-03 — Plugin log buffer.** The **5s** flush avoids SQL when empty. + Change: schedule a bounded flush after the first buffered entry and include + pending writes in drain. **Complete when:** no empty timer is needed and log + persistence has a defined shutdown/sleep boundary. Source: + [plugin-host-services.ts](../../server/src/services/plugin-host-services.ts). + +- [ ] **AUX-04 — Import spools and in-memory status expiry.** Hourly import + cleanup already avoids SQL when the local spool is absent/empty. Runtime + status expiry is memory-only. Change: use actual deadlines or opportunistic + cleanup where timing is not contractual. **Complete when:** neither causes an + unnecessary external wake, while retained files and required cleanup remain + tracked. Sources: + [company-import-transfers.ts](../../server/src/services/company-import-transfers.ts), + [index.ts](../../server/src/index.ts). + +- [ ] **AUX-05 — Plugin worker lifecycle.** Enabled plugins can execute arbitrary + background work; host timers cannot prove they are quiet. Change: explicit + quiesce/resume, scheduled-job registration, ingress ownership, and lazy provider + worker startup where appropriate. **Complete when:** supported plugins prove + quiescence; unsupported plugins remain a named blocker. Source: + [plugin-worker-manager.ts](../../server/src/services/plugin-worker-manager.ts). + +- [ ] **AUX-06 — Verify low-priority and third-party sources at runtime.** + Announcements are request-driven with caching; no dedicated periodic server + application-update check was found. Plugin log retention has no production + caller in the audited revision. **Complete when:** packet traces confirm these + classifications and identify any SDK, gRPC, HTTP agent, monitoring, remote + client, or plugin traffic omitted by static inspection. Source: + [announcement-feed.ts](../../server/src/services/announcement-feed.ts). + +## 8. Sleep eligibility and alternative hosting + +- [ ] **SAFE-01 — Separate claimed inactivity from unclaimed warm standby.** + Existing warm standby bypasses SQL and recurring work only for empty, + unclaimed instances. Change: add claimed-instance behavior through explicit + scheduler/quiescence contracts. **Complete when:** an inactive claimed tenant + can become quiet without impersonating an unclaimed tenant or disrupting + signed claim handling. Source: + [cloud-warm-standby.ts](../../server/src/services/cloud-warm-standby.ts). + +- [ ] **SAFE-02 — Replace conservative blockers one capability at a time.** + Current safety blocks every enabled plugin, configured backups, API keys, + active routines/connections, nonterminal issues, and many retained records. + Change: distinguish current obligations from passive capability/history only + when a validated wake/recovery contract exists. **Complete when:** waiting for + a human or retaining completed history can permit sleep, while unknown state, + cleanup/accounting debt, and unhandled transports remain blockers. Sources: + [idle-sleep-safety.ts](../../server/src/services/idle-sleep-safety.ts), + [idle-local-work.ts](../../server/src/services/idle-local-work.ts). + +- [ ] **SAFE-03 — Validate cold start and stateful resume.** A snapshot preserves + memory, not the validity of external sockets, credentials, or leases. Change: + recheck deadlines and ownership on resume, recover stale connections, and + avoid replaying uncertain operations. **Complete when:** long sleeps, + expired sessions, and remote controller changes produce correct recovery + rather than duplicate execution or prolonged ingress rejection. + +- [ ] **HOST-01 — Evaluate colocated PostgreSQL separately.** Public authenticated + deployments currently reject an unset `DATABASE_URL` rather than falling back + to embedded PostgreSQL. Prototype explicit loopback PostgreSQL or deliberately + revise that deployment contract. **Complete when:** persistent storage, + backups/restores, upgrades, memory cost, and sleep/resume are qualified, with + agent execution isolated from the controller/database. This option does not + remove scheduler or wake requirements. Source: + [index.ts](../../server/src/index.ts), `assertCloudDatabaseContract`. + +## 9. Delivery order and acceptance + +1. Complete mechanism triage for the selected first slice: normal trigger, + producer coverage, durable dispatch, and remaining temporal obligations. +2. Ship DB-free health/probe handling and contained event-driven migrations. + Measure empty-worker SQL reduction while preserving delivery backstops until + their replacements are proven. These changes do not depend on a full scheduler. +3. Build the internal scheduler around true deadlines, starting with routines + and adding required retry/expiry/maintenance paths. Then add the external-waker + adapter and acknowledged alarm protocol before sleeping scheduled workloads. +4. Qualify connector transports, plugin lifecycle, browser demand, and provider + resume behavior. Expand eligibility only with evidence. + +- [ ] **TEST-01 — Measure an empty claimed tenant.** After startup settles, prove + zero app-originated SQL and no unintended outgoing packets, then observe actual + Neon and hosting sleep. Use provider/control-plane metrics so measurement + itself does not wake the tenant DB. Compare against the original baseline. + +- [ ] **TEST-02 — Prove first-use recovery.** After real sleep, perform an + authenticated read, write, and transaction. Verify persistent results, no + duplicate effects, and measured end-to-end latency including authentication + and database startup. + +- [ ] **TEST-03 — Prove scheduled work.** Exercise routines, a moved-earlier + deadline, cancellation, retry, expiry, and maintenance through restart/sleep. + Deliver duplicate/stale wakes and simulate external-waker failure. Verify + catch-up, claims, and awake fallback. + +- [ ] **TEST-04 — Prove event ingress.** Deliver a verified webhook while asleep + and while entering sleep. Exercise a connector with its supported receiver + strategy. Verify acknowledgements, replay, company scope, and persistence. + +- [ ] **TEST-05 — Prove active-work protection.** Attempt sleep during a run, + transaction, backup, finalization, cleanup attempt, and pending accounting + receipt. Verify the intended blocker/hold and safe release after settlement. + +- [ ] **TEST-06 — Prove browser inactivity behavior.** Test hidden tabs, + unattended visible tabs, reconnect/error states, plugin streams, and multiple + tabs. Verify returning users see reconciled state without an accidental wake + storm. + +- [ ] **TEST-07 — Prove crash, ownership, and rollout behavior.** Test a crash + around commit/alarm publication, stale drain owner, provider stop racing a + deployment, and long stateful suspension. Roll back migrated lanes without + duplicate workers. Measure fleet wake concurrency and actual compute savings + before broad enablement. + +Each implementation PR should run the smallest relevant checks first and the +repository-required verification for its handoff scope. This documentation-only +checklist does not claim any provider sleep/wake test has passed. diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index c1b34d21eb6..3f6061deaae 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -3,6 +3,7 @@ import { drizzle as drizzlePg } from "drizzle-orm/postgres-js"; import { readFile, readdir } from "node:fs/promises"; import { fileURLToPath } from "node:url"; import postgres from "postgres"; +import { installDatabaseWorkSignals } from "./work-signals.js"; import * as schema from "./schema/index.js"; const MIGRATIONS_FOLDER = fileURLToPath(new URL("./migrations", import.meta.url)); @@ -268,6 +269,10 @@ export async function withDedicatedDbConnection(db: Db, action: (dedicated: D export function createDb(url: string, options?: DatabaseClientOptions) { const resolved = resolveDatabaseClientOptions(options ?? databaseClientOptionsFromEnv()); + return createDbClient(url, resolved); +} + +function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object) { const sql = postgres(url, postgresJsOptions(resolved)); const key = hostPortKeyOrNull(url); if (key) registerClient(key, sql); @@ -275,10 +280,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) { // postgres.js calls that error "write CONNECTION_CLOSED" too, so the // message cannot establish that replay is safe. Leave retries to callers // that know the complete operation is idempotent. - const db = drizzlePg(sql, { schema }); - dedicatedDbFactories.set(db, () => createDb(url, { + const db = installDatabaseWorkSignals(drizzlePg(sql, { schema }), owner); + dedicatedDbFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: 1, applicationName: "paperclip-workspace-finalization-lock", - })); + }, db)); return db; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 378c9bd37b8..73c6028d067 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -42,3 +42,5 @@ export { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecy export { issueRelations } from "./schema/issue_relations.js"; export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; + +export { signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts new file mode 100644 index 00000000000..9bc3f738b1c --- /dev/null +++ b/packages/db/src/work-signals.test.ts @@ -0,0 +1,165 @@ +import { sql } from "drizzle-orm"; +import { drizzle } from "drizzle-orm/postgres-js"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb, withDedicatedDbConnection } from "./client.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; +import { databaseWorkPending, installDatabaseWorkSignals, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("database work signals", () => { + let db: ReturnType; + let temporary: Awaited>; + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("work-signals-"); + db = createDb(temporary.connectionString); + await db.execute(sql`create table work_signal_test (id text primary key)`); + }, 90_000); + afterAll(async () => { await temporary?.cleanup(); }); + + it("defers nested savepoint notifications until the outer commit and preserves return values", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "nested", event => events.push(event)); + try { + const result = await db.transaction(async tx => { + await tx.transaction(async nested => { + await signalDatabaseWork(nested, "nested"); + await nested.execute(sql`insert into work_signal_test values ('nested')`); + }); + expect(events).toEqual(["intent"]); + expect(databaseWorkPending(db, "nested")).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(0); + return 123; + }); + expect(result).toBe(123); + expect(events).toEqual(["intent", "settled"]); + expect(databaseWorkPending(db, "nested")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'nested'`)).toHaveLength(1); + } finally { unsubscribe(); } + }); + + it("reconciles a rollback without leaving a permanent pending flag", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "rollback"); + await tx.execute(sql`insert into work_signal_test values ('rollback')`); + throw new Error("abort"); + })).rejects.toThrow("abort"); + await reconcileDatabaseWork(db, "rollback"); + expect(databaseWorkPending(db, "rollback")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'rollback'`)).toHaveLength(0); + }); + + it("keeps outer ownership after a caught savepoint rollback", async () => { + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "savepoint", event => events.push(event)); + try { + await db.transaction(async tx => { + await expect(tx.transaction(async nested => { + await signalDatabaseWork(nested, "savepoint"); + await nested.execute(sql`insert into work_signal_test values ('savepoint')`); + throw new Error("rollback savepoint"); + })).rejects.toThrow("rollback savepoint"); + expect(databaseWorkPending(db, "savepoint")).toBe(true); + expect(events).toEqual(["intent"]); + await signalDatabaseWork(tx, "savepoint"); + await tx.execute(sql`insert into work_signal_test values ('savepoint-retry')`); + }); + expect(events).toEqual(["intent", "intent", "settled"]); + expect(databaseWorkPending(db, "savepoint")).toBe(false); + } finally { unsubscribe(); } + }); + + it.each(["commit", "abort"])("does not clear a late %s until PostgreSQL reports a settled transaction", async outcome => { + // Inject a lost response at the client boundary while a REAL transaction + // remains open on PostgreSQL. The status probe and row visibility are real. + const raw = drizzle(db.$client); + const transaction = raw.transaction.bind(raw); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let wrote!: () => void; + const written = new Promise(resolve => { wrote = resolve; }); + let serverDone!: Promise; + raw.transaction = ((callback: any) => { + serverDone = transaction(async tx => { + const result = await callback(tx); + wrote(); + await gate; + if (outcome === "abort") throw new Error("server abort"); + return result; + }); + // Observe rejection immediately; tests await the same result below. + void serverDone.catch(() => {}); + return written.then(() => { throw new Error("lost transaction response"); }); + }) as typeof raw.transaction; + const owner = installDatabaseWorkSignals(raw); + const topic = `late-${outcome}`; + try { + await expect(owner.transaction(async tx => { + await signalDatabaseWork(tx, topic); + await tx.execute(sql`insert into work_signal_test values (${topic})`); + })).rejects.toThrow("lost transaction response"); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(true); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(0); + release(); await serverDone.catch(() => {}); + await reconcileDatabaseWork(owner, topic); + expect(databaseWorkPending(owner, topic)).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = ${topic}`)).toHaveLength(outcome === "commit" ? 1 : 0); + } finally { release(); await serverDone?.catch(() => {}); } + }); + + it("recovers after a real backend disconnect without replaying the queue write", async () => { + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "disconnect"); + await tx.execute(sql`insert into work_signal_test values ('disconnect')`); + const [{ pid }] = await tx.execute(sql`select pg_backend_pid() as pid`); + await db.execute(sql`select pg_terminate_backend(${pid as number})`); + await tx.execute(sql`select 1`); + })).rejects.toThrow(); + await reconcileDatabaseWork(db, "disconnect"); + expect(databaseWorkPending(db, "disconnect")).toBe(false); + expect(await db.execute(sql`select * from work_signal_test where id = 'disconnect'`)).toHaveLength(0); + }); + + it("shares notifications with dedicated child pools but isolates unrelated roots and topics", async () => { + const listener = vi.fn(); + const other = createDb(temporary.connectionString); + const unsubscribe = subscribeDatabaseWork(db, "dedicated", listener); + try { + await other.transaction(tx => signalDatabaseWork(tx, "dedicated")); + await db.transaction(tx => signalDatabaseWork(tx, "other")); + expect(listener).not.toHaveBeenCalled(); + await withDedicatedDbConnection(db, dedicated => dedicated.transaction(tx => signalDatabaseWork(tx, "dedicated"))); + expect(listener.mock.calls.map(([event]) => event)).toEqual(["intent", "settled"]); + } finally { unsubscribe(); await other.$client.end(); } + }); + + it("never changes transaction success because a listener throws", async () => { + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const unsubscribe = subscribeDatabaseWork(db, "listener", () => { throw new Error("listener"); }); + try { + await expect(db.transaction(async tx => { await signalDatabaseWork(tx, "listener"); return 7; })).resolves.toBe(7); + expect(warn).toHaveBeenCalledTimes(2); + } finally { unsubscribe(); warn.mockRestore(); } + }); +}); + +describe("work signal query boundaries", () => { + it("adds no SQL to transactions which do not register queue work", async () => { + const execute = vi.fn(); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + await expect(connection.transaction(async () => 19)).resolves.toBe(19); + expect(execute).not.toHaveBeenCalled(); + }); + it("fails before the write if XID acquisition fails, without retaining uncertainty", async () => { + const execute = vi.fn().mockRejectedValue(new Error("disconnected")); + const connection = installDatabaseWorkSignals({ execute, transaction: async (callback: any) => callback({ execute, transaction: async () => {} }) }); + const write = vi.fn(); + await expect(connection.transaction(async (tx: any) => { await signalDatabaseWork(tx, "failed"); write(); })).rejects.toThrow("disconnected"); + expect(write).not.toHaveBeenCalled(); + expect(databaseWorkPending(connection, "failed")).toBe(false); + }); + it("rejects registration on an autocommit root", async () => { + const connection = installDatabaseWorkSignals({ execute: vi.fn(), transaction: vi.fn() }); + await expect(signalDatabaseWork(connection, "bad")).rejects.toThrow("inside a transaction"); + }); +}); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts new file mode 100644 index 00000000000..219ea924468 --- /dev/null +++ b/packages/db/src/work-signals.ts @@ -0,0 +1,109 @@ +import { sql } from "drizzle-orm"; + +// Process-local hints about existing durable queues. No SQL triggers, tables, +// background timers, or driver query replay. Only opted-in transactions fetch +// an XID, so unrelated transactions keep their existing SQL behavior. +type Listener = (event: "intent" | "settled") => void; +type Transaction = { topics: Set; xid?: Promise; uncertain: boolean }; +type Scope = { listeners: Map>; transactions: Set; root: Connection }; +type Context = { scope: Scope; transaction?: Transaction }; +type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; +const contexts = new WeakMap(); + +function publish(scope: Scope, topic: string, event: "intent" | "settled" = "settled") { + for (const listener of scope.listeners.get(topic) ?? []) { + try { listener(event); } + catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } +} + +/** Installed once by createDb; dedicated child pools share their owner's scope. */ +export function installDatabaseWorkSignals(db: T, owner?: object): T { + const scope = (owner && contexts.get(owner)?.scope) || { listeners: new Map(), transactions: new Set(), root: db }; + install(db, { scope }); + return db; +} + +function install(connection: Connection, context: Context) { + contexts.set(connection, context); + const transaction = connection.transaction; + connection.transaction = async function (callback, ...options) { + // Nested savepoints belong to the outermost transaction. Even a released + // savepoint must not wake a consumer before its parent commits. + const state = context.transaction ?? { topics: new Set(), uncertain: false }; + try { + const result = await transaction.call(this, async (tx: Connection) => { + install(tx, { scope: context.scope, transaction: state }); + return callback(tx); + }, ...options); + if (!context.transaction) context.scope.transactions.delete(state); + return result; + } catch (error) { + if (!context.transaction) { + // An XID is known before the first opted-in write can execute. If + // acquisition itself failed there cannot be a corresponding write. + if (state.xid) state.uncertain = true; + else context.scope.transactions.delete(state); + } + throw error; + } finally { + if (!context.transaction) for (const topic of state.topics) publish(context.scope, topic); + } + }; +} + +/** Await BEFORE the queue write, on its transaction. Nested transactions work. */ +export async function signalDatabaseWork(connection: object, topic: string): Promise { + const context = contexts.get(connection); + if (!context) return; // Lightweight service test doubles have no DB lifecycle. + const state = context.transaction; + if (!state) throw new Error("Database work must be registered inside a transaction"); + state.topics.add(topic); + context.scope.transactions.add(state); + publish(context.scope, topic, "intent"); + if (!state.xid) { + state.xid = (connection as Connection).execute(sql`select pg_current_xact_id()::text as xid`) + .then((rows: { xid: string }[]) => rows[0]!.xid); + } + // Keep a failed acquisition distinguishable from an uncertain queue write. + try { await state.xid; } + catch (error) { state.xid = undefined; throw error; } +} + +export function databaseWorkPending(owner: object, topic: string): boolean { + return [...(contexts.get(owner)?.scope.transactions ?? [])].some(state => state.topics.has(topic)); +} + +/** Resolve only failed transactions, on a fresh root-pool statement. Call + * BEFORE the queue scan so rows from a newly confirmed commit are visible. + * NULL means the XID aged out; it can no longer be in progress, so scanning + * the durable queue is sufficient even though its outcome is unavailable. + */ +export async function reconcileDatabaseWork(owner: object, topic: string): Promise { + const scope = contexts.get(owner)?.scope; + if (!scope) return; + for (const state of [...scope.transactions]) { + if (!state.uncertain || !state.topics.has(topic) || !state.xid) continue; + const xid = await state.xid; + const rows = await scope.root.execute(sql`select pg_xact_status(${xid}::xid8) as status`); + if (rows[0]?.status === "in progress") continue; + if (!rows.length || ![null, "committed", "aborted"].includes(rows[0].status)) { + throw new Error("Unexpected database transaction status"); + } + scope.transactions.delete(state); + for (const affected of state.topics) publish(scope, affected); + } +} + +/** Includes intent, outer settlement, and uncertainty-resolution changes. */ +export function subscribeDatabaseWork(owner: object, topic: string, listener: Listener): () => void { + const context = contexts.get(owner); + if (!context) throw new Error("Database work subscriptions require createDb"); + let listeners = context.scope.listeners.get(topic); + if (!listeners) context.scope.listeners.set(topic, listeners = new Set()); + listeners.add(listener); + if (databaseWorkPending(owner, topic)) { + try { listener("intent"); } catch (error) { process.emitWarning(`Database work listener failed: ${String(error)}`); } + } + return () => { listeners.delete(listener); }; +} diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index bb8e7f076db..694f511d0e5 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -1,3 +1,5 @@ +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { subscribeDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; @@ -7,7 +9,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } fro import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; -import { issueService } from "../services/issues.js"; +import { executeIssuePostCommitActions, type IssuePostCommitAction, issueService } from "../services/issues.js"; import { buildLowTrustSourceTrust } from "../services/source-trust.js"; import { documentService } from "../services/documents.js"; import { instanceSettingsService } from "../services/instance-settings.js"; @@ -51,6 +53,57 @@ const support = await getEmbeddedPostgresTestSupport(); }; return { companyId, agentId, sourceId, runId, task, create, finish, rows, due, wakeup, service, run }; } + it("dispatches a committed completion without activity publication and recovers after restart", async () => { + const f = await seed(); + const run = vi.fn(() => f.service.sweepPending()); + const makeWorker = () => { + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const worker = coordinator.register(DELIVERY_QUEUES.chatCompletion, { retryMs: 60_000, run, hasPending: f.service.hasPending }); + return { ...worker, stop: coordinator.stop }; + }; + const worker = makeWorker(); + try { + await worker.ready; + expect(await f.service.hasPending()).toBe(false); + expect(run).toHaveBeenCalledTimes(1); + await f.finish(); // Service mutation alone: no route activity callback. + await vi.waitFor(() => expect(f.wakeup).toHaveBeenCalledTimes(1)); + expect(await f.service.hasPending()).toBe(true); // queued receipt still owed + } finally { await worker.stop(); } + // Admit the first reporting turn; later completions need a new turn. + await f.run(); + // A second task commits while no consumer is registered. + const next = await f.create(); + await f.finish(next.id); + const restarted = makeWorker(); + try { + await restarted.ready; + expect(f.wakeup).toHaveBeenCalledTimes(2); + } finally { await restarted.stop(); } + }); + it("automatically notifies after a caller-owned transaction commits", async () => { + const f = await seed(); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.chatCompletion, notified); + try { + const actions: IssuePostCommitAction[] = []; + await db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "done" }, tx, [], actions); + expect(notified).not.toHaveBeenCalled(); + }); + expect(notified).toHaveBeenCalledTimes(1); + expect(actions).toEqual([]); + await executeIssuePostCommitActions(db, actions); + expect(notified).toHaveBeenCalledTimes(1); + const rolledBack: IssuePostCommitAction[] = []; + await expect(db.transaction(async tx => { + await issueService(db).update(f.task.id, { status: "todo" }, tx, [], rolledBack); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + expect(notified).toHaveBeenCalledTimes(1); + expect((await f.rows())[0]?.status).toBe("pending"); + } finally { unsubscribe(); } + }); it("records authenticated origins and atomically creates one event per Done transition", async () => { const f = await seed(); expect(await db.select().from(handoffs).where(eq(handoffs.taskId, f.task.id))).toMatchObject([{ conversationId: f.sourceId, sessionGeneration: 0 }]); diff --git a/server/src/__tests__/connection-intents-service.test.ts b/server/src/__tests__/connection-intents-service.test.ts index c6c318c372d..94613ec1f33 100644 --- a/server/src/__tests__/connection-intents-service.test.ts +++ b/server/src/__tests__/connection-intents-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; @@ -773,7 +775,11 @@ describeEmbeddedPostgres("connectionIntentService", () => { it("recovers a resolution after a failed dispatch and acknowledges an already queued wake exactly once", async () => { const service = connectionIntentService(db); const pending = await service.request(claims, "airtable"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.connection, notified); await service.decline(pending.interactionId!, claims.responsible_user_id); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); const wakeup = vi.fn().mockRejectedValueOnce(new Error("simulated crash before durable enqueue")); const deliveries = connectionIntentDeliveryService(db, { wakeup } as never); await expect(deliveries.deliver(pending.interactionId!)).rejects.toThrow("simulated crash"); diff --git a/server/src/__tests__/delivery-work-coordinator.test.ts b/server/src/__tests__/delivery-work-coordinator.test.ts new file mode 100644 index 00000000000..1be5395f9df --- /dev/null +++ b/server/src/__tests__/delivery-work-coordinator.test.ts @@ -0,0 +1,192 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const coordinators: ReturnType[] = []; +afterEach(async () => { for (const c of coordinators.splice(0)) await c.stop(); vi.useRealTimers(); }); +function setup() { + vi.useFakeTimers(); + let outcome = "committed"; + let rejectCommit = false; + const execute = vi.fn(async () => [{ xid: "42", status: outcome }]); + const owner = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise) { + const result = await callback({ execute, transaction: owner.transaction }); + if (rejectCommit) throw new Error("lost commit reply"); + return result; + } }); + const canRun = vi.fn(() => true), canReconcile = vi.fn(() => false), onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile, onError }); + coordinators.push(coordinator); + const enqueue = (queue = DELIVERY_QUEUES.feedback as typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]) => + owner.transaction(tx => notifyDeliveryWork(tx, queue)); + return { owner, canRun, canReconcile, onError, coordinator, enqueue, execute, + failCommit: () => { rejectCommit = true; outcome = "in progress"; }, settle: (status = "committed") => { outcome = status; } }; +} +function task() { return { retryMs: 5000, run: vi.fn(async () => {}), hasPending: vi.fn(async () => false) }; } + +describe("delivery work coordinator", () => { + it("leaves five empty queues with no deadlines, timers, or periodic queries", async () => { + const s = setup(); + const tasks = Object.values(DELIVERY_QUEUES).map(queue => { + const t = task(); return { queue, t, worker: s.coordinator.register(queue, t) }; + }); + await Promise.all(tasks.map(t => t.worker.ready)); + expect(vi.getTimerCount()).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(3_600_000); + for (const { queue, t } of tasks) { + expect(t.run).toHaveBeenCalledTimes(1); + await s.enqueue(queue); await s.enqueue(queue); + } + await vi.advanceTimersByTimeAsync(1); + for (const { t } of tasks) expect(t.run).toHaveBeenCalledTimes(2); + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not postpone an already committed wake when another producer starts writing", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + await s.enqueue(); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const writing = s.owner.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); + await gate; + }); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + release(); await writing; + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(3); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("fences unresolved writes until the server settles, then scans and releases the idle hold", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); + await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + expect(idleWorkSnapshot().active).toBe(1); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); // Empty scan cannot clear unresolved write. + expect(idleWorkSnapshot().active).toBe(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.settle(); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run.mock.calls.length).toBeGreaterThanOrEqual(3); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + it("retains unresolved intent across coordinator replacement", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.failCommit(); await expect(s.enqueue()).rejects.toThrow(); + await s.coordinator.stop(); + expect(idleWorkSnapshot().active).toBe(0); + const replacement = createDeliveryWorkCoordinator({ owner: s.owner, canRun: () => true, onError: vi.fn() }); + coordinators.push(replacement); + await replacement.register(DELIVERY_QUEUES.feedback, t).ready; + expect(idleWorkSnapshot().active).toBe(1); + s.settle(); await vi.advanceTimersByTimeAsync(5001); + expect(idleWorkSnapshot().active).toBe(0); + expect(replacement.nextWakeAt()).toBeNull(); + }); + it("retries pending deliveries and transient failures, then goes quiet", async () => { + const s = setup(), t = task(); + t.run.mockRejectedValueOnce(new Error("database disconnected")); + t.hasPending.mockResolvedValueOnce(true); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + expect(s.onError).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(10_000); + expect(t.run).toHaveBeenCalledTimes(3); + expect(s.coordinator.nextWakeAt()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not overlap a sweep or lose a commit during its final pending check", async () => { + const s = setup(), t = task(); + let release!: (pending: boolean) => void; + t.hasPending.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.connection, t); + await vi.advanceTimersByTimeAsync(1); + await s.enqueue(DELIVERY_QUEUES.connection); + await vi.advanceTimersByTimeAsync(1000); + expect(t.run).toHaveBeenCalledTimes(1); + release(false); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(2); + }); + it("reconciles a rollback during idle drain and releases the empty-queue hold without dispatching", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + s.failCommit(); await expect(s.enqueue()).rejects.toThrow("lost commit reply"); + await vi.advanceTimersByTimeAsync(1); + expect(idleWorkSnapshot().active).toBe(1); + expect(t.run).toHaveBeenCalledTimes(1); + s.settle("aborted"); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(t.hasPending).toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(s.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps committed pending work recoverable during drain without starting deliveries", async () => { + const s = setup(), t = task(); + await s.coordinator.register(DELIVERY_QUEUES.feedback, t).ready; + s.canRun.mockReturnValue(false); + s.canReconcile.mockReturnValue(true); + t.hasPending.mockResolvedValue(true); + await s.enqueue(); + await vi.advanceTimersByTimeAsync(1); + expect(t.run).toHaveBeenCalledTimes(1); + expect(s.coordinator.nextWakeAt()).not.toBeNull(); + s.canRun.mockReturnValue(true); + t.hasPending.mockResolvedValue(false); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(2); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("does not query during standby and resumes when admission opens", async () => { + const s = setup(), t = task(); s.canRun.mockReturnValue(false); + await s.coordinator.register(DELIVERY_QUEUES.toolAction, t).ready; + await s.enqueue(DELIVERY_QUEUES.toolAction); + await vi.advanceTimersByTimeAsync(120_000); + expect(t.run).not.toHaveBeenCalled(); + expect(t.hasPending).not.toHaveBeenCalled(); + s.canRun.mockReturnValue(true); + await vi.advanceTimersByTimeAsync(5001); + expect(t.run).toHaveBeenCalledTimes(1); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("awaits in-flight work on shutdown and releases the subscription for restart", async () => { + const s = setup(), t = task(); + let release!: () => void; + t.run.mockImplementationOnce(() => new Promise(resolve => { release = resolve; })); + s.coordinator.register(DELIVERY_QUEUES.chatCompletion, t); + await vi.advanceTimersByTimeAsync(1); + const stopped = vi.fn(); + const stopping = s.coordinator.stop().then(stopped); + await s.enqueue(DELIVERY_QUEUES.chatCompletion); + expect(stopped).not.toHaveBeenCalled(); + release(); await stopping; + expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("cancels an abortable delivery before awaiting shutdown", async () => { + const s = setup(); + const hasPending = vi.fn(async () => false); + const run = vi.fn((signal: AbortSignal) => new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + })); + s.coordinator.register(DELIVERY_QUEUES.feedback, { retryMs: 5000, run, hasPending }); + await vi.advanceTimersByTimeAsync(1); + await s.coordinator.stop(); + expect(run.mock.calls[0]?.[0].aborted).toBe(true); + expect(hasPending).not.toHaveBeenCalled(); + expect(s.onError).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/server/src/__tests__/feedback-service.test.ts b/server/src/__tests__/feedback-service.test.ts index ed3c7f21898..08a44442038 100644 --- a/server/src/__tests__/feedback-service.test.ts +++ b/server/src/__tests__/feedback-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; @@ -800,6 +802,8 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { sessionId, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.feedback, notified); await flushingSvc.saveIssueVote({ issueId, targetType: "issue_comment", @@ -808,7 +812,12 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { authorUserId: "user-1", allowSharing: true, }); - await flushingSvc.flushPendingFeedbackTraces(); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + // The route and worker may request a flush together after the commit. + await Promise.all([flushingSvc.flushPendingFeedbackTraces(), flushingSvc.flushPendingFeedbackTraces()]); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); expect(uploadTraceBundle).toHaveBeenCalledTimes(1); const bundle = uploadTraceBundle.mock.calls[0]?.[0] as Record | undefined; @@ -1152,6 +1161,30 @@ describeEmbeddedPostgres("feedbackService.saveIssueVote", () => { expect(uploadTraceBundle).toHaveBeenCalledTimes(1); }); + it("leaves aborted and unstarted exports available for recovery after shutdown", async () => { + const first = await seedIssueWithAgentComment(); + const second = await seedIssueWithAgentComment(); + const shutdown = new AbortController(); + const uploadTraceBundle = vi.fn().mockResolvedValue({ objectKey: "saved" }) + .mockImplementationOnce(async (_bundle, signal: AbortSignal) => { + shutdown.abort(new Error("shutdown")); + signal.throwIfAborted(); + }); + const flushingSvc = feedbackService(db, { shareClient: { uploadTraceBundle } }); + for (const item of [first, second]) { + await flushingSvc.saveIssueVote({ issueId: item.issueId, targetType: "issue_comment", + targetId: item.commentId, vote: "up", authorUserId: "user-1", allowSharing: true }); + } + expect(await flushingSvc.flushPendingFeedbackTraces({ signal: shutdown.signal })) + .toMatchObject({ attempted: 1, sent: 0, failed: 1 }); + const rows = await db.select().from(feedbackExports); + expect(rows.filter(row => row.status === "failed")).toHaveLength(1); + expect(rows.filter(row => row.status === "pending")).toHaveLength(1); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(true); + expect(await flushingSvc.flushPendingFeedbackTraces()).toMatchObject({ attempted: 2, sent: 2, failed: 0 }); + expect(await flushingSvc.hasPendingFeedbackTraces()).toBe(false); + }); + it("marks pending shared traces as failed when no feedback export backend is configured", async () => { const { companyId, issueId, commentId } = await seedIssueWithAgentComment(); diff --git a/server/src/__tests__/feedback-share-client.test.ts b/server/src/__tests__/feedback-share-client.test.ts index fe710262839..d47d01ede4e 100644 --- a/server/src/__tests__/feedback-share-client.test.ts +++ b/server/src/__tests__/feedback-share-client.test.ts @@ -1,6 +1,6 @@ import { gunzipSync } from "node:zlib"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; +import { FEEDBACK_UPLOAD_TIMEOUT_MS, createFeedbackTraceShareClientFromConfig } from "../services/feedback-share-client.js"; describe("feedback trace share client", () => { beforeEach(() => { @@ -12,6 +12,7 @@ describe("feedback trace share client", () => { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllGlobals(); }); it("defaults to telemetry.paperclip.ing when no backend url is configured", async () => { @@ -98,4 +99,43 @@ describe("feedback trace share client", () => { expect(parsed.objectKey.endsWith("/export-1.json")).toBe(true); expect(parsed.bundle.envelope).toEqual({ hello: "world" }); }); + it.each(["headers", "body"])("cancels a stalled upload at its deadline while waiting for %s", async phase => { + const deadline = new AbortController(); + const timeout = vi.spyOn(AbortSignal, "timeout").mockReturnValue(deadline.signal); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation(async (_url, options) => { + const block = () => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + }); + if (phase === "headers") return block(); + return { ok: true, json: block } as unknown as Response; + }); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const uploading = client.uploadTraceBundle(bundle); + const rejected = expect(uploading).rejects.toThrow("upload deadline"); + await waiting; + expect(timeout).toHaveBeenCalledWith(FEEDBACK_UPLOAD_TIMEOUT_MS); + deadline.abort(new Error("upload deadline")); + await rejected; + }); + + it("passes shutdown cancellation through to a pending upload", async () => { + const shutdown = new AbortController(); + let started!: () => void; + const waiting = new Promise(resolve => { started = resolve; }); + vi.mocked(fetch).mockImplementation((_url, options) => new Promise((_resolve, reject) => { + options!.signal!.addEventListener("abort", () => reject(options!.signal!.reason), { once: true }); + started(); + })); + const client = createFeedbackTraceShareClientFromConfig({ feedbackExportBackendUrl: undefined, feedbackExportBackendToken: undefined }); + const bundle = { companyId: "company", traceId: "trace" } as Parameters[0]; + const rejected = expect(client.uploadTraceBundle(bundle, shutdown.signal)).rejects.toThrow("shutdown"); + await waiting; + shutdown.abort(new Error("shutdown")); + await rejected; + }); + }); diff --git a/server/src/__tests__/issue-feedback-routes.test.ts b/server/src/__tests__/issue-feedback-routes.test.ts index e59fe6d1e91..643956ae223 100644 --- a/server/src/__tests__/issue-feedback-routes.test.ts +++ b/server/src/__tests__/issue-feedback-routes.test.ts @@ -8,6 +8,7 @@ const mockFeedbackService = vi.hoisted(() => ({ listIssueVotesForUser: vi.fn(), listFeedbackTraces: vi.fn(), saveIssueVote: vi.fn(), + flushPendingFeedbackTraces: vi.fn(() => new Promise(() => {})), })); const mockIssueService = vi.hoisted(() => ({ @@ -18,9 +19,6 @@ const mockIssueService = vi.hoisted(() => ({ findMentionedAgents: vi.fn(), })); -const mockFeedbackExportService = vi.hoisted(() => ({ - flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 1, sent: 1, failed: 0 })), -})); const mockAccessService = vi.hoisted(() => ({ canUser: vi.fn(), hasPermission: vi.fn(), @@ -138,7 +136,7 @@ async function createApp(actor: Record) { (req as any).actor = actor; next(); }); - app.use("/api", issueRoutes({} as any, {} as any, { feedbackExportService: mockFeedbackExportService })); + app.use("/api", issueRoutes({} as any, {} as any)); const routeErrors: string[] = []; app.locals.routeErrors = routeErrors; app.use((error: unknown, _req: express.Request, _res: express.Response, next: express.NextFunction) => { @@ -152,11 +150,6 @@ async function createApp(actor: Record) { describe("issue feedback trace routes", () => { beforeEach(() => { vi.clearAllMocks(); - mockFeedbackExportService.flushPendingFeedbackTraces.mockResolvedValue({ - attempted: 1, - sent: 1, - failed: 0, - }); mockHeartbeatService.wakeup.mockResolvedValue(undefined); mockHeartbeatService.reportRunActivity.mockResolvedValue(undefined); mockHeartbeatService.getRun.mockResolvedValue(null); @@ -174,7 +167,7 @@ describe("issue feedback trace routes", () => { mockLogActivity.mockResolvedValue(undefined); }); - it("flushes a newly shared feedback trace immediately after saving the vote", async () => { + it("returns the saved vote without waiting for feedback uploads", async () => { const targetId = "11111111-1111-4111-8111-111111111111"; mockIssueService.getById.mockResolvedValue({ id: "issue-1", @@ -211,11 +204,9 @@ describe("issue feedback trace routes", () => { }); expect([200, 201]).toContain(res.status); - expect(mockFeedbackExportService.flushPendingFeedbackTraces).toHaveBeenCalledWith({ - companyId: "company-1", - traceId: "trace-1", - limit: 1, - }); + expect(mockFeedbackService.saveIssueVote).toHaveBeenCalledTimes(1); + expect(mockFeedbackService.flushPendingFeedbackTraces).not.toHaveBeenCalled(); + }); it("rejects non-board callers before fetching a feedback trace", async () => { diff --git a/server/src/__tests__/issue-thread-interactions-service.test.ts b/server/src/__tests__/issue-thread-interactions-service.test.ts index 6e909fc8b3a..887b0ea711d 100644 --- a/server/src/__tests__/issue-thread-interactions-service.test.ts +++ b/server/src/__tests__/issue-thread-interactions-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -231,11 +233,15 @@ describeEmbeddedPostgres("issueThreadInteractionService", () => { expect(created.payload.questions.map((question) => question.id)).toEqual(["repo", "scope", "hosting"]); expect(await interactionsSvc.create(issue, input, { userId: "local-board" })).toEqual(created); await expect(interactionsSvc.answerQuestions(issue, created.id, { answers: [{ questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }] }, { userId: "local-board" })).rejects.toThrow("requires an answer"); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.question, notified); const answered = await interactionsSvc.answerQuestions(issue, created.id, { answers: [ { questionId: "repo", optionIds: [], otherText: "https://example.com/repo" }, { questionId: "scope", optionIds: ["all"] }, { questionId: "hosting", optionIds: ["existing", "new"] }, ] }, { userId: "local-board" }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); expect(answered.status).toBe("answered"); }); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 449d8d5b5dd..5eaa40e414d 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -44,11 +44,17 @@ const { const completionSweepMock = vi.fn(async () => undefined); const createAppMock = vi.fn(async () => Object.assign((_: unknown, __: unknown) => {}, { locals: { + deliveryWork: { + register: (_queue: string, task: { run: () => Promise }) => ({ + ready: task.run().catch(() => undefined), wake: vi.fn(), + }), + stop: vi.fn(async () => {}), + }, toolGateway: { sweepActionReviews: vi.fn(async () => ({ scanned: 0 })), cleanupExpiredSessions: vi.fn(async () => ({ deletedCount: 0 })), }, - toolActionDeliveries: { sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, + toolActionDeliveries: { hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, delivered: 0 })) }, }, }) as never); const createBetterAuthInstanceMock = vi.fn(() => ({})); @@ -133,6 +139,7 @@ const { }; const routineServiceFactoryMock = vi.fn(() => routineServiceMock); const feedbackExportServiceMock = { + hasPendingFeedbackTraces: vi.fn(async () => false), flushPendingFeedbackTraces: vi.fn(async () => ({ attempted: 0, sent: 0, failed: 0 })), }; const feedbackServiceFactoryMock = vi.fn(() => feedbackExportServiceMock); @@ -363,16 +370,18 @@ vi.mock("../services/index.js", () => ({ })), })); -vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ sweepPending: completionSweepMock }) })); +vi.mock("../services/chat-completion-delivery.js", () => ({ chatCompletionDeliveryService: () => ({ hasPending: vi.fn(async () => false), sweepPending: completionSweepMock }) })); vi.mock("../services/connection-intent-delivery.js", () => ({ connectionIntentDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, failed: 0 })), })), })); vi.mock("../services/question-response-delivery.js", () => ({ questionResponseDeliveryService: vi.fn(() => ({ + hasPending: vi.fn(async () => false), sweepPending: vi.fn(async () => ({ scanned: 0, steered: 0, @@ -675,6 +684,22 @@ describe("startServer feedback export wiring", () => { expect(completionSweepMock).toHaveBeenCalled(); }); + it("keeps activity-driven completion scans scoped to the changed task", async () => { + await startServer(); + completionSweepMock.mockClear(); + const { publishLiveEvent } = await import("../services/live-events.js"); + publishLiveEvent({ companyId: "company-one", type: "activity.logged", payload: { + action: "issue.updated", entityId: "task-one", + } }); + expect(completionSweepMock).toHaveBeenCalled(); + for (const args of completionSweepMock.mock.calls) { + expect(args).toEqual([{ companyId: "company-one", taskId: "task-one" }]); + } + completionSweepMock.mockClear(); + publishLiveEvent({ companyId: "company-two", type: "activity.logged", payload: { action: "issue.updated" } }); + expect(completionSweepMock).not.toHaveBeenCalled(); + }); + it("never invokes the retired review detector at startup or on periodic recovery", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, diff --git a/server/src/__tests__/tool-gateway-service.test.ts b/server/src/__tests__/tool-gateway-service.test.ts index 3e9b984672d..0db14e6f47f 100644 --- a/server/src/__tests__/tool-gateway-service.test.ts +++ b/server/src/__tests__/tool-gateway-service.test.ts @@ -1,3 +1,5 @@ +import { subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -219,6 +221,8 @@ describeEmbeddedPostgres("tool gateway service", () => { runId: run.id, }); + const notified = vi.fn(); + const unsubscribe = subscribeDeliveryWork(db, DELIVERY_QUEUES.toolAction, notified); await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", @@ -228,6 +232,9 @@ describeEmbeddedPostgres("tool gateway service", () => { details: { instructions: expect.stringContaining("A human approval card was posted on task") }, }); + unsubscribe(); + expect(notified).toHaveBeenCalledTimes(1); + await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", diff --git a/server/src/__tests__/work-scheduler.test.ts b/server/src/__tests__/work-scheduler.test.ts new file mode 100644 index 00000000000..dcfde0220a0 --- /dev/null +++ b/server/src/__tests__/work-scheduler.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createWorkScheduler } from "../services/work-scheduler.js"; +afterEach(() => vi.useRealTimers()); +describe("work scheduler", () => { + it("owns one timer, replaces named deadlines, and exposes the earliest wake", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(); + const first = vi.fn(), second = vi.fn(); + s.schedule("first", 100, first); + s.schedule("second", 50, second); + s.schedule("first", 20, first); + expect(vi.getTimerCount()).toBe(1); + expect(s.nextWakeAt()).toBe(20); + await vi.advanceTimersByTimeAsync(20); + expect(first).toHaveBeenCalledTimes(1); + expect(second).not.toHaveBeenCalled(); + expect(s.nextWakeAt()).toBe(50); + s.cancel("second"); + expect(s.nextWakeAt()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + it("honors cancellation during dispatch and isolates a failing callback", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const warn = vi.spyOn(process, "emitWarning").mockImplementation(() => {}); + const s = createWorkScheduler(), cancelled = vi.fn(), later = vi.fn(); + s.schedule("first", 10, () => { s.cancel("cancelled"); throw new Error("failure"); }); + s.schedule("cancelled", 10, cancelled); + s.schedule("later", 20, later); + await vi.advanceTimersByTimeAsync(20); + expect(cancelled).not.toHaveBeenCalled(); + expect(later).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); s.stop(); + }); + it("does not fire distant deadlines early and cannot restart after stop", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const s = createWorkScheduler(), run = vi.fn(); + s.schedule("far", 3_000_000_000, run); + await vi.advanceTimersByTimeAsync(2_147_483_647); + expect(run).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(852_516_353); + expect(run).toHaveBeenCalledTimes(1); + s.stop(); s.schedule("late", Date.now(), run); + expect(vi.getTimerCount()).toBe(0); + expect(() => createWorkScheduler().schedule("bad", NaN, run)).toThrow("finite"); + }); +}); diff --git a/server/src/app.ts b/server/src/app.ts index bda8217716a..1697f311303 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -1,3 +1,5 @@ +import { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; import { isIdleTaskDrainActive, trackIdleWork } from "./services/task-admission.js"; import { customerSuccessRoutes } from "./routes/customer-success.js"; @@ -483,11 +485,13 @@ export async function createApp( serverPort: number; storageService: StorageService; feedbackExportService?: { + hasPendingFeedbackTraces(): Promise; flushPendingFeedbackTraces(input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; + signal?: AbortSignal; }): Promise; }; databaseBackupService?: InstanceDatabaseBackupService; @@ -877,7 +881,6 @@ export async function createApp( // route prefixes, so this dependency does not change issue-route precedence. api.use(issueRoutes(db, opts.storageService, { chatRunRetries: chatChannels, - feedbackExportService: opts.feedbackExportService, pluginWorkerManager: workerManager, approveToolActionRequest: (input) => toolGateway.approveActionRequest(input), declineToolActionRequest: (input) => toolGateway.declineActionRequest(input), @@ -1190,40 +1193,19 @@ export async function createApp( jobCoordinator.start(); scheduler.start(); - let feedbackExportShuttingDown = false; - let feedbackExportTimer: ReturnType | null = null; - const disableFeedbackExportFlushes = () => { - feedbackExportShuttingDown = true; - if (feedbackExportTimer) { - clearInterval(feedbackExportTimer); - feedbackExportTimer = null; - } - }; - const flushPendingFeedbackExports = async () => { - if (feedbackExportShuttingDown || isWarmStandby() || isIdleTaskDrainActive()) return; - try { - await opts.feedbackExportService?.flushPendingFeedbackTraces(); - } catch (err) { - if (isDatabaseConnectionUnavailableError(err)) { - disableFeedbackExportFlushes(); - logger.warn( - { err }, - "Disabling pending feedback export flushes because the database is unavailable", - ); - return; - } - logger.error({ err }, "Failed to flush pending feedback exports"); - } - }; - - feedbackExportTimer = opts.feedbackExportService - ? setInterval(() => { - void flushPendingFeedbackExports(); - }, FEEDBACK_EXPORT_FLUSH_INTERVAL_MS) - : null; - feedbackExportTimer?.unref?.(); + const deliveryWork = createDeliveryWorkCoordinator({ + owner: db, + canRun: () => !isWarmStandby() && !isIdleTaskDrainActive(), + canReconcile: () => !isWarmStandby(), + onError: (err, queue) => logger.error({ err, queue }, "Delivery reconciliation failed"), + }); + app.locals.deliveryWork = deliveryWork; if (opts.feedbackExportService) { - void flushPendingFeedbackExports(); + deliveryWork.register(DELIVERY_QUEUES.feedback, { + retryMs: FEEDBACK_EXPORT_FLUSH_INTERVAL_MS, + run: (signal) => opts.feedbackExportService!.flushPendingFeedbackTraces({ signal }), + hasPending: () => opts.feedbackExportService!.hasPendingFeedbackTraces(), + }); } emailChannels.start(); const flushChatPublications = async () => { @@ -1389,7 +1371,7 @@ export async function createApp( await publicMcpEvents?.stop(); await dotMcpEvents?.stop(); jobCoordinator.stop(); - disableFeedbackExportFlushes(); + await deliveryWork.stop(); unsubscribeChatPublicationSignals(); chatReconciliation.stop(); if (chatPublicationTimer) { diff --git a/server/src/index.ts b/server/src/index.ts index 4a887104178..6bef07ad23a 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,9 +1,11 @@ +import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; +import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./services/delivery-work-coordinator.js"; import { isIdleTaskDrainActive, beginIdleTrackedWork, trackIdleWork } from "./services/task-admission.js"; import { markIdleStartupComplete } from "./services/idle-local-work.js"; import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "./services/idle-database-backup.js"; import { cloudWarmStandbyServerOptions } from "./middleware/cloud-warm-standby.js"; import { createCloudWarmStandby } from "./services/cloud-warm-standby.js"; -import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; import { chatCompletionDeliveryService } from "./services/chat-completion-delivery.js"; /// // Kicks off the OTel bootstrap as early as possible (no-op unless @@ -1252,20 +1254,31 @@ async function startServerWithDatabaseTeardown( const ENVIRONMENT_LEASE_CLEANUP_SWEEP_BACKOFF_MS = 5 * 60 * 1000; const environmentLeaseCleanupHeartbeat = accountingHeartbeat; const chatCompletionDeliveries = chatCompletionDeliveryService(db as any, environmentLeaseCleanupHeartbeat); - // Activity publication happens after the status transaction commits. This is - // a best-effort fast path; the durable outbox and sweeps remain authoritative. - const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped || event.type !== "activity.logged" || - event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) - .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); - }); - server.on("close", unsubscribeChatCompletions); const connectionDeliveries = connectionIntentDeliveryService(db as any, environmentLeaseCleanupHeartbeat); const questionResponseDeliveries = questionResponseDeliveryService(db as any, { heartbeat: environmentLeaseCleanupHeartbeat, resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); + const deliveryWork = app.locals.deliveryWork as ReturnType; + const deliveryWorkers = ([ + [DELIVERY_QUEUES.chatCompletion, chatCompletionDeliveries], + [DELIVERY_QUEUES.connection, connectionDeliveries], + [DELIVERY_QUEUES.question, questionResponseDeliveries], + [DELIVERY_QUEUES.toolAction, app.locals.toolActionDeliveries as ReturnType], + ] as const).map(([queue, service]) => deliveryWork.register(queue, { + retryMs: config.heartbeatSchedulerIntervalMs, + run: () => service.sweepPending(), + hasPending: () => service.hasPending(), + })); + // Preserve the existing task-scoped activity fast path. Claims in the + // delivery service arbitrate with the coordinator's startup/recovery sweep. + const unsubscribeChatCompletions = subscribeAllCompanyLiveEvents(event => { + if (isIdleTaskDrainActive() || isWarmStandby() || heartbeatSchedulerStopped || event.type !== "activity.logged" || + event.payload.action !== "issue.updated" || typeof event.payload.entityId !== "string") return; + trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending({ companyId: event.companyId, taskId: event.payload.entityId }) + .catch(err => logger.error({ err }, "post-commit chat completion delivery failed"))); + }); + server.on("close", unsubscribeChatCompletions); const runEnvironmentLeaseCleanupSweep = (backoffMs: number) => environmentLeaseCleanupHeartbeat .sweepPendingCleanupLeases({ backoffMs }) @@ -1316,18 +1329,9 @@ async function startServerWithDatabaseTeardown( })); }; - await chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "startup chat completion delivery recovery failed")); - await connectionDeliveries.sweepPending(); await app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "startup tool review recovery failed")); await app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "startup gateway token cleanup failed")); - await app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "startup tool review delivery sweep failed")); - await questionResponseDeliveries.sweepPending().then((result) => { - if (result.scanned > 0) { - logger.info(result, "startup question-response delivery sweep completed"); - } - }).catch((err) => { - logger.error({ err }, "startup question-response delivery sweep failed"); - }); + await Promise.all(deliveryWorkers.map(worker => worker.ready)); scheduleGitHubConnectionEventPoll(); scheduleGitHubConnectionContinuitySweep(); @@ -1781,23 +1785,10 @@ async function startServerWithDatabaseTeardown( logger.error({ err }, "periodic secret proposal expiry sweep failed"); })); - trackHeartbeatSchedulerWork(chatCompletionDeliveries.sweepPending().catch((err) => logger.error({ err }, "chat completion delivery failed"))); - trackHeartbeatSchedulerWork(accountingHeartbeat.reconcileCostAccounting().catch((err) => logger.error({ err }, "Cost accounting recovery failed"))); - trackHeartbeatSchedulerWork(connectionDeliveries.sweepPending().catch((err) => logger.error({ err }, "connection continuation delivery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.sweepActionReviews().catch((err: unknown) => logger.error({ err }, "tool review recovery failed"))); trackHeartbeatSchedulerWork(app.locals.toolGateway.cleanupExpiredSessions().catch((err: unknown) => logger.error({ err }, "gateway token cleanup failed"))); - trackHeartbeatSchedulerWork(app.locals.toolActionDeliveries.sweepPending().catch((err: unknown) => logger.error({ err }, "tool review delivery sweep failed"))); - trackHeartbeatSchedulerWork(questionResponseDeliveries.sweepPending() - .then((result) => { - if (result.scanned > 0) { - logger.info(result, "periodic question-response delivery sweep completed"); - } - }) - .catch((err) => { - logger.error({ err }, "periodic question-response delivery sweep failed"); - })); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { @@ -1974,6 +1965,7 @@ async function startServerWithDatabaseTeardown( await systemdNotify(["--stopping", `--status=Stopping after ${signal}`]); heartbeatSchedulerStopped = true; unsubscribeChatCompletions(); + await deliveryWork.stop(); clearInterval(executionControlInterval); if (heartbeatSchedulerInterval) { clearInterval(heartbeatSchedulerInterval); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 9883c2aa6f6..108ba699448 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -3384,14 +3384,6 @@ export function issueRoutes( ChatChannelService, "prepareFailedChatRunRetry" | "processFailedChatRunRetry" >; - feedbackExportService?: { - flushPendingFeedbackTraces(input?: { - companyId?: string; - traceId?: string; - limit?: number; - now?: Date; - }): Promise; - }; searchService?: CompanySearchService; searchRateLimiter?: CompanySearchRateLimiter; pluginWorkerManager?: PluginWorkerManager; @@ -3647,7 +3639,6 @@ export function issueRoutes( const treeControlSvc = issueTreeControlFactory?.(db) ?? { getActivePauseHoldGate: async () => null, }; - const feedbackExportService = opts?.feedbackExportService; const environmentsSvc = environmentService(db); async function queueTaskWatchdogEvaluation( @@ -19005,21 +18996,8 @@ export function issueRoutes( ); } - if (result.sharingEnabled && result.traceId && feedbackExportService) { - try { - await feedbackExportService.flushPendingFeedbackTraces({ - companyId: issue.companyId, - traceId: result.traceId, - limit: 1, - }); - } catch (err) { - logger.warn( - { err, issueId: issue.id, traceId: result.traceId }, - "failed to flush shared feedback trace immediately", - ); - } - } - + // saveIssueVote has committed the export and notified its worker. + // Return the saved vote without waiting for an upload or backlog. res.status(201).json(result.vote); }, ); diff --git a/server/src/services/chat-completion-delivery.ts b/server/src/services/chat-completion-delivery.ts index 1cbd95b5009..18ddabd2548 100644 --- a/server/src/services/chat-completion-delivery.ts +++ b/server/src/services/chat-completion-delivery.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, asc, eq, inArray, lte, sql } from "drizzle-orm"; import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs, heartbeatRuns, issueComments, issueDocuments, issues, type Db } from "@paperclipai/db"; @@ -31,13 +32,16 @@ export async function recordChatHandoff(tx: Connection, task: Issue, actorRunId: } /** Must run on the same transaction as status projection (including native arbitration). */ -export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue) { - if (before.status === after.status) return; +export async function recordChatCompletion(tx: Connection, before: Issue, after: Issue): Promise { + if (before.status === after.status) return false; await tx.update(deliveries).set({ status: "superseded" }).where(and(eq(deliveries.taskId, after.id), eq(deliveries.companyId, after.companyId), inArray(deliveries.status, [...pending]))); - if (after.status !== "done") return; + if (after.status !== "done") return false; const [handoff] = await tx.select().from(handoffs).where(and(eq(handoffs.taskId, after.id), eq(handoffs.companyId, after.companyId))); - if (handoff) await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + if (!handoff) return false; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatCompletion); + await tx.insert(deliveries).values({ companyId: after.companyId, taskId: after.id, statusVersion: after.statusVersion }).onConflictDoNothing(); + return true; } async function loadAudience(tx: Connection, deliveryId: string) { @@ -243,5 +247,9 @@ export function chatCompletionDeliveryService(db: Db, heartbeat: { wakeup(agentI .orderBy(asc(deliveries.nextAttemptAt)).limit(100); for (const row of due) await deliver(row.id); } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: deliveries.id }).from(deliveries) + .where(inArray(deliveries.status, [...pending])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/connection-intent-delivery.ts b/server/src/services/connection-intent-delivery.ts index 540524bf4ea..c128d309fcd 100644 --- a/server/src/services/connection-intent-delivery.ts +++ b/server/src/services/connection-intent-delivery.ts @@ -167,7 +167,11 @@ export function connectionIntentDeliveryService(db: Db, heartbeat: Pick { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { + async function hasPending() { + return (await db.select({ id: connectionIntentDeliveries.interactionId }).from(connectionIntentDeliveries) + .where(isNull(connectionIntentDeliveries.deliveredAt)).limit(1)).length > 0; + } + return { deliver, hasPending, tryDeliver: async (id: string) => { try { await deliver(id); } catch { /* Persisted delivery remains due after its lease. */ } }, sweepPending: async () => { const rows = await db.select().from(connectionIntentDeliveries).where(and(isNull(connectionIntentDeliveries.deliveredAt), lte(connectionIntentDeliveries.nextAttemptAt, new Date()))) .orderBy(asc(connectionIntentDeliveries.nextAttemptAt)).limit(50); let failed = 0; diff --git a/server/src/services/delivery-work-coordinator.ts b/server/src/services/delivery-work-coordinator.ts new file mode 100644 index 00000000000..34a198671ba --- /dev/null +++ b/server/src/services/delivery-work-coordinator.ts @@ -0,0 +1,100 @@ +import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db"; +import { createWorkScheduler } from "./work-scheduler.js"; +import { type DeliveryQueue } from "./delivery-work-notifications.js"; +import { beginIdleTrackedWork } from "./task-admission.js"; + +export function createDeliveryWorkCoordinator(input: { + owner: object; + canRun: () => boolean; + // Idle drain may reconcile admitted writes without dispatching new work. + // Warm standby can disable both operations. + canReconcile?: () => boolean; + onError: (error: unknown, queue: DeliveryQueue) => void; +}) { + const scheduler = createWorkScheduler(); + const workers = new Map void; stop: () => Promise }>(); + let stopped = false; + + return { + // Only these registered tasks are represented; not yet the whole instance. + nextWakeAt: scheduler.nextWakeAt, + register(queue: DeliveryQueue, task: { + retryMs: number; + run: (signal: AbortSignal) => Promise; + hasPending: () => Promise; + }) { + if (stopped) throw new Error("Delivery coordinator is stopped"); + if (workers.has(queue)) throw new Error(`Delivery worker already registered: ${queue}`); + let dirty = false; + let scheduledAt: number | null = null; + let running: Promise | null = null; + let workerStopped = false; + let controller: AbortController | null = null; + let finishIntent: (() => void) | undefined; + function schedule(delay: number) { + if (workerStopped) return; + const at = Date.now() + delay; + if (scheduledAt !== null && scheduledAt <= at) return; + scheduledAt = at; + scheduler.schedule(queue, at, start); + } + function start() { + if (workerStopped || running) return; + scheduler.cancel(queue); + scheduledAt = null; + if (!input.canRun() && !(input.canReconcile?.() ?? false)) { schedule(task.retryMs); return; } + dirty = false; + const finish = beginIdleTrackedWork(); + const attempt = new AbortController(); + controller = attempt; + running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) + .then(() => workerStopped || !input.canRun() ? undefined : task.run(attempt.signal)).then(() => workerStopped ? false : task.hasPending()).then(pending => { + if (pending || databaseWorkPending(input.owner, queue)) schedule(task.retryMs); + else { finishIntent?.(); finishIntent = undefined; } + }).catch(error => { + schedule(task.retryMs); + // Logging must never turn a recoverable sweep failure into an + // unhandled rejection or prevent other queues from running. + try { if (!workerStopped) input.onError(error, queue); } catch { /* Recovery remains scheduled. */ } + }).finally(() => { + running = null; + controller = null; + finish(); + if (dirty) schedule(0); + }); + } + function wake() { + if (workerStopped) return; + dirty = true; + if (!running) schedule(0); + } + const unsubscribe = subscribeDatabaseWork(input.owner, queue, event => { + if (workerStopped) return; + if (databaseWorkPending(input.owner, queue)) { + finishIntent ??= beginIdleTrackedWork(); + // Give the transaction time to settle. Its settlement notification + // replaces this deadline with an immediate wake on the normal path. + if (!running) schedule(task.retryMs); + } + if (event === "settled") wake(); + }); + const worker = { wake, async stop() { + workerStopped = true; + controller?.abort(new Error("Delivery worker stopped")); + unsubscribe(); + scheduler.cancel(queue); + await running; + finishIntent?.(); finishIntent = undefined; + } }; + workers.set(queue, worker); + const ready = Promise.resolve().then(() => { start(); return running; }); + return { ready, wake }; + }, + async stop() { + stopped = true; + scheduler.stop(); + await Promise.all(Array.from(workers.values(), worker => worker.stop())); + workers.clear(); + }, + }; +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts new file mode 100644 index 00000000000..14b67085d1d --- /dev/null +++ b/server/src/services/delivery-work-notifications.ts @@ -0,0 +1,22 @@ +import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; + +/** Topics for existing durable queues; register intent before their writes. */ +export const DELIVERY_QUEUES = { + feedback: "feedback-exports", + chatCompletion: "chat-completions", + connection: "connection-continuations", + question: "question-responses", + toolAction: "tool-action-receipts", +} as const; +export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; + +export async function notifyDeliveryWork(transaction: object, queue: DeliveryQueue): Promise { + await signalDatabaseWork(transaction, queue); +} +// Public delivery notification observers only see settled work. The coordinator +// uses the lower-level lifecycle subscription to also fence in-flight writes. +export function subscribeDeliveryWork(owner: object, queue: DeliveryQueue, wake: () => void): () => void { + return subscribeDatabaseWork(owner, queue, event => { + if (event === "settled") wake(); + }); +} diff --git a/server/src/services/feedback-share-client.ts b/server/src/services/feedback-share-client.ts index 7fdf97ca4af..ae5a308829d 100644 --- a/server/src/services/feedback-share-client.ts +++ b/server/src/services/feedback-share-client.ts @@ -2,6 +2,8 @@ import { gzipSync } from "node:zlib"; import type { FeedbackTraceBundle } from "@paperclipai/shared"; import type { Config } from "../config.js"; +export const FEEDBACK_UPLOAD_TIMEOUT_MS = 30_000; + const DEFAULT_FEEDBACK_EXPORT_BACKEND_URL = "https://telemetry.paperclip.ing"; function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Date) { @@ -12,7 +14,7 @@ function buildFeedbackShareObjectKey(bundle: FeedbackTraceBundle, exportedAt: Da } export interface FeedbackTraceShareClient { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; } export function createFeedbackTraceShareClientFromConfig( @@ -23,7 +25,10 @@ export function createFeedbackTraceShareClientFromConfig( const endpoint = new URL("/feedback-traces", baseUrl).toString(); return { - async uploadTraceBundle(bundle) { + async uploadTraceBundle(bundle, shutdownSignal) { + const deadline = AbortSignal.timeout(FEEDBACK_UPLOAD_TIMEOUT_MS); + const signal = shutdownSignal ? AbortSignal.any([shutdownSignal, deadline]) : deadline; + signal.throwIfAborted(); const exportedAt = new Date(); const objectKey = buildFeedbackShareObjectKey(bundle, exportedAt); const requestBody = JSON.stringify({ @@ -33,6 +38,7 @@ export function createFeedbackTraceShareClientFromConfig( }); const response = await fetch(endpoint, { method: "POST", + signal, headers: { "content-type": "application/json", ...(token ? { authorization: `Bearer ${token}` } : {}), @@ -49,6 +55,8 @@ export function createFeedbackTraceShareClientFromConfig( } const payload = await response.json().catch(() => null) as { objectKey?: unknown } | null; + // A timed-out response body is not a successful upload acknowledgement. + signal.throwIfAborted(); return { objectKey: typeof payload?.objectKey === "string" && payload.objectKey.trim().length > 0 ? payload.objectKey diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts index 2bbc0361a3c..1f9ceb78721 100644 --- a/server/src/services/feedback.ts +++ b/server/src/services/feedback.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { readFile, readdir } from "node:fs/promises"; import path from "node:path"; import { and, asc, desc, eq, getTableColumns, gte, isNull, lte, ne, or } from "drizzle-orm"; @@ -111,7 +112,7 @@ const feedbackExportColumns = getTableColumns(feedbackExports); const instructionsSvc = agentInstructionsService(); type FeedbackTraceShareClient = { - uploadTraceBundle(bundle: FeedbackTraceBundle): Promise<{ objectKey: string }>; + uploadTraceBundle(bundle: FeedbackTraceBundle, signal?: AbortSignal): Promise<{ objectKey: string }>; }; type FeedbackServiceOptions = { @@ -1715,6 +1716,15 @@ async function buildFeedbackTraceBundleFromRow( } export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { + // Serialize explicit flushes and the worker's batches so callers cannot + // upload the same trace concurrently. HTTP vote handlers only save and notify. + let exportFlush: Promise = Promise.resolve(); + function serializeExportFlush(operation: () => Promise): Promise { + const result = exportFlush.then(operation, operation); + exportFlush = result.catch(() => undefined); + return result; + } + return { listIssueVotesForUser: async (issueId: string, authorUserId: string) => db @@ -1786,13 +1796,23 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { return row ? buildFeedbackTraceBundleFromRow(db, row) : null; }, + hasPendingFeedbackTraces: async () => { + const pending = options.shareClient + ? or(eq(feedbackExports.status, "pending"), eq(feedbackExports.status, "failed")) + : eq(feedbackExports.status, "pending"); + return (await db.select({ id: feedbackExports.id }).from(feedbackExports) + .where(pending).limit(1)).length > 0; + }, + flushPendingFeedbackTraces: async (input?: { companyId?: string; traceId?: string; limit?: number; now?: Date; - }) => { + signal?: AbortSignal; + }) => serializeExportFlush(async () => { const shareClient = options.shareClient; + input?.signal?.throwIfAborted(); if (!shareClient) { const filters = [eq(feedbackExports.status, "pending")]; if (input?.companyId) { @@ -1861,12 +1881,13 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { let failed = 0; for (const row of rows) { + if (input?.signal?.aborted) break; const attemptAt = input?.now ?? new Date(); attempted += 1; try { const bundle = await buildFeedbackTraceBundleFromRow(db, row); - await shareClient.uploadTraceBundle(bundle); + await shareClient.uploadTraceBundle(bundle, input?.signal); await db .update(feedbackExports) @@ -1900,7 +1921,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { sent, failed, }; - }, + }), saveIssueVote: async (input: { issueId: string; @@ -2071,6 +2092,7 @@ export function feedbackService(db: Db, options: FeedbackServiceOptions = {}) { }) .where(eq(feedbackVotes.id, savedVote.id)); + if (sharedWithLabs) await notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback); const [savedTrace] = await tx .insert(feedbackExports) .values({ diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index e0f22ca6120..c8a320374ca 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,15 @@ +import { + cancelHeartbeatNativeRun, + terminateHeartbeatRunProcess, + providerResourceDispositionForTerminalRun, + createHeartbeatRunControl, + type CancelRunOptions, +} from "./heartbeat/run-control.js"; +export { + leaseReleaseStatusForRunStatus, + providerResourceDispositionForTerminalRun, + cancelHeartbeatNativeRun, +} from "./heartbeat/run-control.js"; import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus, @@ -25,7 +37,6 @@ export { } from "./heartbeat/queue.js"; import { isProcessAlive, - CANCELLABLE_HEARTBEAT_RUN_STATUSES, createHeartbeatRecovery, } from "./heartbeat/recovery.js"; import { @@ -247,10 +258,8 @@ import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/as import { externalObjectService } from "./external-objects.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { dotRunnerBroker } from "./dot-runner-broker.js"; -import { isAiAuthenticationBlocked } from "./ai-auth-failure.js"; + import { - nativeRetryCancellationCommitCondition, - claimCancellationRequest, startupCancellationFence, } from "./native-runtime/native-cancellation-request.js"; import { @@ -262,7 +271,10 @@ import { AgentDirectoryReuseInvalidatedError, isAgentDirectoryCopy } from "./age import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { restoreNativeWorkspaceBestEffort } from "./native-runtime/native-workspace-best-effort.js"; -import { cancellationResultJson, canContinueCancelledRun, readRunCancellation, requestedRunCancellation } from "./run-cancellation.js"; +import { + cancellationResultJson, + readRunCancellation, +} from "./run-cancellation.js"; import { withNativeWorkspaceFinalizationOwnership, NativeWorkspaceFinalizationBusyError, @@ -278,11 +290,8 @@ import { hasWorkspaceRestoreFailure, } from "@paperclipai/shared"; -import { settleSlackConversation } from "./slack-conversation-lifecycle.js"; import { toolActionDeliveryService } from "./tool-action-delivery.js"; - -import { readQueuedInteractionResponse } from "./queued-interaction-response.js"; import { isConversation, prepareConversationTurn, @@ -293,29 +302,18 @@ import { getConversationConfirmationContext } from "./conversation-confirmation- import { recordNativeLocalProcessStop, } from "./native-local-process-stop.js"; -import { - isAcknowledgedNativeStop, - acknowledgedNativeStopExecutionHasStopped, -} from "./acknowledged-native-stop.js"; + import { legacyControllerBootId, renewLegacyControllerLease, watchLegacyControllerLease, } from "./legacy-controller-lease.js"; -import { completeTerminatedRemoteNativeSessionCleanup } from "../vendor/paperclip-runner/index.js"; + import { - hasRemoteTerminationReceipt, remoteExecutionHasStopped, - stoppedRemoteCleanupScopes, } from "./remote-execution-termination.js"; import { applyConnectorSkills, prepareConnectorSkillDelivery, resolveConnectorAssignments } from "./connector-runtime.js"; -import { - admitExplicitNativeContinuation, - undeliveredLegacyUserCommentIds, -} from "./explicit-native-continuation.js"; -import { - isCancelledNativeStartup, -} from "./cancelled-native-startup.js"; + import { connectionIntentService } from "./connection-intents.js"; import { prepareManagedAiRuntime, @@ -327,12 +325,11 @@ import { import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared"; import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js"; import { aiConnectionSessionCompatibilityInputs, managedAiSessionIdentityCompatible } from "./ai-connection-session.js"; -import { executionBlockerPredicate, getExecutionBlocker } from "./execution-blocker.js"; import { - CONVERSATION_CONTINUATION_POLICY, + getExecutionBlocker, +} from "./execution-blocker.js"; +import { claimedAdapterType, - runUsedConversationAdapter, - hasConversationContinuationPolicy, } from "./conversation-continuation.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; @@ -345,10 +342,8 @@ import { } from "./legacy-execution-recovery.js"; import { adapterExecutionControls, - captureAdapterStopOwnership, createAdapterExecutionControl, registerAdapterExecutionControl, - waitForAdapterStop, } from "./adapter-execution-control.js"; import { executionFailureRetryCount, type RunRetryEffect } from "../modules/run-retry/index.js"; import { buildHeartbeatRunStatusLiveEventPayload } from "./heartbeat-run-status-payload.js"; @@ -386,14 +381,11 @@ import { eq, exists, getTableColumns, - gt, gte, inArray, isNull, - isNotNull, lt, lte, - ne, notInArray, or, sql, @@ -404,7 +396,6 @@ import { ISSUE_DISPOSITION_REPAIR_RETRY_REASON, PROVIDER_QUOTA_MONITOR_SERVICE_NAME, isEnvironmentDriverSupportedForAdapter, - type EnvironmentLeaseStatus, type ExecutionWorkspace, type IssueExecutionMonitorClearReason, type IssueExecutionMonitorPolicy, @@ -419,11 +410,9 @@ import { chatActions, companies, completionContracts, - environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, - issueRecoveryActions, issues, nativeRunFinalizations, projects, @@ -449,10 +438,7 @@ export { scrubGitCredentialText }; import { publishLiveEvent } from "./live-events.js"; import { - queuedCommentIdsFromWakePayload, queuedCommentIdsFromRunContext, - withQueuedCommentIdsInWakePayload, - withQueuedCommentIdsInRunContext, } from "./issue-queued-comment-queue.js"; import { documentService } from "./documents.js"; import { getTaskPlanContext } from "./task-plan-context.js"; @@ -463,10 +449,8 @@ import { buildNativeExecutionInput, buildNativeExecutionWithCheckpoint, buildNativeRuntimeContext, - cancelNativeSession, claimWarmNativeInstructionCopy, nativeSessionWorkspaceScope, - closeWarmNativeSessionsForEnvironment, reserveWarmNativeInstructionDirectory, dispatchNativeSessionResumptions, ensureNativeCompletionContract, @@ -560,7 +544,9 @@ import { reportRunFailure } from "./run-failure-report.js"; import { collectRunFailureSecretValues, type RunFailureReportOptions } from "./run-failure-diagnostics.js"; import { companySkillService } from "./company-skills.js"; -import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "./budgets.js"; +import { + budgetService, +} from "./budgets.js"; import { secretService, } from "./secrets.js"; @@ -647,7 +633,6 @@ import { } from "./workspace-operations.js"; import { isProcessGroupAlive, - terminateLocalService, } from "./local-service-supervisor.js"; import { isRuntimeOwnedGitBranch, @@ -709,7 +694,6 @@ import { } from "../modules/run-dispatch/index.js"; import { createWakeQueue, - WakeQueueApplicationError, type IssueSnapshot as WakeQueueIssueSnapshot, type PostCommitEffect as WakeQueuePostCommitEffect, type ReleaseRecoveryBlockedNoticeKind, @@ -1018,21 +1002,6 @@ export function getTaskDrainStatus(): { }; } -export function leaseReleaseStatusForRunStatus( - status: string | null | undefined, -): Extract { - if (status === "cancelled") return "expired"; - return status === "failed" || status === "timed_out" ? "failed" : "released"; -} - -export function providerResourceDispositionForTerminalRun( - desired: ProviderResourceDisposition | undefined, - status: string | null | undefined, -): ProviderResourceDisposition | undefined { - if (desired !== "keep_running") return desired; - return status === "succeeded" ? desired : "stop_and_retain"; -} - export interface NativeSandboxLifecycle { runnerProcess: "per_turn" | "warm"; sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; @@ -1283,33 +1252,6 @@ function isSameTaskScope(left: string | null, right: string | null) { return (left ?? null) === (right ?? null); } -async function terminateHeartbeatRunProcess(input: { - pid: number | null | undefined; - processGroupId: number | null | undefined; - graceMs?: number; - signal?: NodeJS.Signals; -}) { - const pid = input.pid ?? null; - const processGroupId = input.processGroupId ?? null; - if (typeof pid !== "number" && typeof processGroupId !== "number") return; - - await terminateLocalService( - { - pid: - typeof pid === "number" && Number.isInteger(pid) && pid > 0 - ? pid - : (processGroupId ?? 0), - processGroupId: - typeof processGroupId === "number" && - Number.isInteger(processGroupId) && - processGroupId > 0 - ? processGroupId - : null, - }, - { forceAfterMs: input.graceMs, signal: input.signal }, - ); -} - export type HeartbeatEnvironmentRuntime = ReturnType< typeof environmentRuntimeService >; @@ -1355,38 +1297,6 @@ export interface HeartbeatServiceOptions { }) => Promise; } -export async function cancelHeartbeatNativeRun(input: { - db: Db; - runId: string; - reason: string; - runtimeMode: string | null; - cancellationRequestId?: string; - cancel?: ( - runId: string, - reason: string, - options: { db: Db; scope: "run"; cancellationRequestId?: string }, - ) => Promise<{ decision: unknown | null; auditId: string | null }>; -}) { - if (input.runtimeMode !== "native") { - return { decision: null, auditId: null }; - } - const cancellation = input.cancel - ? await input.cancel(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }) - : await cancelNativeSession(input.runId, input.reason, { - db: input.db, - scope: "run", - ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), - }); - if (!cancellation.decision || !cancellation.auditId) { - throw new Error("native_cancellation_outcome_not_audited"); - } - return cancellation; -} - class NativeSessionResumeScheduledError extends Error { constructor(readonly original: unknown) { super("Native session recovery has been scheduled for the same run."); @@ -2066,6 +1976,74 @@ export function heartbeatService( return runLifecycle.publishRunLifecyclePluginEvent(...args); } + const runControl = createHeartbeatRunControl(db, { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + }); + + const { + cancelInvocationsForAgentsInternal, + } = runControl; + + // Earlier factories capture these callbacks before run-control construction. + function cancelBudgetScopeWork(...args: Parameters) { + return runControl.cancelBudgetScopeWork(...args); + } + + function releaseIssueExecutionAndPromote(...args: Parameters) { + return runControl.releaseIssueExecutionAndPromote(...args); + } + + function releaseEnvironmentLeasesForRun(...args: Parameters) { + return runControl.releaseEnvironmentLeasesForRun(...args); + } + + function acknowledgeRemoteStop(...args: Parameters) { + return runControl.acknowledgeRemoteStop(...args); + } + + function resumeRemoteStopComments(...args: Parameters) { + return runControl.resumeRemoteStopComments(...args); + } + + function cancelRunInternal(...args: Parameters) { + return runControl.cancelRunInternal(...args); + } + + function resumeExecutionWaitComments(...args: Parameters) { + return runControl.resumeExecutionWaitComments(...args); + } + + function resumeQueuedCommentInterrupt(...args: Parameters) { + return runControl.resumeQueuedCommentInterrupt(...args); + } + + function resumeSavedLegacyComments(...args: Parameters) { + return runControl.resumeSavedLegacyComments(...args); + } + + function cancelActiveForAgentInternal(...args: Parameters) { + return runControl.cancelActiveForAgentInternal(...args); + } + // Applies the post-commit effects a wake-queue release returns, exactly as // the original release function did before its writes moved into that // module: publish + dispatch a promoted or recovery run, log a reopened @@ -2128,422 +2106,6 @@ export function heartbeatService( } const taskWatchdogs = taskWatchdogService(db, { enqueueWakeup }); - async function releaseEnvironmentLeasesForRun(input: { - runId: string; - companyId: string; - agentId: string; - status: string | null | undefined; - failureReason?: string | null; - providerResourceDisposition?: ProviderResourceDisposition; - nativeLifecycleTelemetry?: { - provider: string; - harness: string; - lifecycleMode: "per_turn" | "warm"; - sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; - }; - }) { - const leaseOwnerRun = await getRun(input.runId); - if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; - // Recovery can finish workspace copy-back outside the executor's finally. - // Successful copy-back does not earn warm retention for a failed turn. - const status = leaseOwnerRun?.status ?? input.status; - const providerResourceDisposition = providerResourceDispositionForTerminalRun( - input.providerResourceDisposition, - status, - ); - if (providerResourceDisposition === "destroy") { - const closeResult = await ( - options.closeWarmNativeSessionsForRun ?? - (async ({ runId, reason }) => { - const environmentIds = await db - .selectDistinct({ environmentId: environmentLeases.environmentId }) - .from(environmentLeases) - .where( - and( - eq(environmentLeases.heartbeatRunId, runId), - eq(environmentLeases.status, "active"), - ), - ) - .then((rows) => - rows.flatMap((row) => - typeof row.environmentId === "string" && - row.environmentId.length > 0 - ? [row.environmentId] - : [], - ), - ); - const aggregate = { closed: 0, busy: 0, failed: 0 }; - for (const environmentId of environmentIds) { - const result = await closeWarmNativeSessionsForEnvironment({ - environmentId, - reason, - }); - aggregate.closed += result.closed; - aggregate.busy += result.busy; - aggregate.failed += result.failed; - } - return aggregate; - }) - )({ - runId: input.runId, - reason: "terminal heartbeat run destroyed its environment lease", - }).catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to close warm native sessions before environment lease destruction", - ); - return { closed: 0, busy: 0, failed: 1 }; - }); - // A failed remote checkpoint cannot veto destruction of the isolated - // sandbox after the run stopped. Provider destruction supplies the exit - // proof; it does not turn the interrupted checkpoint into a success. - const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && - ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) - ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( - eq(environmentLeases.companyId, input.companyId), - eq(environmentLeases.heartbeatRunId, input.runId), - )) - : []; - const canDestroyRemote = remoteLeases.length > 0 && - remoteLeases.every(lease => lease.provider && lease.provider !== "local"); - if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { - logger.warn( - { runId: input.runId, warmNativeSessions: closeResult }, - "deferred environment lease destruction until warm native sessions close", - ); - return; - } - } - const releaseResult = await envOrchestrator - .releaseForRun({ - heartbeatRunId: input.runId, - companyId: input.companyId, - agentId: input.agentId, - status: leaseReleaseStatusForRunStatus(status), - failureReason: input.failureReason ?? undefined, - providerResourceDisposition, - nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, - }) - .catch((err) => { - logger.warn( - { err, runId: input.runId }, - "failed to release environment leases for heartbeat run", - ); - return null; - }); - for (const releaseError of releaseResult?.errors ?? []) { - logger.warn( - { - err: releaseError.error, - leaseId: releaseError.leaseId, - runId: input.runId, - }, - "failed to release environment lease for heartbeat run", - ); - } - await acknowledgeRemoteStop(input.runId, input.companyId); - } - - async function acknowledgeRemoteStop(runId: string, companyId: string) { - // The provider receipt arrives after adapter settlement. A remote ACP child - // has no host PID, so only this target-aware boundary can acknowledge Stop. - const stopped = await getRun(runId); - if (stopped?.runtimeMode === "native") { - const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); - for (const remoteCleanupScope of scopes ?? []) { - completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); - } - } - if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && - parseObject(stopped.resultJson?.executionCancellation).state === "requested" && - await runUsedConversationAdapter(db, stopped) && - await remoteExecutionHasStopped(db, companyId, runId)) { - await db.update(heartbeatRuns).set({ - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ - executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), - state: "acknowledged", acknowledgedAt: new Date().toISOString(), - proof: "provider_termination_receipt" }, - conversationContinuation: CONVERSATION_CONTINUATION_POLICY, - })}::jsonb`, - updatedAt: new Date(), - }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.status, "cancelled"))); - } - } - - async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { - if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; - const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt - ? await db.select().from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - const cancelledPreparation = run.runtimeMode === "legacy" && - await isCancelledNativeStartup(db, run, preparationCoordinator); - if (run.runtimeMode !== "native" && !cancelledPreparation && - parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && - !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; - const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); - if (!issueId) return; - const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; - const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, - leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), - )) : []; - // Stop ends one response. Saved user input can enter ordinary admission - // once its executor settles; it does not need a manufactured crash incident. - // The ordinary path still enforces task holds, ownership, and native session - // cleanup before starting a provider. - const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && - !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && - ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && - await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - const legacyContinuation = run.runtimeMode === "legacy" && - hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && - !(await getExecutionBlocker(db, run.companyId, issueId)); - if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; - const pending = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - requestId ? eq(agentWakeupRequests.id, requestId) : undefined, - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); - for (const wake of pending) { - if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; - let payload = parseObject(wake.payload); - if (payload.queuedCommentInterrupt) { - await resumeQueuedCommentInterrupt(wake.companyId, wake.id); - continue; - } - let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); - let commentId = deriveCommentId(context, payload); - let requestedByActorId = wake.requestedByActorId; - const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; - if (run.runtimeMode === "native" || cancelledPreparation) { - const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, - queuedCommentIdsFromWakePayload(payload)); - if (!ids.length) continue; - payload = withQueuedCommentIdsInWakePayload(payload, ids); - context = withQueuedCommentIdsInRunContext(context, ids); - commentId = ids.at(-1)!; - // A coalesced queue can contain several authors. Its saved comments, - // not the outer wake's first author, authorize the remaining input. - const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), - )); - requestedByActorId = author?.id ?? null; - } - if (legacyContinuation || stoppedNativeContinuation) { - if (!commentId || !run.finishedAt || !requestedByActorId || - !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), - sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), - eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), - isNull(issueComments.createdByRunId), - stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), - )); - if (!comment?.body.trim()) continue; - } else { - let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; - const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, - agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, - reason, commentId, successorRunId: randomUUID(), dryRun: true, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - onBlocked: (reason, message) => { wait = { reason, message }; }, - }); - if (!admitted) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); - continue; - } - } - // Re-enter ordinary admission with the original user's authority. It - // atomically adopts the deferred comments and still applies every gate. - await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], - reason, payload, contextSnapshot: context, - requestedByActorType: "user", requestedByActorId, - ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), - idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); - break; - } - } - - async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { - return resumeSavedLegacyComments(companyId, queueId, true, opts); - } - - async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { - const [wake] = await db.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - )); - if (!wake) return; - const payload = parseObject(wake.payload); - let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); - let commentIds = queuedCommentIdsFromWakePayload(payload); - const issueId = readNonEmptyString(payload.issueId); - if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; - const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); - if (!commentIds.length && !response) return; - // Resolved cards are immutable input. Only an explicit Interrupt click can - // authorize continuation across a stopped execution; ordinary completion - // uses normal deferred-wake promotion. - if (response && !interrupted) return; - if (!interrupted) { - commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); - if (!commentIds.length) return; - // The queue itself may have begun as a system wake. The saved human - // comment, not that wake's origin or mutable caller payload, is authority. - const [comment] = await db.select().from(issueComments).where(and( - eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), - eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), - isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), - )).orderBy(desc(issueComments.createdAt)).limit(1); - if (!comment?.body.trim() || !comment.authorUserId) return; - actorId = comment.authorUserId; - } - if (!actorId) return; - const agent = await getAgent(wake.agentId); - if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; - const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (active) return; - if (interrupted && opts?.retryCleanup) { - // Only the HTTP click grants an extra cleanup attempt. Periodic retries - // reuse the intent to deliver, never a fresh provider teardown budget. - const sourceRun = await db.transaction(async tx => { - const [task] = await tx.select().from(issues).where(and( - eq(issues.companyId, companyId), eq(issues.id, issueId), - )).for("update"); - const [current] = await tx.select().from(agentWakeupRequests).where(and( - eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), - eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, - )); - if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || - !current || (!queuedCommentIdsFromWakePayload(current.payload).length && - !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; - const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.agentId, wake.agentId), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), - )).limit(1); - if (successor) return null; - const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); - const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), - eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), - inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), - sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, - )).then(rows => rows[0]) : null; - if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; - // Older ephemeral leases recorded successful cleanup without a provider - // receipt. Re-verify them through the recorded teardown path; a timestamp - // alone never certifies termination. Retained/reusable resources stay put. - const historical = await tx.select().from(environmentLeases).where(and( - eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), - eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), - inArray(environmentLeases.status, ["released", "expired", "failed"]), - )).for("update"); - for (const lease of historical) { - if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; - // Provider resource IDs identify physical sandboxes. A lease in any - // company can still own this resource; never destroy it on behalf of - // this company. This existence-only guard exposes no foreign data. - const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( - ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), - eq(environmentLeases.providerLeaseId, lease.providerLeaseId), - or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), - )).limit(1); - if (otherOwner) continue; - await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) - .where(eq(environmentLeases.id, lease.id)); - } - return run; - }); - if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { - companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", - } }); - } - const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); - delete deliveryPayload.queuedCommentInterrupt; - await enqueueWakeup(wake.agentId, { - source: "on_demand", triggerDetail: "manual", reason: "issue_commented", - payload: deliveryPayload, contextSnapshot: response - ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, - responsibleUserId: actorId } - : withQueuedCommentIdsInRunContext({ - issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, - }, commentIds), - requestedByActorType: "user", requestedByActorId: actorId, - ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), - issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, - idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, - }, queueId); - } - - async function resumeExecutionWaitComments() { - if ((await getSchedulingSuppression()).suppressed) return; - const waits = await db.select({ wake: agentWakeupRequests }) - .from(agentWakeupRequests) - .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), - sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, - eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) - .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( - eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), - executionBlockerPredicate(), - ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), - eq(agentWakeupRequests.requestedByActorType, "user"), - sql`${agentWakeupRequests.payload}->'executionWait' is not null`, - sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - notInArray(issues.status, ["done", "cancelled"]))) - .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); - const seen = new Set(); - for (const { wake } of waits) { - const issueId = String(wake.payload?.issueId); - if (seen.has(issueId)) continue; - seen.add(issueId); - // Advance the cursor even for invalid evidence so one damaged task cannot - // starve later requests in this bounded scan. Preserve concurrent edits. - const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( - eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"), - lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), - )).returning({ id: agentWakeupRequests.id }); - if (!claimed) continue; - // Match normal admission's deterministic current blocker selection. An - // arbitrary historical action must not choose the retry's source run. - const blocker = await getExecutionBlocker(db, wake.companyId, issueId); - const sourceId = blocker?.runId; - if (!sourceId || !isUuidLike(sourceId)) continue; - const run = await getRun(sourceId, { includeExecutionEvidence: true }); - if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; - if (canContinueCancelledRun(run)) { - await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); - }); - continue; - } - await resumeRemoteStopComments(run, wake.id).catch(err => { - logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); - }); - } - } - async function getAgentInvokability( agent: typeof agents.$inferSelect | null | undefined, ) { @@ -10623,683 +10185,6 @@ export function heartbeatService( } } - async function releaseIssueExecutionAndPromote( - run: Pick, - options: { suppressImmediateRecovery?: boolean } = {}, - ) { - try { - const source = await getRun(run.id); - const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ - companyId: run.companyId, - runId: run.id, - now: new Date(), - // A durable authentication card owns recovery. This covers the review path, - // while continuation classification blocks periodic generic retries. - suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), - }); - await applyWakeQueuePostCommitEffects(postCommitEffects); - const completed = await getRun(run.id); - const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) - ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; - if (completed?.status === "succeeded" && issueId) { - await settleSlackConversation(db, run.companyId, issueId).catch((err) => { - logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); - }); - } - } catch (error) { - if ( - error instanceof WakeQueueApplicationError && - error.code === "responsible_user_unresolved" - ) { - // Every other `responsible_user_unresolved` HttpError in this file - // carries `code` inside its own `details`; match that shape here too. - throw new HttpError(422, error.message, { code: error.code, ...error.details }); - } - throw error; - } - } - - async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { - const runIssueId = sql< - string | null - >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await db - .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) - .from(heartbeatRuns) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${runIssueId}`, - ), - ) - .where( - and( - eq(heartbeatRuns.companyId, companyId), - createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, - inArray(heartbeatRuns.status, [ - ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, - ]), - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function listProjectScopedWakeupIds( - companyId: string, - projectId: string, - database: Db = db, - ) { - const wakeIssueId = sql< - string | null - >`${agentWakeupRequests.payload} ->> 'issueId'`; - const effectiveProjectId = sql< - string | null - >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; - - const rows = await database - .selectDistinctOn([agentWakeupRequests.id], { - id: agentWakeupRequests.id, - }) - .from(agentWakeupRequests) - .leftJoin( - issues, - and( - eq(issues.companyId, companyId), - sql`${issues.id}::text = ${wakeIssueId}`, - ), - ) - .where( - and( - eq(agentWakeupRequests.companyId, companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - sql`${effectiveProjectId} = ${projectId}`, - ), - ); - - return rows.map((row) => row.id); - } - - async function cancelPendingWakeupsForBudgetScope( - scope: BudgetEnforcementScope, - ) { - return withCurrentBudgetEnforcement(db, scope, async (tx) => { - const now = new Date(); - let wakeupIds: string[] = []; - - if (scope.scopeType === "company") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else if (scope.scopeType === "agent") { - wakeupIds = await tx - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - eq(agentWakeupRequests.companyId, scope.companyId), - eq(agentWakeupRequests.agentId, scope.scopeId), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - } else { - wakeupIds = await listProjectScopedWakeupIds( - scope.companyId, - scope.scopeId, - tx, - ); - } - - if (wakeupIds.length === 0) return 0; - - await tx - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: "Cancelled due to budget pause", - updatedAt: now, - }) - .where(and( - inArray(agentWakeupRequests.id, wakeupIds), - inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), - isNull(agentWakeupRequests.runId), - scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, - )); - - return wakeupIds.length; - }); - } - - type CancelRunOptions = { - budgetEnforcement?: BudgetEnforcementScope; - /** Optional board request identity, atomically reserved for native Stop. */ - cancellationRequestId?: string; - cancellationRequestedByUserId?: string | null; - errorCode?: string; - resultJson?: Record; - eventMessage?: string; - eventPayload?: Record; - /** Per-call graceful process shutdown window, bounded to a safe range. */ - terminationGraceMs?: number; - /** Caller is immediately scheduling an explicit successor path. */ - suppressImmediateRecovery?: boolean; - }; - - function cancellationTerminationGraceMs( - configuredGraceSec: number, - requestedGraceMs: number | undefined, - ) { - const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; - if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { - return configuredGraceMs; - } - return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); - } - - async function cancelRunInternal( - runId: string, - reason = "Cancelled by control plane", - options: CancelRunOptions = {}, - ) { - let run = await getRun(runId); - if (!run) throw notFound("Heartbeat run not found"); - if (options.cancellationRequestId) { - run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); - } - // The caller claim checked retry eligibility under both durable row locks. - // This is only a cancellation candidate: dispatch rechecks the coordinator - // under lock, and the final CAS requires its own unchanged acknowledged - // retry fence. Re-reading only retryable_failure here would strand replay. - const pendingNativeRetry = - run.runtimeMode === "native" && run.status === "failed" && ( - Boolean(options.cancellationRequestId) || await db - .select({ runId: nativeRunFinalizations.runId }) - .from(nativeRunFinalizations) - .where( - and( - eq(nativeRunFinalizations.runId, run.id), - eq(nativeRunFinalizations.companyId, run.companyId), - eq(nativeRunFinalizations.phase, "retryable_failure"), - ), - ) - .then((rows) => rows.length > 0) - ); - if ( - !pendingNativeRetry && - !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( - run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], - ) - ) - return run; - const agent = await getAgent(run.agentId); - const errorCode = options.errorCode ?? "cancelled"; - const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); - - const pendingProcessCancellation = processRunCancellationSettlements.get( - run.id, - ); - if (pendingProcessCancellation) { - await pendingProcessCancellation.settled; - if (pendingProcessCancellation.failed) - throw pendingProcessCancellation.error; - return getRun(run.id); - } - const running = runningProcesses.get(run.id); - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - const control = stopOwnership?.control; - // Capture the existing adapter owner before waiting on the run lock. Then - // atomically fence preparation and refresh the selected runtime, so Stop - // cannot miss a native handoff that won after its first read. - // Established legacy processes must still be stopped if the database is - // unavailable. Only native or not-yet-dispatched preparation needs this - // additional durable fence before its existing cancellation path. - if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { - const fence = async (tx: Db) => { - const [fenced] = await tx.update(heartbeatRuns).set({ - // Record handoff intent before native cancellation can finalize and release - // the run. Only its audited stop acknowledgement suppresses recovery. - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || - ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true - ? { reassignmentStopRequested: true } : {})}::jsonb || - ${JSON.stringify({ cancellation })}::jsonb || - jsonb_build_object('startupCancellation', jsonb_build_object( - 'requestedAt', ${new Date().toISOString()}::text, - 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' - and ${heartbeatRuns.runtimeModeResolvedAt} is null - and ${heartbeatRuns.executionStage} = 'preparing' - and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) - ))`, - }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, - pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - ))).returning(); - return fenced ?? null; - }; - let fenced: typeof run | null; - try { - fenced = options.budgetEnforcement - ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) - : await fence(db); - } catch (error) { - stopOwnership?.release(); - throw error; - } - if (!fenced) { - stopOwnership?.release(); - return getRun(runId); - } - run = fenced; - } - const resultJson = { ...(agent - ? { - ...mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: parseObject(run.resultJson), - errorCode, - errorMessage: reason, - }), - ...(options.resultJson ?? {}), - } - : options.resultJson), cancellation }; - - try { - let releaseProcessCancellation: (() => void) | undefined; - const processCancellationSettlement = - run.runtimeMode !== "native" && - !control && - running - ? { - settled: new Promise((resolve) => { - releaseProcessCancellation = resolve; - }), - failed: false, - error: undefined as unknown, - } - : undefined; - if (processCancellationSettlement) { - // No await between joining an existing owner above and registering ours. - processRunCancellationSettlements.set( - run.id, - processCancellationSettlement, - ); - } - const cancellation = await (async () => { - try { - if (control) { - await db - .update(heartbeatRuns) - .set({ - error: reason, - errorCode, - // Stop may have read the row before adapter settlement saved - // a required-copyback receipt. Merge into current database JSON - // and leave those server-owned fields to the restore recorder. - resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ - ...resultJson, - ...(!running ? { executionCancellation: { - state: "requested", requestedAt: new Date().toISOString(), - } } : {}), - }, true), - updatedAt: new Date(), - }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ); - control.controller.abort(new Error(reason)); - } - let terminationSettled = false; - try { - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), - }); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM - // can leave commands in their separate process groups alive. - signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, - graceMs: cancellationTerminationGraceMs( - running.graceSec, - options.terminationGraceMs, - ), - }); - } - terminationSettled = true; - } finally { - if ( - (!processCancellationSettlement || terminationSettled) && - runningProcesses.get(run.id) === running - ) { - runningProcesses.delete(run.id); - } - } - - if (control) { - await waitForAdapterStop(control.settled, undefined, { - runId: run.id, - adapterType: agent?.adapterType, - runtimeMode: run.runtimeMode, - abortRequested: control.controller.signal.aborted, - }, control); - const stopped = await getRun(run.id); - if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { - if ( - parseObject(stopped.resultJson?.executionCancellation).state !== - "acknowledged" - ) { - throw conflict( - "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", - ); - } - // The owned adapter already finalized this run and its lifecycle. - // Do not replay the process cancellation side effects below. - return { run: stopped, updated: false }; - } - } - - const finishedAt = new Date(); - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : {}; - return await setRunStatusFromLive( - run.id, - "cancelled", - pendingNativeRetry - ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] - : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], - { - finishedAt, - error: reason, - errorCode, - ...(resultJson || - Object.keys(persistedCancellationResult).length > 0 - ? { - resultJson: { - ...persistedCancellationResult, - ...(resultJson ?? {}), - // A scheduler placeholder has no process to acknowledge. - // Preserve its normal release policy instead of treating - // it as an operator stop of provider work. - ...(processCancellationSettlement && agent && running && ( - (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || - (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) - ) - ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: { - ...resultJson, - executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, - }, - errorCode, errorMessage: reason, - }) - : {}), - // The native cancellation helper may have advanced a durable - // pending intent to its acknowledged state after `run` was - // first read. Never let that stale snapshot overwrite the - // authoritative post-dispatch acknowledgement. - ...(Object.hasOwn( - persistedCancellationResult, - "nativeCancellation", - ) - ? { - nativeCancellation: - persistedCancellationResult.nativeCancellation, - } - : {}), - }, - } - : {}), - }, - undefined, - pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, - ); - } catch (error) { - if (processCancellationSettlement) { - processCancellationSettlement.failed = true; - processCancellationSettlement.error = error; - if (activeRunExecutions.has(run.id)) { - failedProcessRunCancellations.set( - run.id, - processCancellationSettlement, - ); - } - } - throw error; - } finally { - if (processCancellationSettlement) { - if ( - processRunCancellationSettlements.get(run.id) === - processCancellationSettlement - ) { - processRunCancellationSettlements.delete(run.id); - } - // Always settle waiters, including termination and DB errors. A - // failed Stop remains an error to its caller, never a cancellation - // receipt; the signal-bearing executor may then record failure. - releaseProcessCancellation?.(); - } - } - })(); - const cancelled = cancellation.run; - - if (cancellation.updated && cancelled) { - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: cancelled.finishedAt ?? new Date(), - error: reason, - }); - await appendRunEvent(cancelled, { - eventType: "lifecycle", - stream: "system", - level: "warn", - message: options.eventMessage ?? "run cancelled", - payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, - }); - await releaseIssueExecutionAndPromote(cancelled, { - suppressImmediateRecovery: options.suppressImmediateRecovery, - }); - await finalizeAgentStatus(run.agentId, "cancelled", undefined, { - wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), - }); - await startNextQueuedRunForAgent(run.agentId); - } - return cancelled; - } finally { - stopOwnership?.release(); - } - } - - async function cancelActiveForAgentInternal( - agentId: string, - reason = "Cancelled due to agent pause", - errorCode = "cancelled", - ) { - const agent = await getAgent(agentId); - const runs = await db - .select() - .from(heartbeatRuns) - .where( - and( - eq(heartbeatRuns.agentId, agentId), - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - ), - ); - - for (const run of runs) { - const stopOwnership = - run.runtimeMode !== "native" - ? captureAdapterStopOwnership(run.id) - : undefined; - try { - if (stopOwnership?.control) { - await cancelRunInternal(run.id, reason, { errorCode }); - continue; - } - if (run.runtimeMode === "native") { - await db.update(heartbeatRuns).set({ resultJson: - sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`, - }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - }); - } - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : parseObject(run.resultJson); - await setRunStatus(run.id, "cancelled", { - finishedAt: new Date(), - error: reason, - errorCode, - resultJson: { - ...persistedCancellationResult, - ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", { - resultJson: persistedCancellationResult, errorCode, errorMessage: reason, - }) : {}), - cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason), - }, - }); - - await setWakeupStatus(run.wakeupRequestId, "cancelled", { - finishedAt: new Date(), - error: reason, - }); - - const running = runningProcesses.get(run.id); - if (running) { - await terminateHeartbeatRunProcess({ - pid: running.child.pid, - processGroupId: running.processGroupId, - graceMs: Math.max(1, running.graceSec) * 1000, - }); - } - runningProcesses.delete(run.id); - await releaseIssueExecutionAndPromote(run); - } finally { - stopOwnership?.release(); - } - } - - return runs.length; - } - - async function cancelPendingWakeupsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - if (uniqueAgentIds.length === 0) return 0; - - const now = new Date(); - const wakeupIds = await db - .select({ id: agentWakeupRequests.id }) - .from(agentWakeupRequests) - .where( - and( - inArray(agentWakeupRequests.agentId, uniqueAgentIds), - inArray(agentWakeupRequests.status, [ - "queued", - "deferred_issue_execution", - ]), - sql`${agentWakeupRequests.runId} is null`, - ), - ) - .then((rows) => rows.map((row) => row.id)); - - if (wakeupIds.length === 0) return 0; - - await db - .update(agentWakeupRequests) - .set({ - status: "cancelled", - finishedAt: now, - error: reason, - updatedAt: now, - }) - .where(inArray(agentWakeupRequests.id, wakeupIds)); - - return wakeupIds.length; - } - - async function cancelInvocationsForAgentsInternal( - agentIds: string[], - reason: string, - ) { - const uniqueAgentIds = [...new Set(agentIds)].filter( - (agentId) => agentId.length > 0, - ); - let runsCancelled = 0; - for (const agentId of uniqueAgentIds) { - runsCancelled += await cancelActiveForAgentInternal(agentId, reason); - } - const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( - uniqueAgentIds, - reason, - ); - return { - agentIds: uniqueAgentIds, - runsCancelled, - wakeupsCancelled, - }; - } - - async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { - const runIds = scope.scopeType === "project" - ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) - : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, scope.companyId), - scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, - scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, - inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), - )).then((rows) => rows.map((row) => row.id)); - for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); - await cancelPendingWakeupsForBudgetScope(scope); - } - return { waitForRunExecutionDrain: async ( runId: string, diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts new file mode 100644 index 00000000000..7af48aeee59 --- /dev/null +++ b/server/src/services/heartbeat/run-control.test.ts @@ -0,0 +1,310 @@ +import { randomUUID } from "node:crypto"; +import type { Db } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, projects } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { adapterExecutionControls } from "../adapter-execution-control.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { createHeartbeatRunState } from "./run-state.js"; +import { createHeartbeatRunPreparation } from "./run-preparation.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; +import { createHeartbeatRunControl, type HeartbeatRunControlDependencies } from "./run-control.js"; + +vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); +vi.mock("../local-service-supervisor.js", async (importOriginal) => ({ + ...await importOriginal(), + terminateLocalService: vi.fn(async () => undefined), +})); + +type Run = typeof heartbeatRuns.$inferSelect; +function callbacks(db: Db) { + const state = createHeartbeatRunState(db); + const lifecycleDependencies = { + ...state, + ...createHeartbeatRunPreparation(db), + issuesSvc: { + addComment: vi.fn(), + listReviewAttention: vi.fn(), + }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, + companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, + treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), + getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), + emitTerminalAgentTaskRun: vi.fn(), + budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + const lifecycle = createHeartbeatLifecycle(db, lifecycleDependencies); + return { + ...state, + ...lifecycle, + getRun: vi.fn(state.getRun), + getAgent: vi.fn(state.getAgent), + setRunStatusFromLive: vi.fn(lifecycle.setRunStatusFromLive), + setRunStatus: vi.fn(lifecycle.setRunStatus), + setWakeupStatus: vi.fn(lifecycle.setWakeupStatus), + appendRunEvent: vi.fn(lifecycle.appendRunEvent), + finalizeAgentStatus: vi.fn(), + options: { closeWarmNativeSessionsForRun: vi.fn(async () => ({ closed: 1, busy: 0, failed: 0 })) }, + envOrchestrator: { releaseForRun: vi.fn(async () => ({ released: [], errors: [] })) }, + enqueueWakeup: lifecycleDependencies.enqueueWakeup, + startNextQueuedRunForAgent: vi.fn(), + sweepPendingCleanupLeases: vi.fn(), + timerClaimWasFirstHeartbeat: vi.fn(() => undefined), + wakeQueue: { releaseIssueExecution: vi.fn(async () => ({ outcome: { kind: "released" }, postCommitEffects: [] })) }, + applyWakeQueuePostCommitEffects: vi.fn(async () => undefined), + getSchedulingSuppression: vi.fn(async () => ({ suppressed: false })), + activeRunExecutions: new Set(), + processRunCancellationSettlements: new Map; failed: boolean; error?: unknown }>(), + failedProcessRunCancellations: new Map; failed: boolean; error?: unknown }>(), + } satisfies HeartbeatRunControlDependencies; +} +function guardedDatabase() { + const access = vi.fn(() => { throw new Error("Unexpected run-control database access"); }); + return { db: new Proxy({}, { get: access }) as Db, access }; +} + +describe("heartbeat run-control module boundary", () => { + afterEach(() => adapterExecutionControls.clear()); + + it("constructs without database access, cancellation, or cleanup", () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + createHeartbeatRunControl(database.db, deps); + expect(database.access).not.toHaveBeenCalled(); + for (const callback of Object.values(deps)) if (vi.isMockFunction(callback)) expect(callback).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + }); + + it("keeps an already completed run out of cancellation effects", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "finished", status: "succeeded", runtimeMode: "legacy" } as Run; + deps.getRun.mockResolvedValue(run); + expect(await createHeartbeatRunControl(database.db, deps).cancelRunInternal(run.id)).toBe(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("keeps a suppressed saved-input scan out of the database and wake admission", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true }); + await createHeartbeatRunControl(database.db, deps).resumeExecutionWaitComments(); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("ignores an empty cancellation batch", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + expect(await createHeartbeatRunControl(database.db, deps).cancelInvocationsForAgentsInternal(["", ""], "pause")).toEqual({ agentIds: [], runsCancelled: 0, wakeupsCancelled: 0 }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.getAgent).not.toHaveBeenCalled(); + }); + + it("waits for an owned adapter before admitting saved comments", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + const run = { id: "owned", status: "cancelled", runtimeMode: "legacy" } as Run; + adapterExecutionControls.set(run.id, {} as never); + await createHeartbeatRunControl(database.db, deps).resumeRemoteStopComments(run); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.enqueueWakeup).not.toHaveBeenCalled(); + }); + + it("retains leases while native execution ownership is unverified", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ runtimeMode: "native", status: "running", nativePhase: "terminal_failure", errorCode: "native_execution_ownership_unverified" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "held", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(database.access).not.toHaveBeenCalled(); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + }); + + it("uses the persisted terminal outcome when choosing warm resource retention", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "failed", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" }); + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ heartbeatRunId: "failed", status: "failed", providerResourceDisposition: "stop_and_retain" })); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("defers destruction when a warm session remains busy", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "failed", runtimeMode: "legacy" } as Run); + deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: 1, failed: 0 }); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "busy", companyId: "company", agentId: "agent", status: "failed", providerResourceDisposition: "destroy" }); + expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith({ runId: "busy", reason: "terminal heartbeat run destroyed its environment lease" }); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(database.access).not.toHaveBeenCalled(); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +if (!support.supported) console.warn(`Skipping run-control database tests: ${support.reason}`); +describe.skipIf(!support.supported)("heartbeat run-control database wiring", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-run-control-module"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + beforeEach(async () => { + vi.mocked(terminateLocalService).mockReset().mockResolvedValue(undefined); + await db.execute(sql`truncate table companies restart identity cascade`); + }); + afterEach(() => runningProcesses.clear()); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Run control", issuePrefix: "RC" }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Controlled agent", adapterType: "codex_local" }).returning(); + return { company, agent }; + } + async function liveRun(companyId: string, agentId: string) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "legacy", runtimeModeResolvedAt: new Date() }).returning(); + runningProcesses.set(run.id, { child: { pid: 424242 } as never, graceSec: 4, processGroupId: 424242 }); + return run; + } + + it("reports a missing run without entering cancellation", async () => { + const deps = callbacks(db); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(randomUUID())).rejects.toThrow("Heartbeat run not found"); + expect(deps.getAgent).not.toHaveBeenCalled(); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.appendRunEvent).not.toHaveBeenCalled(); + }); + + it("joins concurrent Stop requests across factories through the shared process barrier", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + let releaseTermination!: () => void; + const terminating = new Promise(resolve => { releaseTermination = resolve; }); + vi.mocked(terminateLocalService).mockReturnValue(terminating); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop", { terminationGraceMs: 1 }); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id, "Stop again"); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + releaseTermination(); + const results = await Promise.all([first, second]); + expect(results.map(result => result?.status)).toEqual(["cancelled", "cancelled"]); + expect(deps.setRunStatusFromLive).toHaveBeenCalledTimes(1); + expect(deps.appendRunEvent).toHaveBeenCalledTimes(1); + expect(deps.finalizeAgentStatus).toHaveBeenCalledTimes(1); + expect(deps.startNextQueuedRunForAgent).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(false); + expect(terminateLocalService).toHaveBeenCalledWith({ pid: 424242, processGroupId: 424242 }, { forceAfterMs: 100, signal: "SIGINT" }); + }); + + it("settles every joined Stop caller with the same termination failure", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + const cancellationLookup = vi.spyOn(deps.processRunCancellationSettlements, "get"); + deps.activeRunExecutions.add(run.id); + let rejectTermination!: (error: Error) => void; + vi.mocked(terminateLocalService).mockReturnValue(new Promise((_resolve, reject) => { rejectTermination = reject; })); + const first = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + await vi.waitFor(() => expect(deps.processRunCancellationSettlements.has(run.id)).toBe(true)); + const second = createHeartbeatRunControl(db, deps).cancelRunInternal(run.id); + const results = Promise.allSettled([first, second]); + await vi.waitFor(() => expect(cancellationLookup).toHaveBeenCalledTimes(2)); + const failure = new Error("provider did not stop"); + rejectTermination(failure); + expect(await results).toEqual([{ status: "rejected", reason: failure }, { status: "rejected", reason: failure }]); + expect(terminateLocalService).toHaveBeenCalledTimes(1); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect((await deps.getRun(run.id))?.status).toBe("running"); + }); + + it("retains failed Stop evidence and the owned process until a later request succeeds", async () => { + const { company, agent } = await fixture(); + const run = await liveRun(company.id, agent.id); + const deps = callbacks(db); + deps.activeRunExecutions.add(run.id); + const failure = new Error("termination failed"); + vi.mocked(terminateLocalService).mockRejectedValueOnce(failure); + await expect(createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).rejects.toBe(failure); + expect(deps.failedProcessRunCancellations.get(run.id)).toMatchObject({ failed: true, error: failure }); + expect(deps.processRunCancellationSettlements.size).toBe(0); + expect(runningProcesses.has(run.id)).toBe(true); + expect((await deps.getRun(run.id))?.status).toBe("running"); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(run.id)).toMatchObject({ status: "cancelled" }); + expect(terminateLocalService).toHaveBeenCalledTimes(2); + // The executor still owns removal of this evidence after it observes Stop. + expect(deps.failedProcessRunCancellations.get(run.id)?.error).toBe(failure); + }); + + it("deduplicates agent batches and cancels only unclaimed pending wakes", async () => { + const { company, agent } = await fixture(); + const [otherAgent] = await db.insert(agents).values({ companyId: company.id, name: "Other agent" }).returning(); + const [finished] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "succeeded" }).returning(); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "deferred_issue_execution" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "claimed" }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", runId: finished.id }, + { companyId: company.id, agentId: otherAgent.id, source: "assignment", status: "queued" }, + ]).returning(); + const deps = callbacks(db); + expect(await createHeartbeatRunControl(db, deps).cancelInvocationsForAgentsInternal([agent.id, "", agent.id], "pause")).toEqual({ agentIds: [agent.id], runsCancelled: 0, wakeupsCancelled: 2 }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "cancelled", "claimed", "queued", "queued"]); + expect(deps.getAgent).toHaveBeenCalledTimes(1); + }); + + it("rechecks stale budget enforcement before cancelling runs or queued wakes", async () => { + const { company, agent } = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "queued", runtimeMode: "legacy" }).returning(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: company.id, agentId: agent.id, source: "assignment", status: "queued" }).returning(); + const deps = callbacks(db); + await createHeartbeatRunControl(db, deps).cancelBudgetScopeWork({ companyId: company.id, scopeType: "agent", scopeId: agent.id, enforcement: { policyId: randomUUID(), version: 1 } }); + expect((await deps.getRun(run.id))?.status).toBe("queued"); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued"); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + expect(deps.finalizeAgentStatus).not.toHaveBeenCalled(); + expect(terminateLocalService).not.toHaveBeenCalled(); + }); + + it("limits project budget cancellation by company, effective project, and admission cutoff", async () => { + const { company, agent } = await fixture(); + const [otherCompany] = await db.insert(companies).values({ name: "Other company", issuePrefix: "OC" }).returning(); + const [otherAgent] = await db.insert(agents).values({ companyId: otherCompany.id, name: "Other company agent" }).returning(); + const projectRows = await db.insert(projects).values([ + { companyId: company.id, name: "Target" }, { companyId: company.id, name: "Override" }, + ]).returning(); + const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: projectRows[0].id, title: "Bound task", issueNumber: 1, identifier: "RC-1" }).returning(); + const old = new Date("2026-01-01T00:00:00Z"); + const cutoff = new Date("2026-01-02T00:00:00Z"); + const wakes = await db.insert(agentWakeupRequests).values([ + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: old, payload: { issueId: issue.id, projectId: projectRows[1].id } }, + { companyId: company.id, agentId: agent.id, source: "assignment", status: "queued", createdAt: cutoff, payload: { projectId: projectRows[0].id } }, + { companyId: otherCompany.id, agentId: otherAgent.id, source: "assignment", status: "queued", createdAt: old, payload: { projectId: projectRows[0].id } }, + ]).returning(); + await createHeartbeatRunControl(db, callbacks(db)).cancelBudgetScopeWork({ companyId: company.id, scopeType: "project", scopeId: projectRows[0].id, createdBefore: cutoff }); + const persisted = await db.select().from(agentWakeupRequests); + expect(wakes.map(wake => persisted.find(row => row.id === wake.id)?.status)).toEqual(["cancelled", "queued", "queued", "queued"]); + }); +}); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts new file mode 100644 index 00000000000..dd3f9d8dc3c --- /dev/null +++ b/server/src/services/heartbeat/run-control.ts @@ -0,0 +1,1360 @@ +import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; +import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js"; +import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js"; +import { + type WakeupOptions, +} from "./run-preparation.js"; +import { preserveWorkspaceRestoreRecoveryMetadataSql } from "../legacy-workspace-restore-recovery.js"; +import { isAiAuthenticationBlocked } from "../ai-auth-failure.js"; +import { + nativeRetryCancellationCommitCondition, + claimCancellationRequest, +} from "../native-runtime/native-cancellation-request.js"; +import { + canContinueCancelledRun, + readRunCancellation, + requestedRunCancellation, +} from "../run-cancellation.js"; +import { settleSlackConversation } from "../slack-conversation-lifecycle.js"; +import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; +import { + isAcknowledgedNativeStop, + acknowledgedNativeStopExecutionHasStopped, +} from "../acknowledged-native-stop.js"; +import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js"; +import { + hasRemoteTerminationReceipt, + remoteExecutionHasStopped, + stoppedRemoteCleanupScopes, +} from "../remote-execution-termination.js"; +import { + admitExplicitNativeContinuation, + undeliveredLegacyUserCommentIds, +} from "../explicit-native-continuation.js"; +import { isCancelledNativeStartup } from "../cancelled-native-startup.js"; +import { + executionBlockerPredicate, + getExecutionBlocker, +} from "../execution-blocker.js"; +import { + CONVERSATION_CONTINUATION_POLICY, + runUsedConversationAdapter, + hasConversationContinuationPolicy, +} from "../conversation-continuation.js"; +import { + adapterExecutionControls, + captureAdapterStopOwnership, + waitForAdapterStop, +} from "../adapter-execution-control.js"; +import { randomUUID } from "node:crypto"; +import { + and, + asc, + desc, + eq, + exists, + gt, + inArray, + isNull, + isNotNull, + lt, + lte, + ne, + notInArray, + or, + sql, +} from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; +import { + type EnvironmentLeaseStatus, +} from "@paperclipai/shared"; +import { + agentWakeupRequests, + companies, + environmentLeases, + heartbeatRuns, + issueComments, + issueRecoveryActions, + issues, + nativeRunFinalizations, +} from "@paperclipai/db"; +import { + conflict, + HttpError, + notFound, +} from "../../errors.js"; +import { logger } from "../../middleware/logger.js"; +import { + queuedCommentIdsFromWakePayload, + withQueuedCommentIdsInWakePayload, + withQueuedCommentIdsInRunContext, +} from "../issue-queued-comment-queue.js"; +import { + cancelNativeSession, + closeWarmNativeSessionsForEnvironment, +} from "../native-runtime/index.js"; +import { runningProcesses } from "../../adapters/index.js"; +import { parseObject } from "../../adapters/utils.js"; +import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js"; +import { + withCurrentBudgetEnforcement, + type BudgetEnforcementScope, +} from "../budgets.js"; +import { terminateLocalService } from "../local-service-supervisor.js"; +import { deriveCommentId } from "../../modules/run-dispatch/index.js"; +import { WakeQueueApplicationError } from "../../modules/wake-queue/index.js"; +import { isUuidLike } from "@paperclipai/shared"; +import { + type ProviderResourceDisposition, +} from "../environment-runtime.js"; + +import type { createHeartbeatRunState } from "./run-state.js"; +import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; +import type { createHeartbeatRetries, HeartbeatRetryDependencies } from "./retries.js"; +import type { createHeartbeatQueue } from "./queue.js"; +import type { createHeartbeatRecovery } from "./recovery.js"; +import type { createWakeQueue, PostCommitEffect as WakeQueuePostCommitEffect } from "../../modules/wake-queue/index.js"; +import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js"; + +type ProcessRunCancellationSettlement = { + settled: Promise; + failed: boolean; + error?: unknown; +}; + +/** Cancellation shares executor ownership and delegates lifecycle and admission effects. */ +export interface HeartbeatRunControlDependencies extends Pick, + "getRun" | "getAgent">, + Pick, + "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">, + Pick { + options: { + closeWarmNativeSessionsForRun?: (input: { + runId: string; + reason: string; + }) => Promise<{ closed: number; busy: number; failed: number }>; + }; + envOrchestrator: Pick, "releaseForRun">; + enqueueWakeup: ReturnType["enqueueWakeup"]; + startNextQueuedRunForAgent: ReturnType["startNextQueuedRunForAgent"]; + sweepPendingCleanupLeases: ReturnType["sweepPendingCleanupLeases"]; + timerClaimWasFirstHeartbeat: ReturnType["timerClaimWasFirstHeartbeat"]; + wakeQueue: Pick, "releaseIssueExecution">; + applyWakeQueuePostCommitEffects: (effects: WakeQueuePostCommitEffect[]) => Promise; + getSchedulingSuppression: () => Promise<{ suppressed: boolean }>; + // Scheduler and route service instances must retain the same in-process barriers. + activeRunExecutions: Set; + processRunCancellationSettlements: Map; + failedProcessRunCancellations: Map; +} + +export function leaseReleaseStatusForRunStatus( + status: string | null | undefined, +): Extract { + if (status === "cancelled") return "expired"; + return status === "failed" || status === "timed_out" ? "failed" : "released"; +} + +export function providerResourceDispositionForTerminalRun( + desired: ProviderResourceDisposition | undefined, + status: string | null | undefined, +): ProviderResourceDisposition | undefined { + if (desired !== "keep_running") return desired; + return status === "succeeded" ? desired : "stop_and_retain"; +} + +export async function terminateHeartbeatRunProcess(input: { + pid: number | null | undefined; + processGroupId: number | null | undefined; + graceMs?: number; + signal?: NodeJS.Signals; +}) { + const pid = input.pid ?? null; + const processGroupId = input.processGroupId ?? null; + if (typeof pid !== "number" && typeof processGroupId !== "number") return; + + await terminateLocalService( + { + pid: + typeof pid === "number" && Number.isInteger(pid) && pid > 0 + ? pid + : (processGroupId ?? 0), + processGroupId: + typeof processGroupId === "number" && + Number.isInteger(processGroupId) && + processGroupId > 0 + ? processGroupId + : null, + }, + { forceAfterMs: input.graceMs, signal: input.signal }, + ); +} + +export async function cancelHeartbeatNativeRun(input: { + db: Db; + runId: string; + reason: string; + runtimeMode: string | null; + cancellationRequestId?: string; + cancel?: ( + runId: string, + reason: string, + options: { db: Db; scope: "run"; cancellationRequestId?: string }, + ) => Promise<{ decision: unknown | null; auditId: string | null }>; +}) { + if (input.runtimeMode !== "native") { + return { decision: null, auditId: null }; + } + const cancellation = input.cancel + ? await input.cancel(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }) + : await cancelNativeSession(input.runId, input.reason, { + db: input.db, + scope: "run", + ...(input.cancellationRequestId ? { cancellationRequestId: input.cancellationRequestId } : {}), + }); + if (!cancellation.decision || !cancellation.auditId) { + throw new Error("native_cancellation_outcome_not_audited"); + } + return cancellation; +} + +function readNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +export type CancelRunOptions = { + budgetEnforcement?: BudgetEnforcementScope; + /** Optional board request identity, atomically reserved for native Stop. */ + cancellationRequestId?: string; + cancellationRequestedByUserId?: string | null; + errorCode?: string; + resultJson?: Record; + eventMessage?: string; + eventPayload?: Record; + /** Per-call graceful process shutdown window, bounded to a safe range. */ + terminationGraceMs?: number; + /** Caller is immediately scheduling an explicit successor path. */ + suppressImmediateRecovery?: boolean; + }; + +/** Run control binds cancellation, resource release, and saved-input resumption. */ +export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunControlDependencies) { + const { + getRun, + options, + envOrchestrator, + activeRunExecutions, + enqueueWakeup, + getAgent, + sweepPendingCleanupLeases, + getSchedulingSuppression, + wakeQueue, + applyWakeQueuePostCommitEffects, + processRunCancellationSettlements, + mergeRunStopMetadataForAgent, + setRunStatusFromLive, + failedProcessRunCancellations, + setWakeupStatus, + appendRunEvent, + finalizeAgentStatus, + timerClaimWasFirstHeartbeat, + startNextQueuedRunForAgent, + setRunStatus, + } = dependencies; + + async function releaseEnvironmentLeasesForRun(input: { + runId: string; + companyId: string; + agentId: string; + status: string | null | undefined; + failureReason?: string | null; + providerResourceDisposition?: ProviderResourceDisposition; + nativeLifecycleTelemetry?: { + provider: string; + harness: string; + lifecycleMode: "per_turn" | "warm"; + sandboxResource: "keep_running" | "stop_and_reuse" | "destroy_after_turn"; + }; + }) { + const leaseOwnerRun = await getRun(input.runId); + if (leaseOwnerRun && isNativeRunnerOwnershipHeld(leaseOwnerRun)) return; + // Recovery can finish workspace copy-back outside the executor's finally. + // Successful copy-back does not earn warm retention for a failed turn. + const status = leaseOwnerRun?.status ?? input.status; + const providerResourceDisposition = providerResourceDispositionForTerminalRun( + input.providerResourceDisposition, + status, + ); + if (providerResourceDisposition === "destroy") { + const closeResult = await ( + options.closeWarmNativeSessionsForRun ?? + (async ({ runId, reason }) => { + const environmentIds = await db + .selectDistinct({ environmentId: environmentLeases.environmentId }) + .from(environmentLeases) + .where( + and( + eq(environmentLeases.heartbeatRunId, runId), + eq(environmentLeases.status, "active"), + ), + ) + .then((rows) => + rows.flatMap((row) => + typeof row.environmentId === "string" && + row.environmentId.length > 0 + ? [row.environmentId] + : [], + ), + ); + const aggregate = { closed: 0, busy: 0, failed: 0 }; + for (const environmentId of environmentIds) { + const result = await closeWarmNativeSessionsForEnvironment({ + environmentId, + reason, + }); + aggregate.closed += result.closed; + aggregate.busy += result.busy; + aggregate.failed += result.failed; + } + return aggregate; + }) + )({ + runId: input.runId, + reason: "terminal heartbeat run destroyed its environment lease", + }).catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to close warm native sessions before environment lease destruction", + ); + return { closed: 0, busy: 0, failed: 1 }; + }); + // A failed remote checkpoint cannot veto destruction of the isolated + // sandbox after the run stopped. Provider destruction supplies the exit + // proof; it does not turn the interrupted checkpoint into a success. + const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && + ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) + ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), + )) + : []; + const canDestroyRemote = remoteLeases.length > 0 && + remoteLeases.every(lease => lease.provider && lease.provider !== "local"); + if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { + logger.warn( + { runId: input.runId, warmNativeSessions: closeResult }, + "deferred environment lease destruction until warm native sessions close", + ); + return; + } + } + const releaseResult = await envOrchestrator + .releaseForRun({ + heartbeatRunId: input.runId, + companyId: input.companyId, + agentId: input.agentId, + status: leaseReleaseStatusForRunStatus(status), + failureReason: input.failureReason ?? undefined, + providerResourceDisposition, + nativeLifecycleTelemetry: input.nativeLifecycleTelemetry, + }) + .catch((err) => { + logger.warn( + { err, runId: input.runId }, + "failed to release environment leases for heartbeat run", + ); + return null; + }); + for (const releaseError of releaseResult?.errors ?? []) { + logger.warn( + { + err: releaseError.error, + leaseId: releaseError.leaseId, + runId: input.runId, + }, + "failed to release environment lease for heartbeat run", + ); + } + await acknowledgeRemoteStop(input.runId, input.companyId); + } + + async function acknowledgeRemoteStop(runId: string, companyId: string) { + // The provider receipt arrives after adapter settlement. A remote ACP child + // has no host PID, so only this target-aware boundary can acknowledge Stop. + const stopped = await getRun(runId); + if (stopped?.runtimeMode === "native") { + const scopes = await stoppedRemoteCleanupScopes(db, companyId, runId); + for (const remoteCleanupScope of scopes ?? []) { + completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); + } + } + if (stopped?.runtimeMode === "legacy" && stopped.status === "cancelled" && + parseObject(stopped.resultJson?.executionCancellation).state === "requested" && + await runUsedConversationAdapter(db, stopped) && + await remoteExecutionHasStopped(db, companyId, runId)) { + await db.update(heartbeatRuns).set({ + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ + executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), + state: "acknowledged", acknowledgedAt: new Date().toISOString(), + proof: "provider_termination_receipt" }, + conversationContinuation: CONVERSATION_CONTINUATION_POLICY, + })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"))); + } + } + + async function resumeRemoteStopComments(run: typeof heartbeatRuns.$inferSelect, requestId?: string) { + if (!isHeartbeatRunTerminalStatus(run.status) || adapterExecutionControls.has(run.id)) return; + const [preparationCoordinator] = run.runtimeMode === "legacy" && run.status === "cancelled" && !run.runtimeModeResolvedAt + ? await db.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + const cancelledPreparation = run.runtimeMode === "legacy" && + await isCancelledNativeStartup(db, run, preparationCoordinator); + if (run.runtimeMode !== "native" && !cancelledPreparation && + parseObject(run.resultJson?.startupCancellation).beforeNativeSelection !== true && + !(await remoteExecutionHasStopped(db, run.companyId, run.id))) return; + const issueId = run.nativeIssueId ?? (typeof run.contextSnapshot?.issueId === "string" ? run.contextSnapshot.issueId : null); + if (!issueId) return; + const currentRun = run.runtimeMode === "native" ? await getRun(run.id, { includeExecutionEvidence: true }) : null; + const [coordinator] = currentRun ? await db.select({ phase: nativeRunFinalizations.phase, + leaseOwner: nativeRunFinalizations.leaseOwner }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), + )) : []; + // Stop ends one response. Saved user input can enter ordinary admission + // once its executor settles; it does not need a manufactured crash incident. + // The ordinary path still enforces task holds, ownership, and native session + // cleanup before starting a provider. + const stoppedNativeContinuation = currentRun && isAcknowledgedNativeStop(currentRun) && + !activeRunExecutions.has(run.id) && !coordinator?.leaseOwner && + ["terminal_failure", "applied"].includes(coordinator?.phase ?? "") && + await acknowledgedNativeStopExecutionHasStopped(db, currentRun) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + const legacyContinuation = run.runtimeMode === "legacy" && + hasConversationContinuationPolicy((await getRun(run.id, { includeExecutionEvidence: true }))?.resultJson) && + !(await getExecutionBlocker(db, run.companyId, issueId)); + if (run.runtimeMode !== "native" && run.runtimeMode !== "legacy") return; + const pending = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + requestId ? eq(agentWakeupRequests.id, requestId) : undefined, + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + )).orderBy(asc(agentWakeupRequests.requestedAt)).limit(50); + for (const wake of pending) { + if (wake.idempotencyKey?.startsWith("chat-inbound:")) continue; + let payload = parseObject(wake.payload); + if (payload.queuedCommentInterrupt) { + await resumeQueuedCommentInterrupt(wake.companyId, wake.id); + continue; + } + let context = parseObject(payload[DEFERRED_WAKE_CONTEXT_KEY]); + let commentId = deriveCommentId(context, payload); + let requestedByActorId = wake.requestedByActorId; + const reason = readNonEmptyString(context.wakeReason) ?? wake.reason; + if (run.runtimeMode === "native" || cancelledPreparation) { + const ids = await undeliveredLegacyUserCommentIds(db, run.companyId, issueId, run.agentId, + queuedCommentIdsFromWakePayload(payload)); + if (!ids.length) continue; + payload = withQueuedCommentIdsInWakePayload(payload, ids); + context = withQueuedCommentIdsInRunContext(context, ids); + commentId = ids.at(-1)!; + // A coalesced queue can contain several authors. Its saved comments, + // not the outer wake's first author, authorize the remaining input. + const [author] = await db.select({ id: issueComments.authorUserId }).from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), eq(issueComments.id, commentId), + )); + requestedByActorId = author?.id ?? null; + } + if (legacyContinuation || stoppedNativeContinuation) { + if (!commentId || !run.finishedAt || !requestedByActorId || + !["issue_commented", "issue_reopened_via_comment"].includes(reason ?? "")) continue; + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, run.companyId), eq(issueComments.issueId, issueId), + sql`${issueComments.id}::text = ${commentId}`, eq(issueComments.authorType, "user"), + eq(issueComments.authorUserId, requestedByActorId), isNull(issueComments.deletedAt), + isNull(issueComments.createdByRunId), + stoppedNativeContinuation ? undefined : gt(issueComments.createdAt, run.finishedAt), + )); + if (!comment?.body.trim()) continue; + } else { + let wait = { reason: "execution_recovery", message: "Waiting for execution recovery. Your message is saved." }; + const admitted = await admitExplicitNativeContinuation({ db, companyId: run.companyId, issueId, + agentId: run.agentId, actorType: wake.requestedByActorType, actorId: requestedByActorId, + reason, commentId, successorRunId: randomUUID(), dryRun: true, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + onBlocked: (reason, message) => { wait = { reason, message }; }, + }); + if (!admitted) { + await db.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + continue; + } + } + // Re-enter ordinary admission with the original user's authority. It + // atomically adopts the deferred comments and still applies every gate. + await enqueueWakeup(run.agentId, { source: wake.source as WakeupOptions["source"], triggerDetail: (wake.triggerDetail ?? undefined) as WakeupOptions["triggerDetail"], + reason, payload, contextSnapshot: context, + requestedByActorType: "user", requestedByActorId, + ...((run.runtimeMode === "native" || cancelledPreparation) ? { queuedCommentRequestId: wake.id } : {}), + idempotencyKey: `remote-stop-comment:${run.id}:${wake.id}` }, wake.id); + break; + } + } + + async function resumeQueuedCommentInterrupt(companyId: string, queueId: string, opts?: { retryCleanup?: boolean }) { + return resumeSavedLegacyComments(companyId, queueId, true, opts); + } + + async function resumeSavedLegacyComments(companyId: string, queueId: string, interrupted = false, opts?: { retryCleanup?: boolean }) { + const [wake] = await db.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + )); + if (!wake) return; + const payload = parseObject(wake.payload); + let actorId = readNonEmptyString(parseObject(payload.queuedCommentInterrupt).actorId); + let commentIds = queuedCommentIdsFromWakePayload(payload); + const issueId = readNonEmptyString(payload.issueId); + if (!issueId || wake.idempotencyKey?.startsWith("chat-inbound:")) return; + const response = await readQueuedInteractionResponse(db, companyId, issueId, payload); + if (!commentIds.length && !response) return; + // Resolved cards are immutable input. Only an explicit Interrupt click can + // authorize continuation across a stopped execution; ordinary completion + // uses normal deferred-wake promotion. + if (response && !interrupted) return; + if (!interrupted) { + commentIds = await undeliveredLegacyUserCommentIds(db, companyId, issueId, wake.agentId, commentIds); + if (!commentIds.length) return; + // The queue itself may have begun as a system wake. The saved human + // comment, not that wake's origin or mutable caller payload, is authority. + const [comment] = await db.select().from(issueComments).where(and( + eq(issueComments.companyId, companyId), eq(issueComments.issueId, issueId), + eq(issueComments.id, commentIds[commentIds.length - 1]!), eq(issueComments.authorType, "user"), + isNull(issueComments.createdByRunId), isNull(issueComments.deletedAt), + )).orderBy(desc(issueComments.createdAt)).limit(1); + if (!comment?.body.trim() || !comment.authorUserId) return; + actorId = comment.authorUserId; + } + if (!actorId) return; + const agent = await getAgent(wake.agentId); + if (!agent || agent.companyId !== companyId || (agent.adapterType === "paperclip_runner" && !response?.source.requiresFreshSession)) return; + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (active) return; + if (interrupted && opts?.retryCleanup) { + // Only the HTTP click grants an extra cleanup attempt. Periodic retries + // reuse the intent to deliver, never a fresh provider teardown budget. + const sourceRun = await db.transaction(async tx => { + const [task] = await tx.select().from(issues).where(and( + eq(issues.companyId, companyId), eq(issues.id, issueId), + )).for("update"); + const [current] = await tx.select().from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.id, queueId), eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.agentId, wake.agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"), + sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId}`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt'->>'actorId' = ${actorId}`, + )); + if (!task || task.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(task.status) || + !current || (!queuedCommentIdsFromWakePayload(current.payload).length && + !await readQueuedInteractionResponse(tx as unknown as Db, companyId, issueId, current.payload))) return null; + const [successor] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, wake.agentId), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + inArray(heartbeatRuns.status, ["running", "queued", "scheduled_retry"]), + )).limit(1); + if (successor) return null; + const blocker = await getExecutionBlocker(tx as unknown as Db, companyId, issueId); + const run = blocker?.runId ? await tx.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, blocker.runId), + eq(heartbeatRuns.agentId, wake.agentId), eq(heartbeatRuns.runtimeMode, "legacy"), + inArray(heartbeatRuns.status, ["failed", "timed_out", "interrupted", "cancelled"]), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + )).then(rows => rows[0]) : null; + if (!run || activeRunExecutions.has(run.id) || adapterExecutionControls.has(run.id)) return null; + // Older ephemeral leases recorded successful cleanup without a provider + // receipt. Re-verify them through the recorded teardown path; a timestamp + // alone never certifies termination. Retained/reusable resources stay put. + const historical = await tx.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, run.id), + eq(environmentLeases.leasePolicy, "ephemeral"), isNotNull(environmentLeases.releasedAt), + inArray(environmentLeases.status, ["released", "expired", "failed"]), + )).for("update"); + for (const lease of historical) { + if (!lease.provider || lease.provider === "local" || !lease.providerLeaseId || hasRemoteTerminationReceipt(lease)) continue; + // Provider resource IDs identify physical sandboxes. A lease in any + // company can still own this resource; never destroy it on behalf of + // this company. This existence-only guard exposes no foreign data. + const [otherOwner] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + ne(environmentLeases.id, lease.id), eq(environmentLeases.provider, lease.provider), + eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + or(isNull(environmentLeases.releasedAt), inArray(environmentLeases.status, ["active", "retained", "pending_cleanup"])), + )).limit(1); + if (otherOwner) continue; + await tx.update(environmentLeases).set({ status: "pending_cleanup", updatedAt: new Date() }) + .where(eq(environmentLeases.id, lease.id)); + } + return run; + }); + if (sourceRun) await sweepPendingCleanupLeases({ explicitRetry: { + companyId, runId: sourceRun.id, actorId, reason: "queued_comment_interrupt", + } }); + } + const deliveryPayload = response ? { ...payload } : withQueuedCommentIdsInWakePayload(payload, commentIds); + delete deliveryPayload.queuedCommentInterrupt; + await enqueueWakeup(wake.agentId, { + source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: deliveryPayload, contextSnapshot: response + ? { ...parseObject(payload._paperclipWakeContext), issueId, triggeredBy: "board", actorId, + responsibleUserId: actorId } + : withQueuedCommentIdsInRunContext({ + issueId, triggeredBy: "board", actorId, responsibleUserId: actorId, + }, commentIds), + requestedByActorType: "user", requestedByActorId: actorId, + ...(interrupted ? { queuedCommentInterruptId: queueId } : { queuedCommentRequestId: queueId }), + issueStateGuard: { assigneeAgentId: wake.agentId, statuses: ["todo", "in_progress", "in_review", "blocked"] }, + idempotencyKey: `queued-comment-${interrupted ? "interrupt" : "delivery"}:${queueId}`, + }, queueId); + } + + async function resumeExecutionWaitComments() { + if ((await getSchedulingSuppression()).suppressed) return; + const waits = await db.select({ wake: agentWakeupRequests }) + .from(agentWakeupRequests) + .innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), + sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, + eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) + .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) + .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), + executionBlockerPredicate(), + ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), + eq(agentWakeupRequests.requestedByActorType, "user"), + sql`${agentWakeupRequests.payload}->'executionWait' is not null`, + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is null`, + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + notInArray(issues.status, ["done", "cancelled"]))) + .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); + const seen = new Set(); + for (const { wake } of waits) { + const issueId = String(wake.payload?.issueId); + if (seen.has(issueId)) continue; + seen.add(issueId); + // Advance the cursor even for invalid evidence so one damaged task cannot + // starve later requests in this bounded scan. Preserve concurrent edits. + const [claimed] = await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(and( + eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), + lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), + )).returning({ id: agentWakeupRequests.id }); + if (!claimed) continue; + // Match normal admission's deterministic current blocker selection. An + // arbitrary historical action must not choose the retry's source run. + const blocker = await getExecutionBlocker(db, wake.companyId, issueId); + const sourceId = blocker?.runId; + if (!sourceId || !isUuidLike(sourceId)) continue; + const run = await getRun(sourceId, { includeExecutionEvidence: true }); + if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; + if (canContinueCancelledRun(run)) { + await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); + }); + continue; + } + await resumeRemoteStopComments(run, wake.id).catch(err => { + logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); + }); + } + } + + async function releaseIssueExecutionAndPromote( + run: Pick, + options: { suppressImmediateRecovery?: boolean } = {}, + ) { + try { + const source = await getRun(run.id); + const { postCommitEffects } = await wakeQueue.releaseIssueExecution({ + companyId: run.companyId, + runId: run.id, + now: new Date(), + // A durable authentication card owns recovery. This covers the review path, + // while continuation classification blocks periodic generic retries. + suppressImmediateRecovery: options.suppressImmediateRecovery || isAiAuthenticationBlocked(source), + }); + await applyWakeQueuePostCommitEffects(postCommitEffects); + const completed = await getRun(run.id); + const issueId = readNonEmptyString(completed?.contextSnapshot?.issueId) + ?? readNonEmptyString(completed?.contextSnapshot?.taskId) ?? completed?.nativeIssueId; + if (completed?.status === "succeeded" && issueId) { + await settleSlackConversation(db, run.companyId, issueId).catch((err) => { + logger.warn({ err, runId: run.id }, "Slack conversation settlement deferred to reconciliation"); + }); + } + } catch (error) { + if ( + error instanceof WakeQueueApplicationError && + error.code === "responsible_user_unresolved" + ) { + // Every other `responsible_user_unresolved` HttpError in this file + // carries `code` inside its own `details`; match that shape here too. + throw new HttpError(422, error.message, { code: error.code, ...error.details }); + } + throw error; + } + } + + async function listProjectScopedRunIds(companyId: string, projectId: string, createdBefore?: Date) { + const runIssueId = sql< + string | null + >`${heartbeatRuns.contextSnapshot} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${heartbeatRuns.contextSnapshot} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await db + .selectDistinctOn([heartbeatRuns.id], { id: heartbeatRuns.id }) + .from(heartbeatRuns) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${runIssueId}`, + ), + ) + .where( + and( + eq(heartbeatRuns.companyId, companyId), + createdBefore ? lt(heartbeatRuns.createdAt, createdBefore) : undefined, + inArray(heartbeatRuns.status, [ + ...CANCELLABLE_HEARTBEAT_RUN_STATUSES, + ]), + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function listProjectScopedWakeupIds( + companyId: string, + projectId: string, + database: Db = db, + ) { + const wakeIssueId = sql< + string | null + >`${agentWakeupRequests.payload} ->> 'issueId'`; + const effectiveProjectId = sql< + string | null + >`coalesce(${agentWakeupRequests.payload} ->> 'projectId', ${issues.projectId}::text)`; + + const rows = await database + .selectDistinctOn([agentWakeupRequests.id], { + id: agentWakeupRequests.id, + }) + .from(agentWakeupRequests) + .leftJoin( + issues, + and( + eq(issues.companyId, companyId), + sql`${issues.id}::text = ${wakeIssueId}`, + ), + ) + .where( + and( + eq(agentWakeupRequests.companyId, companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + sql`${effectiveProjectId} = ${projectId}`, + ), + ); + + return rows.map((row) => row.id); + } + + async function cancelPendingWakeupsForBudgetScope( + scope: BudgetEnforcementScope, + ) { + return withCurrentBudgetEnforcement(db, scope, async (tx) => { + const now = new Date(); + let wakeupIds: string[] = []; + + if (scope.scopeType === "company") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else if (scope.scopeType === "agent") { + wakeupIds = await tx + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + eq(agentWakeupRequests.companyId, scope.companyId), + eq(agentWakeupRequests.agentId, scope.scopeId), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + } else { + wakeupIds = await listProjectScopedWakeupIds( + scope.companyId, + scope.scopeId, + tx, + ); + } + + if (wakeupIds.length === 0) return 0; + + await tx + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: "Cancelled due to budget pause", + updatedAt: now, + }) + .where(and( + inArray(agentWakeupRequests.id, wakeupIds), + inArray(agentWakeupRequests.status, ["queued", "deferred_issue_execution"]), + isNull(agentWakeupRequests.runId), + scope.createdBefore ? lt(agentWakeupRequests.createdAt, scope.createdBefore) : undefined, + )); + + return wakeupIds.length; + }); + } + + function cancellationTerminationGraceMs( + configuredGraceSec: number, + requestedGraceMs: number | undefined, + ) { + const configuredGraceMs = Math.max(1, configuredGraceSec) * 1000; + if (requestedGraceMs === undefined || !Number.isFinite(requestedGraceMs)) { + return configuredGraceMs; + } + return Math.max(100, Math.min(30_000, Math.trunc(requestedGraceMs))); + } + + async function cancelRunInternal( + runId: string, + reason = "Cancelled by control plane", + options: CancelRunOptions = {}, + ) { + let run = await getRun(runId); + if (!run) throw notFound("Heartbeat run not found"); + if (options.cancellationRequestId) { + run = await claimCancellationRequest(db, runId, run.companyId, options.cancellationRequestId, options.cancellationRequestedByUserId ?? null); + } + // The caller claim checked retry eligibility under both durable row locks. + // This is only a cancellation candidate: dispatch rechecks the coordinator + // under lock, and the final CAS requires its own unchanged acknowledged + // retry fence. Re-reading only retryable_failure here would strand replay. + const pendingNativeRetry = + run.runtimeMode === "native" && run.status === "failed" && ( + Boolean(options.cancellationRequestId) || await db + .select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, run.id), + eq(nativeRunFinalizations.companyId, run.companyId), + eq(nativeRunFinalizations.phase, "retryable_failure"), + ), + ) + .then((rows) => rows.length > 0) + ); + if ( + !pendingNativeRetry && + !CANCELLABLE_HEARTBEAT_RUN_STATUSES.includes( + run.status as (typeof CANCELLABLE_HEARTBEAT_RUN_STATUSES)[number], + ) + ) + return run; + const agent = await getAgent(run.agentId); + const errorCode = options.errorCode ?? "cancelled"; + const cancellation = requestedRunCancellation(options.resultJson ?? {}, reason); + + const pendingProcessCancellation = processRunCancellationSettlements.get( + run.id, + ); + if (pendingProcessCancellation) { + await pendingProcessCancellation.settled; + if (pendingProcessCancellation.failed) + throw pendingProcessCancellation.error; + return getRun(run.id); + } + const running = runningProcesses.get(run.id); + const stopOwnership = + run.runtimeMode !== "native" + ? captureAdapterStopOwnership(run.id) + : undefined; + const control = stopOwnership?.control; + // Capture the existing adapter owner before waiting on the run lock. Then + // atomically fence preparation and refresh the selected runtime, so Stop + // cannot miss a native handoff that won after its first read. + // Established legacy processes must still be stopped if the database is + // unavailable. Only native or not-yet-dispatched preparation needs this + // additional durable fence before its existing cancellation path. + if (options.budgetEnforcement || (!options.cancellationRequestId && (run.runtimeMode === "native" || (!run.runtimeModeResolvedAt && !running && !control)))) { + const fence = async (tx: Db) => { + const [fenced] = await tx.update(heartbeatRuns).set({ + // Record handoff intent before native cancellation can finalize and release + // the run. Only its audited stop acknowledgement suppresses recovery. + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || + ${JSON.stringify(options.errorCode === "issue_reassigned" && options.resultJson?.reassignmentStopConfirmed === true + ? { reassignmentStopRequested: true } : {})}::jsonb || + ${JSON.stringify({ cancellation })}::jsonb || + jsonb_build_object('startupCancellation', jsonb_build_object( + 'requestedAt', ${new Date().toISOString()}::text, + 'beforeNativeSelection', ${heartbeatRuns.runtimeMode} = 'legacy' + and ${heartbeatRuns.runtimeModeResolvedAt} is null + and ${heartbeatRuns.executionStage} = 'preparing' + and coalesce(${heartbeatRuns.runnerProfileJson}->'adapterDispatch'->>'adapterType' = 'paperclip_runner', false) + ))`, + }).where(and(eq(heartbeatRuns.id, runId), inArray(heartbeatRuns.status, + pendingNativeRetry ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + ))).returning(); + return fenced ?? null; + }; + let fenced: typeof run | null; + try { + fenced = options.budgetEnforcement + ? await withCurrentBudgetEnforcement(db, options.budgetEnforcement, fence) + : await fence(db); + } catch (error) { + stopOwnership?.release(); + throw error; + } + if (!fenced) { + stopOwnership?.release(); + return getRun(runId); + } + run = fenced; + } + const resultJson = { ...(agent + ? { + ...mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: parseObject(run.resultJson), + errorCode, + errorMessage: reason, + }), + ...(options.resultJson ?? {}), + } + : options.resultJson), cancellation }; + + try { + let releaseProcessCancellation: (() => void) | undefined; + const processCancellationSettlement = + run.runtimeMode !== "native" && + !control && + running + ? { + settled: new Promise((resolve) => { + releaseProcessCancellation = resolve; + }), + failed: false, + error: undefined as unknown, + } + : undefined; + if (processCancellationSettlement) { + // No await between joining an existing owner above and registering ours. + processRunCancellationSettlements.set( + run.id, + processCancellationSettlement, + ); + } + const cancellation = await (async () => { + try { + if (control) { + await db + .update(heartbeatRuns) + .set({ + error: reason, + errorCode, + // Stop may have read the row before adapter settlement saved + // a required-copyback receipt. Merge into current database JSON + // and leave those server-owned fields to the restore recorder. + resultJson: preserveWorkspaceRestoreRecoveryMetadataSql({ + ...resultJson, + ...(!running ? { executionCancellation: { + state: "requested", requestedAt: new Date().toISOString(), + } } : {}), + }, true), + updatedAt: new Date(), + }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ); + control.controller.abort(new Error(reason)); + } + let terminationSettled = false; + try { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), + }); + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid, + processGroupId: running.processGroupId, + // Codex handles Ctrl-C by cancelling its tool sessions. SIGTERM + // can leave commands in their separate process groups alive. + signal: !control && agent?.adapterType === "codex_local" ? "SIGINT" : undefined, + graceMs: cancellationTerminationGraceMs( + running.graceSec, + options.terminationGraceMs, + ), + }); + } + terminationSettled = true; + } finally { + if ( + (!processCancellationSettlement || terminationSettled) && + runningProcesses.get(run.id) === running + ) { + runningProcesses.delete(run.id); + } + } + + if (control) { + await waitForAdapterStop(control.settled, undefined, { + runId: run.id, + adapterType: agent?.adapterType, + runtimeMode: run.runtimeMode, + abortRequested: control.controller.signal.aborted, + }, control); + const stopped = await getRun(run.id); + if (stopped && isHeartbeatRunTerminalStatus(stopped.status)) { + if ( + parseObject(stopped.resultJson?.executionCancellation).state !== + "acknowledged" + ) { + throw conflict( + "Execution ended, but provider termination could not be verified. Inspect the stopped run before continuing.", + ); + } + // The owned adapter already finalized this run and its lifecycle. + // Do not replay the process cancellation side effects below. + return { run: stopped, updated: false }; + } + } + + const finishedAt = new Date(); + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : {}; + return await setRunStatusFromLive( + run.id, + "cancelled", + pendingNativeRetry + ? [...CANCELLABLE_HEARTBEAT_RUN_STATUSES, "failed"] + : [...CANCELLABLE_HEARTBEAT_RUN_STATUSES], + { + finishedAt, + error: reason, + errorCode, + ...(resultJson || + Object.keys(persistedCancellationResult).length > 0 + ? { + resultJson: { + ...persistedCancellationResult, + ...(resultJson ?? {}), + // A scheduler placeholder has no process to acknowledge. + // Preserve its normal release policy instead of treating + // it as an operator stop of provider work. + ...(processCancellationSettlement && agent && running && ( + (Number.isInteger(running.child.pid) && (running.child.pid ?? 0) > 0) || + (Number.isInteger(running.processGroupId) && (running.processGroupId ?? 0) > 0) + ) + ? mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: { + ...resultJson, + executionCancellation: { state: "acknowledged", acknowledgedAt: finishedAt.toISOString() }, + }, + errorCode, errorMessage: reason, + }) + : {}), + // The native cancellation helper may have advanced a durable + // pending intent to its acknowledged state after `run` was + // first read. Never let that stale snapshot overwrite the + // authoritative post-dispatch acknowledgement. + ...(Object.hasOwn( + persistedCancellationResult, + "nativeCancellation", + ) + ? { + nativeCancellation: + persistedCancellationResult.nativeCancellation, + } + : {}), + }, + } + : {}), + }, + undefined, + pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, + ); + } catch (error) { + if (processCancellationSettlement) { + processCancellationSettlement.failed = true; + processCancellationSettlement.error = error; + if (activeRunExecutions.has(run.id)) { + failedProcessRunCancellations.set( + run.id, + processCancellationSettlement, + ); + } + } + throw error; + } finally { + if (processCancellationSettlement) { + if ( + processRunCancellationSettlements.get(run.id) === + processCancellationSettlement + ) { + processRunCancellationSettlements.delete(run.id); + } + // Always settle waiters, including termination and DB errors. A + // failed Stop remains an error to its caller, never a cancellation + // receipt; the signal-bearing executor may then record failure. + releaseProcessCancellation?.(); + } + } + })(); + const cancelled = cancellation.run; + + if (cancellation.updated && cancelled) { + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: cancelled.finishedAt ?? new Date(), + error: reason, + }); + await appendRunEvent(cancelled, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: options.eventMessage ?? "run cancelled", + payload: { ...options.eventPayload, cancellation: readRunCancellation(cancelled.resultJson) }, + }); + await releaseIssueExecutionAndPromote(cancelled, { + suppressImmediateRecovery: options.suppressImmediateRecovery, + }); + await finalizeAgentStatus(run.agentId, "cancelled", undefined, { + wasFirstHeartbeat: timerClaimWasFirstHeartbeat(run), + }); + await startNextQueuedRunForAgent(run.agentId); + } + return cancelled; + } finally { + stopOwnership?.release(); + } + } + + async function cancelActiveForAgentInternal( + agentId: string, + reason = "Cancelled due to agent pause", + errorCode = "cancelled", + ) { + const agent = await getAgent(agentId); + const runs = await db + .select() + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.agentId, agentId), + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + ), + ); + + for (const run of runs) { + const stopOwnership = + run.runtimeMode !== "native" + ? captureAdapterStopOwnership(run.id) + : undefined; + try { + if (stopOwnership?.control) { + await cancelRunInternal(run.id, reason, { errorCode }); + continue; + } + if (run.runtimeMode === "native") { + await db.update(heartbeatRuns).set({ resultJson: + sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ cancellation: requestedRunCancellation({}, reason) })}::jsonb`, + }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, run.status))); + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + }); + } + const persistedCancellationResult = + run.runtimeMode === "native" + ? await getRun(run.id).then((current) => + parseObject(current?.resultJson), + ) + : parseObject(run.resultJson); + await setRunStatus(run.id, "cancelled", { + finishedAt: new Date(), + error: reason, + errorCode, + resultJson: { + ...persistedCancellationResult, + ...(agent ? mergeRunStopMetadataForAgent(agent, "cancelled", { + resultJson: persistedCancellationResult, errorCode, errorMessage: reason, + }) : {}), + cancellation: readRunCancellation(persistedCancellationResult) ?? requestedRunCancellation({}, reason), + }, + }); + + await setWakeupStatus(run.wakeupRequestId, "cancelled", { + finishedAt: new Date(), + error: reason, + }); + + const running = runningProcesses.get(run.id); + if (running) { + await terminateHeartbeatRunProcess({ + pid: running.child.pid, + processGroupId: running.processGroupId, + graceMs: Math.max(1, running.graceSec) * 1000, + }); + } + runningProcesses.delete(run.id); + await releaseIssueExecutionAndPromote(run); + } finally { + stopOwnership?.release(); + } + } + + return runs.length; + } + + async function cancelPendingWakeupsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + if (uniqueAgentIds.length === 0) return 0; + + const now = new Date(); + const wakeupIds = await db + .select({ id: agentWakeupRequests.id }) + .from(agentWakeupRequests) + .where( + and( + inArray(agentWakeupRequests.agentId, uniqueAgentIds), + inArray(agentWakeupRequests.status, [ + "queued", + "deferred_issue_execution", + ]), + sql`${agentWakeupRequests.runId} is null`, + ), + ) + .then((rows) => rows.map((row) => row.id)); + + if (wakeupIds.length === 0) return 0; + + await db + .update(agentWakeupRequests) + .set({ + status: "cancelled", + finishedAt: now, + error: reason, + updatedAt: now, + }) + .where(inArray(agentWakeupRequests.id, wakeupIds)); + + return wakeupIds.length; + } + + async function cancelInvocationsForAgentsInternal( + agentIds: string[], + reason: string, + ) { + const uniqueAgentIds = [...new Set(agentIds)].filter( + (agentId) => agentId.length > 0, + ); + let runsCancelled = 0; + for (const agentId of uniqueAgentIds) { + runsCancelled += await cancelActiveForAgentInternal(agentId, reason); + } + const wakeupsCancelled = await cancelPendingWakeupsForAgentsInternal( + uniqueAgentIds, + reason, + ); + return { + agentIds: uniqueAgentIds, + runsCancelled, + wakeupsCancelled, + }; + } + + async function cancelBudgetScopeWork(scope: BudgetEnforcementScope) { + const runIds = scope.scopeType === "project" + ? await listProjectScopedRunIds(scope.companyId, scope.scopeId, scope.createdBefore) + : await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, scope.companyId), + scope.scopeType === "agent" ? eq(heartbeatRuns.agentId, scope.scopeId) : undefined, + scope.createdBefore ? lt(heartbeatRuns.createdAt, scope.createdBefore) : undefined, + inArray(heartbeatRuns.status, [...CANCELLABLE_HEARTBEAT_RUN_STATUSES]), + )).then((rows) => rows.map((row) => row.id)); + for (const runId of runIds) await cancelRunInternal(runId, "Cancelled due to budget pause", { budgetEnforcement: scope }); + await cancelPendingWakeupsForBudgetScope(scope); + } + + return { + cancelBudgetScopeWork, + releaseIssueExecutionAndPromote, + releaseEnvironmentLeasesForRun, + acknowledgeRemoteStop, + resumeRemoteStopComments, + cancelRunInternal, + resumeExecutionWaitComments, + resumeQueuedCommentInterrupt, + resumeSavedLegacyComments, + cancelActiveForAgentInternal, + cancelInvocationsForAgentsInternal, + }; +} diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index d28d6a4ec07..58555ccb48d 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,4 +1,5 @@ import { normalizeEscapedLineBreaks } from "@paperclipai/shared/validators/text"; +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { activeIssueInteractionCondition, historicalQuestionCondition } from "./issue-question-context.js"; import { currentContinuationOrigins, @@ -2829,6 +2830,7 @@ export function issueThreadInteractionService( .returning(); if (!row) throw interactionAlreadyResolvedError(); if (status === "accepted" || status === "rejected") { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); await tx.insert(connectionIntentDeliveries).values({ interactionId, companyId: issue.companyId }).onConflictDoNothing(); } return row; @@ -5026,6 +5028,7 @@ export function issueThreadInteractionService( // This answer updates conversation history only. It must not resume // the completed source run or enqueue new work for the closed task. if (!historicalAnswer) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.question); await tx .insert(issueQuestionResponseDeliveries) .values(questionResponseDeliveryValues(answered)); diff --git a/server/src/services/question-response-delivery.ts b/server/src/services/question-response-delivery.ts index 5db1e051508..dad35c77d7e 100644 --- a/server/src/services/question-response-delivery.ts +++ b/server/src/services/question-response-delivery.ts @@ -1340,5 +1340,9 @@ export function questionResponseDeliveryService( return counts; } - return { deliver, sweepPending }; + async function hasPending() { + return (await db.select({ id: issueQuestionResponseDeliveries.id }).from(issueQuestionResponseDeliveries) + .where(inArray(issueQuestionResponseDeliveries.status, ["pending", "delivering"])).limit(1)).length > 0; + } + return { deliver, sweepPending, hasPending }; } diff --git a/server/src/services/tool-action-delivery.ts b/server/src/services/tool-action-delivery.ts index 63f217de9a0..0718ed2d449 100644 --- a/server/src/services/tool-action-delivery.ts +++ b/server/src/services/tool-action-delivery.ts @@ -345,6 +345,12 @@ export function toolActionDeliveryService( } return { deliver, + async hasPending() { + // Include receipts awaiting review/execution: their later terminal write + // must remain recoverable even if its optional fast-path callback fails. + return (await db.select({ id: toolActionDeliveries.actionRequestId }).from(toolActionDeliveries) + .where(isNull(toolActionDeliveries.deliveredAt)).limit(1)).length > 0; + }, async deliverForRun(input: { companyId: string; runId: string }) { // A review can settle before its source run yields. Retry that run's // receipts as soon as execution cleanup finishes; the periodic sweep diff --git a/server/src/services/tool-action-review.ts b/server/src/services/tool-action-review.ts index b429b35ce69..3d0effa9f07 100644 --- a/server/src/services/tool-action-review.ts +++ b/server/src/services/tool-action-review.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, eq } from "drizzle-orm"; import { issues, @@ -206,6 +207,7 @@ export async function commitToolActionReview( : { version: 1, outcome: "rejected", reason: input.reason }, }) .where(eq(issueThreadInteractions.id, interaction.id)); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); await tx .insert(toolActionDeliveries) .values({ diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index 86d79cd2f9c..4ef4f1092e8 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -1,3 +1,4 @@ +import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { composeConnectionInstructions } from "./connection-instructions.js"; import { isInsufficientConnectionScope, INSUFFICIENT_CONNECTION_SCOPE_MESSAGE } from "./connection-permission-errors.js"; import { boundedMcpToolName } from "./mcp-tool-names.js"; @@ -2615,15 +2616,18 @@ export function createToolGatewayService( ); } - await db - .insert(toolActionDeliveries) - .values({ - companyId: input.session.companyId, - actionRequestId: actionRequest.id, - issueId: input.session.issueId, - interactionId: interaction.id, - }) - .onConflictDoNothing(); + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.toolAction); + await tx + .insert(toolActionDeliveries) + .values({ + companyId: input.session.companyId, + actionRequestId: actionRequest.id, + issueId: input.session.issueId!, + interactionId: interaction.id, + }) + .onConflictDoNothing(); + }); await writeToolCallEvent({ invocationId: input.invocation.id, diff --git a/server/src/services/work-scheduler.ts b/server/src/services/work-scheduler.ts new file mode 100644 index 00000000000..1d2cdadfe36 --- /dev/null +++ b/server/src/services/work-scheduler.ts @@ -0,0 +1,45 @@ +/** In-process deadlines. Owners restore durable tasks at startup. */ +export function createWorkScheduler() { + const tasks = new Map void }>(); + let timer: ReturnType | null = null; + let stopped = false; + const nextWakeAt = () => tasks.size ? Math.min(...Array.from(tasks.values(), task => task.at)) : null; + + function arm() { + if (timer) clearTimeout(timer); + timer = null; + const next = nextWakeAt(); + if (stopped || next === null) return; + timer = setTimeout(() => { + timer = null; + const now = Date.now(); + const due = Array.from(tasks.entries()).filter(([, task]) => task.at <= now); + for (const [key, task] of due) { + // Another due callback can cancel or replace this task. + if (tasks.get(key) !== task) continue; + tasks.delete(key); + try { task.run(); } + catch (error) { process.emitWarning(`Scheduled work failed: ${String(error)}`); } + } + arm(); + }, Math.min(2_147_483_647, Math.max(0, next - Date.now()))); + timer.unref?.(); + } + + return { + nextWakeAt, + schedule(key: string, at: number, run: () => void) { + if (stopped) return; + if (!Number.isFinite(at)) throw new Error("Work deadline must be finite"); + tasks.set(key, { at, run }); + arm(); + }, + cancel(key: string) { tasks.delete(key); arm(); }, + stop() { + stopped = true; + tasks.clear(); + if (timer) clearTimeout(timer); + timer = null; + }, + }; +}