diff --git a/.github/scripts/tests/docker-canary-promotion.test.mjs b/.github/scripts/tests/docker-canary-promotion.test.mjs index a9b0727465f..fe80e25168b 100644 --- a/.github/scripts/tests/docker-canary-promotion.test.mjs +++ b/.github/scripts/tests/docker-canary-promotion.test.mjs @@ -11,7 +11,7 @@ const script = job.split(" run: |\n")[1].split("\n").map(line => line.rep const sha = "a".repeat(40); test("canary promotion waits for the standard manifest and keeps its serialized channel", () => { - assert.match(job, /needs: \[merge-and-push\]/); + assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/); assert.match(job, /group: docker-canary-channel-promotion/); assert.match(job, /cancel-in-progress: false/); }); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ed52dfa700d..ec7c71000e5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,35 @@ concurrency: cancel-in-progress: false jobs: + reuse_canary_image: + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + attestations: read + outputs: + reused: ${{ steps.reuse.outputs.reused }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Login to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Reuse the verified master image when present + id: reuse + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/reuse-canary-image.mjs + # Multi-arch by native runner, not QEMU. # # This was one job building linux/amd64,linux/arm64 together on an x86 @@ -56,6 +85,8 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: + needs: [reuse_canary_image] + if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish # nothing, but it must not also hide the other's logs behind a @@ -277,6 +308,7 @@ jobs: # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: + if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} needs: build-and-push runs-on: ubuntu-latest timeout-minutes: 20 @@ -407,11 +439,8 @@ jobs: # promotion is gated on its own completed pushes). Every interleaving # converges the Docker channel onto the npm channel. promote_canary_channel: - if: startsWith(github.ref, 'refs/tags/canary/v') - # merge-and-push, not build-and-push: the per-arch legs push untagged - # digests, and the production `sha-*` tags this promotion retags only - # exist once the manifest merge has named them. - needs: [merge-and-push] + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }} + needs: [reuse_canary_image, merge-and-push] runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b1b73c1f085..dae203d6216 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -202,7 +202,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/doc/RELEASING.md b/doc/RELEASING.md index dbf40956a50..70e28fe798f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -86,7 +86,7 @@ internal traceability tag. moves them to the versioned name - canaries, nightlies, and betas never create GitHub Releases - canaries, nightlies, and betas never require changelog generation -- Docker `:latest` moves only on stable releases; master builds publish +- Docker `:latest` moves only on stable releases; canary dispatches promote `:canary`, nightly builds `:nightly`, and beta builds `:beta` ## TL;DR @@ -310,12 +310,12 @@ The workflow: ## Docker Image Tags -[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted -image and the `-cloud` variant with the same lane mapping: +[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: | Build ref | Tags | | --- | --- | -| `master` push | `:canary`, `:sha-` | +| `master` push | `:sha-`, `:sha-` | +| `canary/v*` dispatch | `:sha-`, then `:canary` for the current npm canary | | `nightly/v*` tag | `:nightly`, `:sha-` | | `beta/v*` tag | `:beta`, `:sha-` | | `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-` | @@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs therefore dispatch `docker.yml` explicitly at the new tag ref; the tag mapping keys off `github.ref` either way. +Canary dispatches first look up the full-SHA master image. Reuse requires a +valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching +registry digest, and signed provenance from this repository's `docker.yml` +on `master` at the exact source commit. The image is retagged by immutable +digest. This preserves the original build inputs and skips both native builds +and the manifest merge. A missing full-SHA image runs the normal build instead; +registry errors or invalid provenance fail the run. Rerun after the master +image's attestation finishes if the image became visible before its proof. + +Both paths retain the serialized promotion that resolves the current npm +`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and +master builds retain their existing build paths. Downstream image builds can +continue to consume the signed full-SHA master image independently of npm +canary publication. + The Docker build matrix and the release preview image builder configure BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as diff --git a/doc/evals.md b/doc/evals.md index 78c9c3ce467..bc0b7c9f729 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -76,6 +76,12 @@ one bound continuation, and an exact saved confirmation. Its four development cells use the recorded Codex and Claude models. Registration and calibration do not establish Slack, staging authentication, warm lifecycle, or live qualification. +The explicit `followup-while-running` [Product journey](../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +checks a saved message during independently observed native work, one attributed +reply with the late requirement, and persistence after reload. Its attached and +managed targets share the same story and permit one steered run or one normal +successor. Registration and calibration are separate from a live staging pass. + [Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit deployment targets and starting-state recipes. The shared task-properties journey uses no model; Cloud onboarding supplies reusable verified state. Cloud diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index e535edcd569..7b8ef115430 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers. Caller timeouts retain their slots until the actual worker reply or exit; timeouts cannot accumulate additional provider reads in later sweeps. +If a failed ephemeral lease has no usable local inspection path, a managed +Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks +the stack's historical credential organization and reads the exact allocation +with its manager credential. Only confirmed absence can produce a signed receipt +bound to the stack, company, environment, lease, provider handle and nullable run. +The separate cleanup type and audience cannot authorize run recovery. Older +Cloud servers, unsupported providers, invalid signatures, changed organizations +and failed reads leave the lease unresolved. Existing bounded cleanup batches +and retry deadlines own these reads; no new idle polling is added. An explicit +local provider observation is retained rather than replaced by this fallback. + Host-owned verification metadata binds the company, environment, resource, provider handle, optional run, and exact row version. Concurrent resume, replacement, cleanup, or configuration changes invalidate verification. diff --git a/doc/observability.md b/doc/observability.md index e2bb4065caf..996d3a807f5 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -328,6 +328,14 @@ once known, `paperclip.native_reconciliation.candidates` (a count). A failed pass marks the pass span and the phase that was running with an error status. Faster passes record nothing. +Heartbeat configuration preparation overlaps independent reads. Skill preparation +runs alongside connector assignments; standing connection instructions wait for +assignment catalog repairs, but can overlap the skill branch. These spans can +finish out of order. Do not add their durations as serial elapsed time. The +historical `skills.prepare` span ends when the skill branch settles, rather than +when its concurrent branches finish. Each preparation attempt settles its started +branches before it returns an error or releases acquired credentials. + ### Detailed execution phases The `paperclip.execution` tracer adds these content-free spans beneath the current diff --git a/doc/plans/2026-10-30-attention-retention-sleep.md b/doc/plans/2026-10-30-attention-retention-sleep.md new file mode 100644 index 00000000000..92241aefc4f --- /dev/null +++ b/doc/plans/2026-10-30-attention-retention-sleep.md @@ -0,0 +1,50 @@ +# Archive attention sources without repeated feed scans + +Attention retention now reconstructs source deadlines at startup and after committed source changes. It no longer rebuilds every active company's full feed on each heartbeat tick. The archive age remains 90 days. Keep and manual archive state remain authoritative, and automatic archive still commits its notification outbox atomically. + +This document records source ownership, producer coverage, and the evidence needed to review the replacement. It makes no claim about host suspension or external wake alarms. + +## Existing source rows own eligibility and deadlines + +`attention-retention.ts` selects at most 100 unfinished source candidates per reconstruction page. It excludes inactive companies and kept or archived retention states. The scoped coordinator retains eligible deadlines, retries concrete failures, and cancels entries that leave attention. It does not schedule another scan after settlement unless a producer changes a relevant source. + +| Source | Eligible durable state | Activity owner | +| --- | --- | --- | +| Decision | Open decision with a visible execution issue | Decision updated time | +| Interaction | Pending board-visible interaction; newest ordinary confirmation or independent quality gate | Interaction updated time | +| Recovery action | Active or escalated action owned by a user or board, with a visible source issue | Recovery updated time | +| Blocker attention | Board unblock descriptor or non-live terminal blocker selected by the issue owner | Blocked transition or terminal issue updated time | +| Review | Human participant, user assignee, pending approval, or stalled review selected by the issue owner | Issue updated time | +| Failed run | Failed or timed-out run with an exhaustion event, eligible agent and issue, and no newer matching run | Finished time, falling back to updated or created time | +| Budget alert | Open hard incident, or soft incident meeting the existing 85 percent rounded threshold | Incident updated time | +| Agent error | Agent in error status | Agent updated time | + +The feed and worker share confirmation collapse, newer-run probes, issue visibility, review eligibility, and terminal-blocker predicates. Issue graph and agent invokability logic remain with their existing owners. Blocker entries retain both the root used to reconstruct the graph and the terminal subject that owns retention state. PostgreSQL deadlines round submillisecond timestamps upward. The worker locks the source and terminal subject before archival. + +`productivity_review` generates no feed items and owes no retention work. Queue retention-day overrides control shelf presentation, which is computed on reads. They do not change the fixed automatic archive age. Approval retention, join-request retention, and archive notification delivery keep their existing workers. + +## Existing mutation boundaries reactivate work + +Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. Positive live-work evidence from the bounded returned rows is retained independently of node/edge admission, including running status, active runs and pending confirmations. Filling the graph with explicit blockers cannot hide a subsequently returned active child. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included. + +Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit. + +The writer contract follows the existing delivery workers: serving processes use the transaction-aware service boundaries, and each serving process registers its workers. Independent raw SQL writers must supply equivalent ingress or restart reconstruction. Restores reconstruct work at startup. The change adds no permanent listener, independent timer, table, or Cloud wake mechanism. Existing sleep-safety gates remain in force. + +## Real PostgreSQL evidence proves quiet and reactivation + +The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. Additional ordinary producer controls cover child creation with a parent-blocking relation, budget incident creation and resolution, and repeated company-scoped execution-control writes. + +It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary. + +The old aggregate path issued 12 statements per tick in each of three configured-idle samples. The replacement issued zero statements during 300 seconds after settlement. One transaction-status read after an injected rollback is classified as uncertainty reconciliation; the following 300-second drain window must remain quiet. + +The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, validates its dedicated `_test` database name, applies pending migrations, and prewarms a six-connection pool before advancing fake time. Providers are not involved in retention selection; the existing archive outbox delivery tests verify the separate controlled notification boundary. Fake time proves scheduling and SQL quiet, not database or host suspension. + +## Review and operational limits remain explicit + +Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate. + +Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change. + +Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward. diff --git a/doc/plans/2026-10-30-backup-expiration-sleep.md b/doc/plans/2026-10-30-backup-expiration-sleep.md new file mode 100644 index 00000000000..307b4f67f47 --- /dev/null +++ b/doc/plans/2026-10-30-backup-expiration-sleep.md @@ -0,0 +1,63 @@ +# Backup expiration without idle polling + +The backup expiration duty retains its existing minimum age and safety policy. +It uses the shared scoped deadline coordinator. The archived runner evidence and +checkpoint expiration duties keep their separate registrations. + +The durable source is each controller-owned `failover-backups/current` directory, +its control-plane session identity, and the associated run, issue, finalization, +and environment lease rows. The next eligible instant is one millisecond after +the later of the backup mtime and every related issue's close time, plus the +configured retention duration. An open issue, live run or lease, unsettled +finalization, or cleanup activation has no time-derived expiry; its producer +must invalidate the source when it changes. A busy process or failed I/O leaves +real unresolved cleanup and uses the shared retry backoff. Dry-run reports an +eligible source once and waits for source invalidation or restart. + +Startup pages existing disk scopes and indexes each backup's actual session, +run, coalesced native-task/task, agent and lease references, including protected +backups. Ordinary task, run, lease and process-release notifications retain +these identities and inspect only affected roots. New run or lease references +resolve through the owner's database relationships. Local publication retains +its exact digest. Explicit company or instance configuration changes can still +reconstruct their whole affected scope. Roots without backup or cleanup tombstones +perform no backup SQL. After discovery and all obligations settle, there is no +recurring SQL or filesystem scan. Startup reconstructs future work; inspecting a +source never changes its business timestamps. Runtime task close/reopen and +retention timestamp changes, native environment and retained session writes, +local backup publication, and process release invalidate the company or digest. +Signals from database producers register before writes inside their owning +transaction, including caller-owned transactions and nested savepoints. + +The controller remains the only supported writer of its local backup files. +An independent process writing those files must use the same producer ingress; +manual disk edits require reconstruction on restart. Database writer ingress +continues to use the existing transaction-aware work signals. This change adds +no permanent database listener and does not add a polling table or migration. + +Read-only reconstruction does not lock issue rows. Transient identity I/O errors +retain a retry obligation; missing or malformed state remains unrecognized. +Byte accounting occurs before the retirement transaction. The current backup +age is rechecked inside the transaction before the rename. + +The existing issue share locks now remain held through the retirement rename. +This fences task reopen while the backup becomes unavailable. A failed database +transaction restores `.retiring-*`; a process interruption is recovered from +that existing file journal at startup. Journal reconciliation runs before +identity or owner lookup, so an agent purge cannot strand expired physical +cleanup or prevent restoration of a retiring backup. After database settlement, `.expired-*` +records outstanding physical removal so a partial delete retries idempotently. +Ownership, session identity, live run, lease and local execution safety checks +remain in the backup owner. No host-sleep blocker or expiry-on-use rule changes. +An in-process deadline does not register a Cloud wake alarm. + +Acceptance uses isolated real PostgreSQL for service writes, transaction signals, +safety checks, resume pointer deletion, and restart reconstruction. A temporary +filesystem is the controlled storage boundary. These tests do not claim cloud +object storage behavior, deployed sleep, or fleet savings. + +Review validation also covers expired-only directories whose listing fails with +EIO or EACCES. Only ENOENT means absent; other listing failures retain retry debt. +Both controls failed before the fix; all 104 expiration tests pass afterward. +Executor release tests assert both the preserved broad cleanup scope and the +precise source identity used for backup invalidation. diff --git a/doc/plans/2026-10-30-deferred-execution-sleep.md b/doc/plans/2026-10-30-deferred-execution-sleep.md new file mode 100644 index 00000000000..56e3611d939 --- /dev/null +++ b/doc/plans/2026-10-30-deferred-execution-sleep.md @@ -0,0 +1,23 @@ +# Deferred execution receipt ownership + +This slice removes the periodic `resumeQueuedRuns` sweep after queue dispatch migrated separately. It also delegates native status wake intents out of the periodic orphan reaper. Existing queue admission, authorization, task recovery, cancellation, resource and controller checks remain the mutation authority. + +| Duty | Durable pending truth | Committing producers | Deadline and exit | +| --- | --- | --- | --- | +| Task Retry | deferred task_retry wake with certified actor/generation | requestTaskRetry through execution control; execution, issue, agent, company and budget eligibility | immediate; current admission links canonical intent to one dispatch/run, supersession or terminal policy retires it; blocked input awaits its actual gate mutation | +| Saved comments and interaction responses | deferred wake carrying saved comment IDs or interaction mutation | wake admission insert; merge only genuinely new comment IDs or interaction identity; issue/lock/assignment/dependency and execution eligibility | immediate after terminal source; original release owner revalidates comment/card authority, process/cleanup evidence and issue claim | +| Execution wait | deferred wake with executionWait | original admission/restored source receipt; actual recovery action, execution and eligibility changes | original updatedAt + 30 seconds, then current owner revalidation; unchanged blocked receipt carries no recurring scan | +| Interrupt retry | deferred wake with queuedCommentInterrupt | authorized route persists interrupt marker; cancellation lifecycle and cleanup/eligibility transitions | original updatedAt + 30 seconds; cancelled legacy run naming exact queue bypasses initial grace as before; owner retains explicit interrupt and stop evidence | +| Cancelled legacy queue | cancelled legacy run result names exact still-deferred queue | run lifecycle terminal/control commits; original interrupt intent | immediate restart reconstruction; original release/admission owner settles or leaves saved input behind its actual blocker | +| Task chat receipt settlement | deferred task_retry with taskRetry.chatActionId | failed chat retry action settlement | immediate settlement-only pass folds linked action result; it cannot restage its own processed-but-deferred chat action. Startup/actual eligibility retains normal re-admission policy | +| Native status wake intent | system/native-status-committer wake queued/claimed without run | status committer transaction; actual execution/issue/agent/company/budget eligibility | immediate queued intent; claimedAt + 60 seconds for crash recovery; error updatedAt + 30 seconds. Existing deterministic dispatcher identity/claims prevent duplicate delivery; original intent becomes linked/coalesced/skipped while a separate blocked dispatch receipt owns saved admission | + +Three scoped owners reconstruct only their actual durable receipt predicates, with keyset pages of 100. Normal signals contain normalized company/request identity. Instance scheduler settings alone may request instance-wide reconstruction. Repeated source changes coalesce reconstruction for 30 seconds; already-known stored deadlines preempt that cooldown. Pending checks read memory only. Shutdown uses the existing coordinator and physical execution tracking; task drain reconciles producer transactions and releases the settled intent hold while retaining work for resume. Warm standby and disabled registration do not execute domain work. + +The generic run-dispatch notifier and broad issue review-state notifier deliberately do not fan out deferred eligibility. Queue bookkeeping may write them while processing its own saved input. Separate eligibility commits and settlement-only chat notifications avoid a self-restaging loop. + +C2 remains separate and mandatory: active output silence deadlines, remaining orphan/process/controller/remote authority and resource debts, legacy shutdown settlements, stranded agent status, and final aggregate chain removal. Retained workspace and migrated native finalization workers remain their existing owners. + +Review controls cover queue edits that move an already loaded grace deadline. Each due check re-reads current stored grace or claim time before admission. Subtree hold release, subtree restore, and member archival publish eligibility inside their own transactions. Member archival preserves the cancelled Retry tombstone created by an earlier user reassignment. + +A resume that first records a new execution-wait reason arms its existing 30-second grace once. Writing the same closed-gate reason again does not publish another hint. A clock obligation, an eligibility change, and an infrastructure failure therefore retain separate exits: stored deadline, owning committed producer, and bounded failure backoff. Refreshing a diagnostic timestamp alone does not create permanent polling behind a deliberate pause. diff --git a/doc/plans/2026-10-30-external-objects-sleep.md b/doc/plans/2026-10-30-external-objects-sleep.md new file mode 100644 index 00000000000..e5401596a0e --- /dev/null +++ b/doc/plans/2026-10-30-external-objects-sleep.md @@ -0,0 +1,124 @@ +# External object refresh deadlines + +External-object refresh uses the delivery coordinator. The server registers the +service in both heartbeat-scheduler configurations. It no longer calls the +company-wide aggregate sweep on each heartbeat tick. + +`external-objects.ts` owns automatic eligibility, resolver selection, refresh +claims, provider results and the next deadline. `index.ts` supplies registration +and admission gates. The existing scoped deadline helper owns wake coalescing, +page reconciliation, fair company selection, retry backoff and shutdown. + +## Durable obligations and quiet states + +An automatic refresh needs an active company, a retained mention in that company, +a non-null `next_refresh_at` and an available resolver. Injected resolvers, +including custom `url/link` resolvers, remain supported. A plugin resolver needs a +ready plugin with `external.objects.read`, a matching provider/object declaration, +and no company setting that disables it. Built-in resolvers remain available +independently of optional plugin support. + +A plugin result carries its selected plugin ID inside the host owner. Persistence +rechecks that exact plugin's readiness, read capability, object declaration and +same-company enabled setting in the result update. Another available resolver +cannot authorize an obsolete plugin response. A rejected response releases only +its own claim; the due row remains for a fresh allowed resolver. Plugin RPC +payloads and public result shapes do not contain this host selection context. + + +Terminal objects are passive. The existing exception remains: GitHub +`not_found` snapshots without `data.probeVerified=true` need a repository-visibility +probe before their terminal verdict is definitive. A null deadline retains its +existing never-refresh meaning. Registration does not change stored timestamps. + +The worker reconstructs eligible deadlines in pages of at most 250 rows. A +company notification reads that company only. Ordinary ingestion uses object IDs +or stable provider/object/external identity hints, including concurrent upserts, +so one mention write does not reconstruct all company history. A due callback reads only its +object. Pages select scheduling columns and retain compact deadlines in memory. +Dispatch uses fair batches of at most 50 objects. The deadline is the later of +`next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic +refresh tokens continue to exclude another claimant. + +Unsupported generic links, orphan objects, disabled plugins, empty configuration +and retained terminal history do not schedule fallback discovery. Manual refresh +retains its existing `no_resolver` result. Actual provider failures retain their +saved retry deadline. Unexpected failures retain scoped coordinator backoff; +restart reconstructs the saved object and any unexpired claim. + +The supported-resolver and retained-reference approach follows the related draft +[PR #17298](https://github.com/paperclipai/paperclip/pull/17298) by +[@kaluli123123](https://github.com/kaluli123123). This change adapts that eligibility +pattern to scoped scheduling, adds dynamic plugin/company permission predicates, +and fences automatic result persistence. It does not modify that draft PR. + +## Producer and transaction audit + +| Mutation | Owner and activation | +| --- | --- | +| Issue title/description, comment and document references | Existing sync methods register object intent before mention replacement and stable identity intent before object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. | +| Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. | +| Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. | +| Issue deletion | Issue service registers its company before cascading mention deletion. | +| Company status/archive/deletion, including budget pause/resume | Company and budget services register company reconstruction before status or deletion writes. | +| Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. | +| Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. | +| Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. | +| Manual and automatic resolver result | Result persistence registers object intent in the same transaction. Automatic persistence rechecks reference and resolver eligibility. Lost support releases the worker's own claim without publishing the stale result. | +| Normalized GitHub ingress | The event service registers company intent with the snapshot write. It clears the old refresh token, so an older provider response cannot overwrite that event. | +| Import and restore | No independent object-table importer exists in the source baseline. Supported reference ingestion uses the ordinary sync boundary. Startup reconstructs restored durable objects. | + +Notifications are awaited before writes. Caller-owned and nested transactions +retain the outer commit boundary through the existing database-work protocol. +Rollback does not publish the write. Notifications received during a scan or an +active resolver remain dirty until a successful reconciliation. + +The supported live writer model is one shared `createDb` signal owner, including +its transactions and child pools. These process-local signals do not cover +independent SQL clients or other replicas. Such writers must invoke the supported +service boundary or arrange explicit reconstruction/restart. Existing normalized +GitHub ingress uses the supported boundary. No permanent database listener or +periodic global safety scan is added. + +## Acceptance evidence + +The focused suite uses real PostgreSQL with a controlled simulated provider. +Ordinary sync and registry/settings writes create work; the fixture does not +pre-create the tested resolved outcome. Assertions independently read saved +objects. Tests measure owner reads and provider calls across former tick windows. + +Coverage includes empty and terminal quiet, unsupported links, orphan removal and +reattachment, feature/company eligibility, plugin readiness and permission loss +during a held provider call, exact and earlier deadlines, durable provider retry, +rollback and outer commit after nested savepoint release, startup and paginated +reconstruction, inactive companies, never-refresh rows, first work during drain, +warm standby, failed refresh after restart, lease renewal, duplicate exclusion, +and shutdown while a provider call remains active. The GitHub ingress suite also +checks a real normalized event superseding a held resolver response. + +The conservative external-object host-sleep blocker remains. An in-process +deadline cannot wake stopped compute. This change proves worker quiet and +reactivation in the tested process; it does not establish Cloud alarm coverage, +host/database suspension or deployed savings. + + +## Host cancellation and registered ownership + +The resolver contract accepts an optional host `AbortSignal`. The worker passes +its coordinator signal through refresh to the resolver. This reuses the narrow +provider-owned capability introduced with merge-confirmation work: GitHub combines +host cancellation with a ten-second request budget for real fetch, response-body +reads and visibility probes. Plugin RPC payloads remain unchanged. The host +observes cancellation while retaining handlers for any eventual plugin/custom +result; it releases only its own claim and cannot publish that late result. +Startup does not await provider work. Shutdown closes the active GitHub HTTP +request, and its saved due row can resume immediately after restart. + +The old `refreshDueObjects*` batch helpers are removed. Existing selection, +lease, concurrent manual refresh, terminal-history and not-found recovery tests +now enter through the registered scheduler owner. The removed baseline control +recorded six SQL reads over three empty aggregate ticks; the current quiet-work +acceptance measures zero recurring SQL after settlement. The scheduled check retains its +post-refresh scoped read to revalidate current company, reference and resolver +eligibility and recover an authoritative lease after a competing writer; all such +reads target one object rather than rebuilding a company. diff --git a/doc/plans/2026-10-30-issue-recovery-sleep.md b/doc/plans/2026-10-30-issue-recovery-sleep.md new file mode 100644 index 00000000000..0c6cdf6b4d3 --- /dev/null +++ b/doc/plans/2026-10-30-issue-recovery-sleep.md @@ -0,0 +1,28 @@ +# Issue recovery without periodic scans + +The issue recovery owners reconstruct their current obligations once at startup. Ordinary comment writes invalidate only the affected issue for assignment and lock recovery. Lifecycle, graph, run, configuration and eligibility changes invalidate their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database. + +| Owner | Durable truth and producer boundary | Deadline or event | Exit | +| --- | --- | --- | --- | +| Stranded assignment | Visible assigned todo/in-progress/review issues; issue create/update/checkout/release; run lifecycle and execution control; agent/company/budget/configuration; interaction, approval and subtree hold changes | A committed eligibility change. A transient continuation uses the latest finish plus the existing exponential delay. | An execution path, durable wait, escalation, or current ineligibility. | +| Orphan blocker assignment | Agent-created unassigned todo/blocked blockers with a nonterminal dependent; issue, relation, agent and company changes | Once per changed company, independent of assignment candidates. | Assigned, no longer blocking, or creator cannot execute until its next eligibility change. | +| Active recovery actions | Active/escalated action records, including actions on completed sources; action, issue, run and wait changes | Once per changed company, independent of assignment candidates. | Resolved, restored path, explicit hold or a separately owned retry. | +| Resolved dependency | Blocked assigned issues and company-scoped blocker relations; blocker status or relation changes through issue service and import; run/control and pause gate changes | The committing change. A failed enqueue retains that issue with error backoff. | Existing idempotent wake, live path, unresolved dependency or deliberate self-block. | +| Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. | +| Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. | + +Local comment hints do not broaden into an assignment check for every open task. A real PostgreSQL regression starts with1,000 open assigned issues, writes one comment through `issueService.addComment`, and observes exactly one assignment check; a subsequent company eligibility notification still checks all1,000. Restoring the previous comment hint makes that same regression check all1,000 after the comment. The test substitutes the expensive assignment decision only to count its invocations; it uses the real transaction, notification, coordinator and paged SQL. Separate producer acceptance cases exercise real recovery decisions and process execution. + +Parent/descendant disposition repair and shared execution references make status and run changes broader than a comment. Their existing notification boundaries deliberately retain company scope. Orphan blockers, active recovery actions, resolved dependencies and watchdogs also keep their company/subtree duties; an issue comment does not drop those independent obligations. The scope serializer selects known primitive fields and never serializes an entire insert or update object. + +Reconstruction uses keyset pages of at most100rows. Repeated company hints coalesce reconstruction for30seconds. Already-known entry deadlines preempt that spacing without an early probe on each comment. Scheduling suppression is read once per pass. A failed read keeps scoped backoff debt. The two subsidiary company repairs do not run inside the per-issue loop. Legacy direct aggregate API calls retain both repairs. + +The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry. + +Producer coverage uses the existing run-dispatch, execution-control, issue-review-state, native status reassessment and external conversation settlement boundaries. Direct gaps have explicit signals inside their owning transaction: document writes/import, attachment removal, blocker import, recovery-action adoption/mutation, conversation pause release, Telegram failure comment replacement, native artifact title/metadata updates and company deletion. Metadata-only comment classification that does not change timestamp, visibility or execution eligibility needs no additional watchdog activation. Approval links and decisions use their existing issue-review/continuation effects. Work-product create/update/delete/import uses its existing native status boundary. + +The new owners replace only these four startup and periodic calls. Deferred comments, silent active runs and residual orphan/resource recovery retain their existing backstop until the next slice. Every previously migrated native-owner skip remains unchanged. + +A controlled process test prints an execution proof but cannot apply a task disposition. The test operator completes that task through issue service. If the existing continuation policy already saved a disposition-repair retry, the test preserves it until its stored deadline. Only after all provider work has joined does it advance the clock; dispatch then revalidates the completed task and cancels the retry. The final quiet proof requires no registered deadlines, no tracked in-flight work and no owner SQL over twenty former polling minutes. It does not claim a saved future retry is already drained. + +Active action repair has no separate grace or lease countdown. Its delayed disposition repair first writes a scheduled-retry run, then mirrors the date into action wakePolicy.retryAt and timeoutAt. The dispatch owner owns that saved run deadline. Other skipped gates wait for their existing execution, wait, pause, budget, or finalizer producer; the company action worker does not add a polling fallback. diff --git a/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md new file mode 100644 index 00000000000..1c691c19509 --- /dev/null +++ b/doc/plans/2026-10-30-merged-pr-confirmations-sleep.md @@ -0,0 +1,118 @@ +# Merged pull request confirmation scheduling + +The merged-PR confirmation duty previously ran from every heartbeat maintenance +callback, including when there were no merge requests. It loaded all pending +confirmation cards and then applied the merge-only parser in memory. + +The interaction service still owns merge-only eligibility, provider access, +automatic acceptance, current resolver authority, audit, and continuation. +The worker registers the existing scoped-deadline helper with a dedicated delivery +topic. Shared scheduling receives interaction/company identities and deadlines; +it receives no GitHub payloads. No database migration is required. + +## Durable truth and producers + +Pending `request_confirmation` interaction rows remain the durable work source. +The candidate query pages pending cards in bounded groups of 100 and excludes +terminal and hidden issues and linked tool actions. The existing fail-closed +parser is the sole text eligibility source: it requires merge intent and a PR +reference and excludes additional actions, governed actions, and document/attachment targets. +No accepted or cancelled history keeps a deadline alive. A full page without any +merge candidates still advances its keyset cursor; later candidates are not lost. + +Ordinary creation registers the topic before inserting a confirmation in its +transaction. Existing confirmation-resolution notifications invalidate its issue +before acceptance, rejection, withdrawal, supersession, and administrative expiry. +Issue status and visibility changes invalidate that issue before their update. +The signal helper joins caller-owned outer transactions; nested savepoint release +is not a commit. Startup reconstructs existing candidates. Direct tool, secret, +and OAuth interaction inserts do not create eligible merge-only work. Company +portability does not import pending interaction rows. + +The supported writer model is the instance's ordinary service/API process. +Independent SQL writers must use the same transaction-aware notification contract +or explicitly invalidate/restart the owning worker. Process-local notifications +do not establish a cross-replica ingress guarantee. No permanent database listener +or empty recovery scan is added. + +## External observation and retries + +The existing normalized `applyPullRequestEvent` to +`sweepMergedPullRequestConfirmations` seam remains available. A normalized merge +hint checks matching references in its company; unrelated tenants and PRs do not +cause provider reads. Existing provider/binding authorization remains unchanged. + +A pending candidate can reference a PR whose external changes have no dependable +webhook coverage. Each such candidate retains a five-minute observation deadline. +Resolution, withdrawal, supersession, hiding, or terminal issue state removes the +obligation. Provider/read failures and unknown states retain the candidate, using +the existing scoped helper's 30-second exponential backoff capped at five minutes. +These retries do not scan completed history. Removing candidate reconciliation +requires a proven durable delivery/replay guarantee for every referenced PR, +including the first event after idle and supported non-Cloud installations. + +This is candidate-backed external observation. It does not eliminate all polling, +register a Cloud wake alarm, or prove that the host/database suspends. Existing +sleep-safety blockers remain in force. + +Startup registers this observation without waiting for live GitHub requests. +The internal resolver invocation accepts an optional host-owned abort signal; +that signal is not part of external-object records or plugin/RPC payloads. +The GitHub owner combines cancellation with a ten-second request budget and +passes it to the actual object fetch and repository-visibility probes, including +response body reads. Shutdown aborts the request and waits for its cancellation; +no post-abort acceptance is committed. A timeout leaves the durable card pending. + +## Acceptance evidence + +Focused tests use real PostgreSQL, the actual service producers, transaction +signals, coordinator, scheduler, acceptance transaction, and persisted audit and +chat continuation outbox. GitHub state and the direct heartbeat wake callback are +controlled simulation boundaries. The tests do not claim live GitHub delivery or +real agent execution. + +Cases cover empty/configured idle and retained terminal history with no recurring +SQL/provider work across old cadence windows; the first ordinary creation; +outer commit/rollback; exact candidate and failure deadlines; withdrawal; +provider failure across drain; standby/resume; restart; tenant-scoped and duplicate +merge hints; latest/superseded cards during provider observation; hidden/terminal +issues; URL targets; no-continuation policy; and shutdown during an in-flight read. +Existing startup and normalized GitHub event tests exercise the changed consumers. +Controlled local HTTP tests hold headers, partial response bodies, 404 probes, +and public-visibility probes. They observe server-side response close after +shutdown within one second, rather than only racing a promise. A shortened +request budget times out a real request; the saved card then accepts after +restart when that same HTTP fixture serves a merged response. Existing GitHub +provider and external-object service tests preserve their authorization behavior. + +## Verification follow-up + +The original head passed full-workspace typecheck and build. A later required +Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in +`managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package` +where `local_storage` was expected. Both the isolated case and the complete +unchanged 17-test file pass against fresh local PostgreSQL. The original Linux cause is +not established; that failure is retained and is not called a flake. + +After merging current master, the same fixture failed locally because the new +managed-Cloud policy from PR #18350 correctly refuses local disk. A separate +test-only prerequisite commit changes that fixture to the real S3 provider with +a controlled HTTP endpoint, valid fixture credentials, and a stable test key. +It verifies remote reads without seeding or mutating runtime configuration, +rejects corrupt and missing objects, and recovers after restoring canonical bytes. +The full existing 17-test file passes with these failure controls. Production +configuration validation remains unchanged. Current-head required CI remains +the completion gate. + +## Existing limitations outside this migration + +Direct non-chat continuation still uses the existing postcommit `wakeup` call. +If that call fails after acceptance commits, the previous periodic sweep also +skipped the accepted row and did not reconstruct the missing wake. This migration +preserves that behavior; a general durable interaction-continuation contract is +a separate follow-up. Real chat bindings retain the existing transactional +`interaction_wakeup` outbox, verified through ordinary creation and acceptance. + +The separate review-authority issue tracked in public issue #17704 and PR #17711 +remains owned there. The worker isolates candidate failures and preserves current +review authorization; it does not grant a system actor new review authority. diff --git a/doc/plans/2026-10-30-routine-status-sleep.md b/doc/plans/2026-10-30-routine-status-sleep.md new file mode 100644 index 00000000000..0afe809bb8a --- /dev/null +++ b/doc/plans/2026-10-30-routine-status-sleep.md @@ -0,0 +1,61 @@ +# Routine execution status settlement + +The Cloud routine status worker repairs a routine invocation when its linked +execution task and saved routine status disagree. The previous heartbeat callback +scanned every five minutes, even with no routines or only completed history. + +## Ownership and trigger + +`routine-run-status.ts` owns the projection, custody checks and repair transaction. +The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue +creation, status/origin changes, checkout/release, routine issue association, +execution control writes, review state writes and subtree status restoration. +Bulk restoration registers each issue hint in the same transaction before its +grouped write. Restored history does not depend on an optional agent wake. It uses the existing +transaction-aware work notification protocol. An outer transaction owns the hint; +a savepoint release does not wake the repair worker before commit. + +An execution must retain reciprocal task/invocation links in the same company +and belong to a routine in that company. This is the existing routine status +owner contract. Origin labels can change when a plugin declaration changes or +is removed; retained reciprocal links preserve historical execution custody. Repair +locks the routine and task and rechecks the linked invocation before writing. +Direct route hooks also lock the execution task and linked invocation in their +own transaction, nested under caller transactions when present. An older hook +cannot write after a newer task status has committed. It preserves skipped/coalesced history and unrelated dispatch failures. A task +that resumes clears an execution-task transient failure through the same owner +used by native completion and issue routes. + +`activeIssue` remains derived from open tasks and live heartbeat runs. A terminal +heartbeat with an open task does not create a completion obligation. The worker +does not retry merely because a run is active or a routine exists. + +## Lifecycle and writer contract + +`registerScopedDeadlineWork` owns coalescing, fair batches, failure backoff, drain +and shutdown. A failed owner retries with bounded backoff. Successful settlement +removes that owner. Settled configuration, healthy active tasks and completed +history leave no recurring worker SQL. The existing Cloud and heartbeat scheduler +enablement gates remain in place. Ordinary routine scheduling is unchanged. + +Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before +their mutation. Common execution writers without an issue hint request debt reconstruction only +for their recorded company. Projection debt settles even when that company is +paused; company activity does not create or cancel this obligation. Backups and independent SQL import/repair tools must restore +before startup or call that same transaction contract; process-local hints do +not observe independent SQL or another process. No permanent listener or periodic +configured-routine scan is added. + +The in-process retry deadline does not establish external Cloud wake coverage. +This change preserves the existing host sleep safety checks and does not claim +measured host or database suspension. + +## Acceptance + +Tests exercise ordinary issue writes and a real heartbeat lifecycle transition +against PostgreSQL with the production coordinator. Provider wakeup is simulated. +They inspect durable routine rows and the normal routine detail read separately. +Coverage includes empty/configured/history quiet periods, first producer, +terminal outcomes and reopen, legacy mismatch reconstruction, skipped/coalesced +custody, outer commit/rollback, retry/restart, drain/resume and standby/disable. +Focused worker tests exercise in-flight invalidations, retry isolation and stop. diff --git a/doc/plans/2026-10-30-run-dispatch-sleep.md b/doc/plans/2026-10-30-run-dispatch-sleep.md new file mode 100644 index 00000000000..e5cd0a9e014 --- /dev/null +++ b/doc/plans/2026-10-30-run-dispatch-sleep.md @@ -0,0 +1,34 @@ +# Run dispatch without idle retry scans + +The run-dispatch module owns retry admission, its durable `scheduled_retry_at` +deadline, and atomic promotion. The heartbeat queue owns concurrency, current +execution authority, and dispatch. The shared coordinator owns notification +coalescing, nonoverlap, transaction uncertainty, drain/resume and shutdown. +The heartbeat facade and `index.ts` only connect these owners. + +A scheduled retry is a future obligation. Startup reconstructs its next stored +deadline through the run-dispatch adapter. Writes that create, move, cancel or +invalidate retries notify the retry worker in their transaction. Promotion uses +the existing gate and compare-and-set, and its committed write wakes ordinary +queued dispatch. A queued run behind occupied capacity waits for the owning +run's lifecycle transition. Configuration and authority changes also wake it. +An idle queue has no recurring read or timer-driven dispatch attempt. Notifications batch for one second and carry the changed company to queued dispatch. Known retry deadlines still preempt batching. Cancellation-owned retries stay with their cancellation worker and do not retain a promotion deadline. + +The supported writer contract is service writes through the shared database +connection scope, including caller-owned and nested transactions. Existing +explicit domain notification boundaries cover lifecycle, issue review state, +agent configuration/status, company lifecycle and scheduling settings. Direct +SQL writers and independent replicas must provide an explicit ingress signal or +restart reconstruction; a process-local signal does not observe their commits. +This change adds no permanent database listener or write interception. + +The remaining periodic recovery chain still owns deferred comment receipts, +execution-wait cancellation recovery, orphan/resource recovery, stranded issue +assignments, dependency wake repair, task watchdogs, silent active runs and stale +locks. Those duties retain their backstop until their separate producer/deadline +migrations are qualified. This slice removes only the periodic scheduled-retry +promotion call. Ordinary queued dispatch can still be reached by that backstop; +the existing atomic claim and per-agent start lock prevent duplicate execution. + +These workers use internal process deadlines. They do not register Cloud wake +alarms and do not relax the durable idle-sleep safety proof. diff --git a/doc/plans/2026-10-30-status-cards-sleep.md b/doc/plans/2026-10-30-status-cards-sleep.md new file mode 100644 index 00000000000..3f3b8933cbc --- /dev/null +++ b/doc/plans/2026-10-30-status-cards-sleep.md @@ -0,0 +1,157 @@ +# Status-card evaluation deadlines + +Status-card evaluation uses existing cards, updates and issues as durable truth. +`index.ts` registers a typed worker. The status-card owner controls evaluation, +generation claims, assignment through the ordinary issue wake API, and failed +assignment cancellation. Shared delivery infrastructure controls transaction +settlement, coalescing, retries, admission drain, standby and shutdown. + +Manual cards have no automatic obligation. Interval cards retain the requested +evaluation cadence. Reactive cards evaluate committed source changes and retain +only an actual debounce, hourly rate, daily budget, active-hour or saved-query +membership deadline. Unchanged cards disarm. A changed experimental setting +invalidates the worker's cached feature gate. Other source writes do not reread +settings. Startup reconstructs every disarmed reactive scope together with any +concurrent company-scoped hint, without moving saved eligibility/retry deadlines. + +Company search reports the earliest matching `updatedWithin` expiry through an +optional internal callback in its existing aggregate query. It includes rows +outside the returned page: an expiring priority-ranked row ahead of an offset +can change the watched set. The callback exposes only a normalized deadline; +ordinary search consumers and API responses retain their existing contract. +Overlapping query scopes can require an evaluation when one query expires even +if another still retains the same issue; no generation occurs without a filtered +card change. No recurring scan replaces the expired query obligation. + +## Claims and recovery + +A feature-owned PostgreSQL transaction advisory lock serializes scheduled +per-card evaluation across connections. Its transaction holds no card row lock +while existing issue/card services commit through their own transactions. The +lock remains held through actual refresh settlement. Existing card deadline CAS, +generating-issue ownership and issue idempotency keys remain in force. Manual +refresh and older instances continue to use those existing guards. A busy lock +retains a company-scoped retry; consuming a hint is not successful evaluation. +A failed scheduled refresh retains the existing five-minute durable claim. + +Once a generation exists, the normal issue assignment authority remains its +owner. Assignment failure cancels that issue and finalizes the card's failed +ledger entry. The user's ordinary refresh action can retry it. Restart recovery +must not repeat an already completed generation. No new timer table, automatic +write replay, execution-authority exemption or sleep-safety relaxation is added. + +## Supported producer coverage + +All hints are registered before the relevant write within its owning transaction. +Nested service writes share outer commit ownership. Company-scoped hints batch +reactive reconciliation; ordinary execution writes without known scope retain +all existing execution consumers without sending an instance-wide card hint. + +| Producer | Notification boundary and coverage | +| --- | --- | +| Card create, configuration, archive/restore, delete, compile, query/summary result, evaluation claim | Status-card mutation transactions; stalled issue finalization signals before releasing the card claim | +| Experimental status-card setting | Instance experimental-settings transaction; full reconstruction is appropriate for this instance-wide setting | +| Company pause/archive/reactivate/delete | Company lifecycle/deletion transactions and budget-owned company hard-stop/resume transactions | +| Issue create/update/delete and label changes | Ordinary issue transactions; label deletion signals its known company before cascades; title and prefix changes signal their existing owner transactions | +| Issue checkout/adoption/release/reset and environment-selection recency | Existing issue transaction, or a transaction around the previously standalone write; company comes from the existing issue/run read | +| Comment create, edit, delete, tombstone, import and recency | Common comment insert; outer root comment transaction; queued-comment and Telegram edit transactions; existing metadata-upgrade path; remove/tombstone transactions | +| Issue import, comment import, document import | One notification per affected company in the existing batch transaction; no per-row notification lookup | +| Document create/update/restore/delete and search membership | Document owner transactions; only the existing company is passed, without copying search predicates into producers | +| Review, interaction and execution transitions | Existing issue-review-state and native-status-reassessment boundaries; scoped execution-control helpers; interaction recency touches own a transaction when called after settlement | +| Run admission, retries, monitors, watchdog/recovery and native settlement/reclaim | Existing scoped review/milestone/status/execution-control hooks; explicit notifications at the uncovered issue execution bind, legacy claim, monitor claim/release and review-participant recovery transaction | +| Agent deletion/access reassignment and project deletion/privacy cascades | Existing company-scoped owner transactions before foreign-key or recursive issue changes | +| Workspace selection/binding, plugin/provider workspace and cleanup/audit touches | Existing owner transactions, including standalone cleanup and audit recency writes now enclosed with their intent | +| Skill-test harness expiry and task-watchdog fingerprint touch | Existing owner mutation transaction; no feature policy is moved into the writer | + +The audit includes direct issue/comment mutations, raw recursive privacy updates, +bulk insert helpers, label/project/agent cascades, linked document search content +and timestamp-dependent search membership. Metadata-only writes that do not +change the fingerprint, matching fields or visibility do not create an evaluation +obligation: examples include run identity linkage, workspace IDs without a +recency change, document locks and source-trust receipts without searched content +changes. Routine and status-summary documents that have no issue-document link +do not change issue search membership. + +This follows the existing [single-active-writer contract](../idle-sleep-safety.md). +Supported services, import/recovery writers, caller-owned transactions and startup +reconstruction participate. Independent SQL writers or another active server +process require explicit wake ingress; process-local hints do not establish +cross-replica notification delivery. The advisory lock supplies competing +scheduler exclusion, not cross-process event transport. Internal deadlines do +not establish a Cloud wake alarm or prove host/database suspension. + +## Verification + +Real PostgreSQL acceptance uses ordinary card create/compile and authorized +summarizer writeback. The assignment boundary is controlled: it persists a wake, +and the test explicitly performs the authorized result write; no model runs. +Checks cover empty/manual/reactive/disabled quiet SQL, first source change, +debounce and durable wake/result, earlier/cancelled deadlines, outer commit and +rollback, drain/resume, competing connections, busy locks, restart eligibility, +active hours/rate/budget gates, multiple companies, budget-owned company resume, +ignored activity, document/comment membership, relative query expiry including +an off-page candidate, failed assignment cleanup/manual retry and the persisted +five-minute internal-failure retry across restart/drain. + +Shared coordinator regressions also cover warm standby, ambiguous transaction +intent, failure entering drain and different domain/coordinator gates. The paused +worker releases only settled holds and retains a future in-memory resume wake. +An expired saved deadline cannot schedule an immediate skipped-attempt loop. +Existing startup, status-card, company-search, execution and monitor suites cover +the production wiring and unchanged consumers. Exact commands and results belong +in the PR verification record; this document does not claim deployed savings. + +### Review corrections: pool capacity and restart deadline ownership + +Scheduled evaluation holds its advisory transaction on `withDedicatedDbConnection`, +not on the ordinary write pool. The helper ends that physical connection in +`finally`, including contention and failed evaluation. Its lock still spans the +actual refresh settlement. Summary writeback runs watched-query reads on its +existing transaction, so a one-connection main pool completes the full flow. + +Migration0419 adds nullable internal `next_eval_reason` to the existing card row. +Membership deadlines may be reconstructed at startup after a committed source +hint was lost. Policy eligibility and the atomically recorded five-minute retry +lease retain their saved deadline. Explicit refresh deadlines remain due work. +The HTTP card response removes this internal provenance; the public schema stays +unchanged. + +For legacy rows, a pending change hash preserves policy eligibility. An untagged +active deadline farther than five minutes away cannot be an outstanding +five-minute claim, so startup rechecks it. An ambiguous untagged deadline within +five minutes keeps its remaining time without extension. This bounds migration +compatibility delay while preserving prior crash/failure recovery. After the +first successful evaluation all newly saved deadlines have explicit ownership. +No recurring compatibility scan is added. + +Source notifications use the shared deadline helper's fixed one-second batching +window. The first hint after idle can reconcile immediately. Later hints merge +by company until that window ends. The window does not slide with each new +write, so a sustained source stream cannot postpone reconciliation forever. +One second keeps reactive cards prompt (the removed heartbeat default was 30s) +and bounds burst SQL. No minimum spacing is set: an already-known policy, +expiry, or retry deadline can preempt batching. The window alone creates no +scheduled work after settlement. Real issue/comment producer acceptance measures +due/next queries and actual searches under 100 writes, preserves both companies' +hints, dispatches another company's saved deadline inside the batching window, +and checks SQL quietness after authorized results. + +Card deletion resolves the card's owning company and emits only that company's +deadline-reconstruction hint. Every card mutation requires a company ID; only +instance-level settings retain wildcard ingress. Deleting an already-missing +card emits no hint. The real two-company regression proves deletion does not +re-run unchanged reactive searches or wake either summarizer, then stays quiet. + +Active-hours eligibility keeps the existing exact UTC-minute search across DST +changes. A bounded 32-entry timezone formatter cache removes repeated ICU +construction. A bounded 256-entry cache reuses the next opening for the exact +(timezone,start,end) policy only while the candidate instant stays within its +previously searched closed range. The precise instant is checked first, so +historical second-based offsets, the opening itself, and millisecond deadlines +keep their behavior. Clock rollback and policy changes cannot reuse the wrong +range. Empty windows return without formatter or search work. A cold nonempty +policy can still perform at most 2881 conversions including its first check; warm +closed-range evaluations perform one conversion, not another minute walk. +Both caches use bounded LRU retention and hold no company/card data. Tests prove +eviction, skipped/repeated hours, half-hour DST, date-line changes, rollback, +strict boundaries, and actual twenty-card source delivery followed by quiet. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index 3e21ef72901..f2d4250a4cb 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -62,6 +62,14 @@ authority as company-agent API access. Gaps: Instance start/stop and task workspace start/stop are different capabilities. Local service control is not agent company authority. +Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence +reads authenticate the tenant host with its server capability; they expose no +new board or agent endpoint. Their signed scope cannot authorize task recovery. +[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts) +cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts) +cover signature, identity, freshness and separation from recovery authority. +These checks do not establish a deployed sleep/wake result. + ## `configuration` Capability ID: `instance-operations/configuration`. diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 7a01c24ebf6..dd43901323e 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect before headers arrive. Confirm that neither leaves a permanent request hold. An unfinished application write must still prevent sleep after its client leaves. +Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts) +exercises the runtime inspection path for failed Cloud sandbox leases, including +verification leases without a run. A signed cleanup-only absence observation +can remove this blocker without changing failed outcomes or granting run +continuation. An unresolved provider resource still prevents idle sleep. +Manual: on an owned Cloud staging workspace, confirm the exact published build, +reconcile failed test allocations, then verify actual provider suspension and +normal hosted entry after wake. Local tests do not prove that hosted journey. + ### `settings-backup` Cloud startup reads nonsecret application choices from a private, versioned diff --git a/maps/feature-map/steering.md b/maps/feature-map/steering.md index 840fcd1ba74..611f0e56275 100644 --- a/maps/feature-map/steering.md +++ b/maps/feature-map/steering.md @@ -60,6 +60,13 @@ inspect the resulting task and run in Paperclip. The explicit `stop-redirect` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) covers `steering/stop-and-pause` only for Stop and one new direction. It requires active work, native cancellation acknowledgement, resource cleanup, and completion without later original-turn activity. It does not qualify pause, interrupt, or all queued-message behavior. Source registration is not a live Cloud pass. +The explicit `followup-while-running` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up) +covers `steering/queue`: one follow-up is saved while a bound native command is +active, its late requirement appears in one attributed final reply, the queue +empties, and the result survives reload. One steered turn or one normal successor +is allowed. Unit calibration and source registration are not live acceptance. + + Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use an agent that stays busy long enough to inspect the queue. Record the adapter, native/legacy mode, and advertised steering capability. Test supported and diff --git a/packages/db/src/migrations/0419_moaning_vulture.sql b/packages/db/src/migrations/0419_moaning_vulture.sql new file mode 100644 index 00000000000..03ea349aec1 --- /dev/null +++ b/packages/db/src/migrations/0419_moaning_vulture.sql @@ -0,0 +1 @@ +ALTER TABLE "status_cards" ADD COLUMN "next_eval_reason" text; \ No newline at end of file diff --git a/packages/db/src/migrations/meta/0414_snapshot.json b/packages/db/src/migrations/meta/0419_snapshot.json similarity index 99% rename from packages/db/src/migrations/meta/0414_snapshot.json rename to packages/db/src/migrations/meta/0419_snapshot.json index 2679e7d98bf..d03afbb279d 100644 --- a/packages/db/src/migrations/meta/0414_snapshot.json +++ b/packages/db/src/migrations/meta/0419_snapshot.json @@ -1,6 +1,6 @@ { - "id": "27c9008c-60e3-449b-9630-9e4977fad444", - "prevId": "5690a14a-f9cc-41a6-af51-c27f688f4ef6", + "id": "cce00822-b7a4-4acd-9c59-0f6f5822c0cf", + "prevId": "c777fa84-85a6-418f-bea1-0d60b56cf071", "version": "7", "dialect": "postgresql", "tables": { @@ -19052,6 +19052,98 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.company_skill_mutation_receipts": { + "name": "company_skill_mutation_receipts", + "schema": "", + "columns": { + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "skill_id": { + "name": "skill_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "version_id": { + "name": "version_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "company_skill_mutation_receipts_key_idx": { + "name": "company_skill_mutation_receipts_key_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "company_skill_mutation_receipts_company_id_companies_id_fk": { + "name": "company_skill_mutation_receipts_company_id_companies_id_fk", + "tableFrom": "company_skill_mutation_receipts", + "tableTo": "companies", + "columnsFrom": [ + "company_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_skill_repository_snapshots": { "name": "company_skill_repository_snapshots", "schema": "", @@ -19079,7 +19171,46 @@ "name": "file_inventory", "type": "jsonb", "primaryKey": false, - "notNull": true + "notNull": true, + "default": "'[]'::jsonb" + }, + "storage": { + "name": "storage", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "upload": { + "name": "upload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "restore": { + "name": "restore", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "maintenance_at": { + "name": "maintenance_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, + "restore_retry_at": { + "name": "restore_retry_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false }, "created_at": { "name": "created_at", @@ -19110,6 +19241,38 @@ "concurrently": false, "method": "btree", "with": {} + }, + "company_skill_snapshots_migration_idx": { + "name": "company_skill_snapshots_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_snapshots_maintenance_idx": { + "name": "company_skill_snapshots_maintenance_idx", + "columns": [ + { + "expression": "maintenance_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} } }, "foreignKeys": { @@ -19999,6 +20162,25 @@ "primaryKey": false, "notNull": false }, + "package_snapshot_id": { + "name": "package_snapshot_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "package_file_map": { + "name": "package_file_map", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "migration_at": { + "name": "migration_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false, + "default": "now()" + }, "revision_number": { "name": "revision_number", "type": "integer", @@ -20057,6 +20239,52 @@ } }, "indexes": { + "company_skill_versions_migration_idx": { + "name": "company_skill_versions_migration_idx", + "columns": [ + { + "expression": "migration_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"company_skill_versions\".\"migration_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_package_snapshot_idx": { + "name": "company_skill_versions_package_snapshot_idx", + "columns": [ + { + "expression": "package_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "company_skill_versions_repository_snapshot_idx": { + "name": "company_skill_versions_repository_snapshot_idx", + "columns": [ + { + "expression": "repository_snapshot_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, "company_skill_versions_skill_revision_idx": { "name": "company_skill_versions_skill_revision_idx", "columns": [ @@ -20168,6 +20396,19 @@ "onDelete": "restrict", "onUpdate": "no action" }, + "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": { + "name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk", + "tableFrom": "company_skill_versions", + "tableTo": "company_skill_repository_snapshots", + "columnsFrom": [ + "package_snapshot_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + }, "company_skill_versions_author_agent_id_agents_id_fk": { "name": "company_skill_versions_author_agent_id_agents_id_fk", "tableFrom": "company_skill_versions", @@ -20613,6 +20854,78 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.skill_package_deletions": { + "name": "skill_package_deletions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_key": { + "name": "archive_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "manifest_key": { + "name": "manifest_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delete_after": { + "name": "delete_after", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "skill_package_deletions_due_idx": { + "name": "skill_package_deletions_due_idx", + "columns": [ + { + "expression": "delete_after", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.company_transfer_runs": { "name": "company_transfer_runs", "schema": "", @@ -33849,7 +34162,7 @@ } ], "isUnique": false, - "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE", + "where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE", "concurrently": false, "method": "btree", "with": {} @@ -51261,6 +51574,195 @@ "checkConstraints": {}, "isRLSEnabled": false }, + "public.sandbox_creation_operations": { + "name": "sandbox_creation_operations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true + }, + "company_id": { + "name": "company_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "original_environment_id": { + "name": "original_environment_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "request_key": { + "name": "request_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'run'" + }, + "issue_id": { + "name": "issue_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "execution_workspace_id": { + "name": "execution_workspace_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "driver_key": { + "name": "driver_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "config": { + "name": "config", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "lease_id": { + "name": "lease_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "sandbox_creation_operations_pending_deadline_idx": { + "name": "sandbox_creation_operations_pending_deadline_idx", + "columns": [ + { + "expression": "next_attempt_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"sandbox_creation_operations\".\"state\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "sandbox_creation_operations_owner_idx": { + "name": "sandbox_creation_operations_owner_idx", + "columns": [ + { + "expression": "company_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "original_environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "request_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "sandbox_creation_operations_environment_id_environments_id_fk": { + "name": "sandbox_creation_operations_environment_id_environments_id_fk", + "tableFrom": "sandbox_creation_operations", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, "public.secret_access_events": { "name": "secret_access_events", "schema": "", @@ -52268,6 +52770,12 @@ "primaryKey": false, "notNull": false }, + "next_eval_reason": { + "name": "next_eval_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, "next_eval_at": { "name": "next_eval_at", "type": "timestamp with time zone", diff --git a/packages/db/src/migrations/meta/_journal.json b/packages/db/src/migrations/meta/_journal.json index 47d9d449f1a..3cc42512d8c 100644 --- a/packages/db/src/migrations/meta/_journal.json +++ b/packages/db/src/migrations/meta/_journal.json @@ -2920,6 +2920,13 @@ "when": 1793395364937, "tag": "0418_curved_lady_mastermind", "breakpoints": true + }, + { + "idx": 419, + "version": "7", + "when": 1793402408013, + "tag": "0419_moaning_vulture", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/schema/status_cards.ts b/packages/db/src/schema/status_cards.ts index 06efaa0368b..fd18d22fb8b 100644 --- a/packages/db/src/schema/status_cards.ts +++ b/packages/db/src/schema/status_cards.ts @@ -56,6 +56,9 @@ export const statusCards = pgTable( lastModel: text("last_model"), generatingIssueId: uuid("generating_issue_id").references(() => issues.id, { onDelete: "set null" }), failureReason: text("failure_reason"), + // Internal deadline ownership; null denotes a row written before provenance + // was recorded. Membership may be rechecked at startup, retry/policy may not. + nextEvalReason: text("next_eval_reason").$type<"membership" | "policy" | "retry" | "refresh">(), nextEvalAt: timestamp("next_eval_at", { withTimezone: true }), archivedAt: timestamp("archived_at", { withTimezone: true }), archivedByUserId: text("archived_by_user_id"), diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 0fbef29f508..f04aab08a23 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -98,6 +98,35 @@ digest, and refuse the rest. A profile identity does not establish hosted harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc) summarizes these practices for contributors. +Qualified ACPX native executables use Runner's persistent digest cache. The cache +stores a computed SHA-256 digest under a key made from device, inode, size, +nanosecond modification time, and nanosecond change time. Each check compares +that digest with the independently selected build-owned pin. A changed file +identity, missing entry, or unusable cache causes a full descriptor read and +hash. A hit reuses the earlier digest; metadata identity is not a fresh content +hash. Claude, Codex, and Grok use the same native executable admission path. + +Cache entries live in the operating-system temporary directory under +`paperclip-runner-/executable-digests-v1`, outside task workspaces such as +`/home/agent`. Neither `HOME` nor the task working directory selects this path. +Node's `os.tmpdir()` selects the root and honors the platform's absolute +`TMPDIR`, `TMP`, or `TEMP` override. Hosts must keep that root outside task +workspaces; a relative or unusable root disables caching. Temporary-directory +eviction removes the optimization until a later full hash fills it again. The +directory and files are private to that user. Writes use a temporary file and +atomic rename, so later Runner or sidecar processes can reuse complete entries. +Cache errors fall back to hashing. The cache stores no credentials or file paths. + +Every admission still opens the executable without following its final symlink, +checks its regular-file type, bounds, execute permission, and before/after file +identity, and retains the descriptor for launch. Package qualification and the +preparation-to-launch identity check remain. The cache deliberately trusts the +operating-system user: a process with that user's access can plant a digest for +a chosen fingerprint. It is not authenticated with a secret or signature. +An administrator-owned read-only Runner can therefore use a user-writable cache, +but this provides weaker protection against that same user than hashing every +executable on each admission. Removing the cache restores full hashing. + ## External controller checkpoints The optional controller persistence callback stores recovery control state, not diff --git a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts index 897a4d675be..af78eef13bb 100644 --- a/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts @@ -45,9 +45,9 @@ reader.on('line', line => { if(request.method==='initialize') { process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{protocolVersion:1,agentInfo:{name:'antigravity-acp',version:'1.4.3'},agentCapabilities:{_meta:{antigravity:{paperclipAdmission:1}}},authMethods:[{id:'oauth-personal'}]}})+'\\n'); } else if(request.method==='authenticate') { - const url = mode==='bad-url' ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; + const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; const message='Open the following link to authenticate the ACP server: '+url+'\\n'; - process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42)); + if (!mode.endsWith('url-after-result')) {process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42));} if(mode==='blocked' || mode==='bad-url' || mode==='stubborn' || mode==='retained') return; if(mode==='error') {process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,error:{code:-32000,message:'do not expose fixture-refresh-token'}})+'\\n');return;} fs.mkdirSync(path.join(home,'antigravity-acp'),{mode:0o700}); @@ -55,7 +55,14 @@ reader.on('line', line => { process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{}})+'\\n'); } else {throw new Error('inference/session request forbidden');} }); -reader.on('close',()=>mode==='stubborn'||mode==='retained'?setInterval(()=>{},1000):process.exit(0)); +reader.on('close',()=> { + if (mode.endsWith('url-after-result')) { + const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())}; + // EOF proves the client received the authenticate result before this stderr. + process.stderr.write('Open the following link to authenticate the ACP server: '+url+'\\n', () => process.exit(0)); + } else if(mode==='stubborn'||mode==='retained') setInterval(()=>{},1000); + else process.exit(0); +}); `); setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) }); setup.spawn.mockImplementation((_args, options) => { @@ -95,8 +102,8 @@ describe("Antigravity setup-only Google authentication", () => { // The login predicate keeps its five-second bound. Allow ten more seconds // for fixture startup and the durable mocked credential checks on a busy CI worker. - it("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls", async () => { - const home = await fixture(), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch); + it.each(["success", "url-after-result"])("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls (%s)", async mode => { + const home = await fixture(mode), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch); const login = startAntigravityBrowserLogin(home); await vi.waitFor(() => expect(login.outcome).toBe("success"), { timeout: 5_000 }); expect(login.authorizationUrl).toBe(authorizationUrl()); @@ -118,13 +125,13 @@ describe("Antigravity setup-only Google authentication", () => { expect(fetch).toHaveBeenCalledTimes(6); }, 15_000); - it.each(["error", "bad-url"])("rejects %s without leaking provider text or calling Google", async mode => { + it.each(["error", "bad-url", "bad-url-after-result"])("rejects %s without leaking provider text or calling Google", async mode => { const home = await fixture(mode), fetch = vi.fn(); vi.stubGlobal("fetch", fetch); const login = startAntigravityBrowserLogin(home); await vi.waitFor(() => expect(login.outcome).toBe("failure"), { timeout: 5_000 }); await vi.waitFor(() => expect(setup.close).toHaveBeenCalledOnce(), { timeout: 5_000 }); expect(login.error).not.toContain("fixture"); - if (mode === "bad-url") expect(login.authorizationUrl).toBeUndefined(); + if (mode.startsWith("bad-url")) expect(login.authorizationUrl).toBeUndefined(); expect(fetch).not.toHaveBeenCalled(); }); diff --git a/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts new file mode 100644 index 00000000000..f33eb4fd39f --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createExecutableDigestCache, executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; +import { openVerifiedRuntimeExecutable } from "./installation-integrity.js"; + +const temporary: string[] = []; +const identity: ExecutableFileIdentity = { device: "1", inode: "2", size: "3", modifiedNanoseconds: "4", changedNanoseconds: "5" }; +const digest = `sha256:${"a".repeat(64)}`; +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(temporary.splice(0).map(path => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "runner-executable-digest-cache-")); + temporary.push(root); + const directory = join(root, "cache"); + const cache = createExecutableDigestCache(directory); + return { root, directory, cache }; +} + +describe("persistent executable digest cache", () => { + it("stores the actual digest in a private, versioned entry and survives a new cache instance", async () => { + const f = await fixture(); + expect(await f.cache.read(identity)).toBeNull(); + await f.cache.write(identity, digest); + expect(await createExecutableDigestCache(f.directory).read(identity)).toBe(digest); + const [name] = await readdir(f.directory); + expect(name).toMatch(/^v1-[a-f0-9]{64}\.json$/); + expect(JSON.parse(await readFile(join(f.directory, name), "utf8"))).toEqual({ version: 1, digest }); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o700); + expect((await lstat(join(f.directory, name))).mode & 0o777).toBe(0o600); + }); + + it.each(Object.keys(identity) as (keyof ExecutableFileIdentity)[])("invalidates a changed %s", async key => { + const f = await fixture(); + await f.cache.write(identity, digest); + expect(await f.cache.read({ ...identity, [key]: "changed" })).toBeNull(); + }); + + it.each(["malformed", "version", "digest", "oversized", "permissions", "symlink"] as const)( + "treats an unusable %s entry as a miss", async kind => { + const f = await fixture(); + await f.cache.write(identity, digest); + const [name] = await readdir(f.directory), file = join(f.directory, name); + if (kind === "malformed") await writeFile(file, "{"); + else if (kind === "version") await writeFile(file, JSON.stringify({ version: 2, digest })); + else if (kind === "digest") await writeFile(file, JSON.stringify({ version: 1, digest: "expected" })); + else if (kind === "oversized") await writeFile(file, "x".repeat(1025)); + else if (kind === "permissions") await chmod(file, 0o644); + else { await rename(file, `${file}.target`); await symlink(`${file}.target`, file); } + expect(await f.cache.read(identity)).toBeNull(); + }, + ); + + it("falls back when the cache directory cannot be used and leaves existing permissions unchanged", async () => { + const f = await fixture(); + await writeFile(f.directory, "not a directory"); + await expect(f.cache.write(identity, digest)).resolves.toBeUndefined(); + expect(await f.cache.read(identity)).toBeNull(); + await rm(f.directory); + await mkdir(f.directory, { mode: 0o755 }); + await f.cache.write(identity, digest); + expect(await f.cache.read(identity)).toBeNull(); + expect((await lstat(f.directory)).mode & 0o777).toBe(0o755); + }); + + it("does not resolve a relative cache directory through the task working directory", async () => { + const cache = createExecutableDigestCache("relative-task-cache"); + await cache.write(identity, digest); + expect(await cache.read(identity)).toBeNull(); + }); + + it("publishes complete entries under concurrent writers without leftover temporary files", async () => { + const f = await fixture(); + await Promise.all(Array.from({ length: 8 }, () => f.cache.write(identity, digest))); + expect(await f.cache.read(identity)).toBe(digest); + expect(await readdir(f.directory)).toHaveLength(1); + }); +}); + +describe.skipIf(!["linux", "darwin"].includes(process.platform))("descriptor admission with cached digests", () => { + async function executable() { + const f = await fixture(), path = join(f.root, "native"), bytes = Buffer.from("qualified native fixture"); + await writeFile(path, bytes, { mode: 0o700 }); + vi.spyOn(executableDigestCache, "read").mockImplementation(f.cache.read); + vi.spyOn(executableDigestCache, "write").mockImplementation(f.cache.write); + return { ...f, path, digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` }; + } + + it("hashes a miss, reuses the actual digest, and still rejects a different independent pin", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + expect(executableDigestCache.write).toHaveBeenCalledOnce(); + await expect(openVerifiedRuntimeExecutable(f.path, digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("rehashes changed contents and rejects corruption despite a previous valid entry", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "corrupted native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it.each(["permissions", "symlink", "replacement"] as const)("retains admission checks after cached %s changes", async kind => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + if (kind === "permissions") { + await chmod(f.path, 0o600); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("bounded executable file"); + } else if (kind === "symlink") { + await rename(f.path, `${f.path}.original`); await symlink(`${f.path}.original`, f.path); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("real regular file"); + } else { + await rename(f.path, `${f.path}.original`); await writeFile(f.path, "new invalid executable", { mode: 0o700 }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + } + }); + + it("rejects mutation while a cache hit is being read", async () => { + const f = await executable(); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + vi.mocked(executableDigestCache.read).mockImplementationOnce(async () => { + await writeFile(f.path, "mutation during lookup"); + return f.digest; + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("checks executable identity after publishing a cold cache entry", async () => { + const f = await executable(); + vi.mocked(executableDigestCache.write).mockImplementationOnce(async (identity, digest) => { + await f.cache.write(identity, digest); + await writeFile(f.path, "mutation during cache publication"); + }); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified"); + }); + + it("falls back to hashing when the cache is unavailable", async () => { + const f = await executable(); + await writeFile(f.directory, "unavailable cache"); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + await writeFile(f.path, "invalid native fixture"); + await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch"); + }); + + it("trusts a digest planted by the same operating-system user for current metadata", async () => { + const f = await executable(); + await writeFile(f.path, "content changed by the same user"); + const metadata = await lstat(f.path, { bigint: true }); + await f.cache.write({ device: metadata.dev.toString(), inode: metadata.ino.toString(), size: metadata.size.toString(), + modifiedNanoseconds: metadata.mtimeNs.toString(), changedNanoseconds: metadata.ctimeNs.toString() }, f.digest); + await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close(); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts index fbf39064e18..1fab0150997 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts @@ -2,6 +2,7 @@ import { MAX_ACPX_RUNTIME_EXECUTABLE_BYTES, ACPX_PRIVATE_SNAPSHOT_ENV, createAcp import { createNativeAcpxDistributionSnapshot, NATIVE_ACPX_BOOTSTRAP_NAME, readNativeAcpxDistributionEntries, type NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; export type { NativeAcpxDistributionInput } from "./native-distribution-integrity.js"; import { createHash } from "node:crypto"; +import { executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js"; import { spawn as spawnChildProcess, type ChildProcess, @@ -606,13 +607,7 @@ export async function awaitVerifiedAcpxProviderExit( await exitProof; } -interface VerifiedAcpxCommandIdentity { - device: string; - inode: string; - size: string; - modifiedNanoseconds: string; - changedNanoseconds: string; -} +type VerifiedAcpxCommandIdentity = ExecutableFileIdentity; interface VerifiedAcpxRuntimeExecutable { path: string; @@ -1204,7 +1199,7 @@ async function inspectCommand( } } -async function openVerifiedRuntimeExecutable( +export async function openVerifiedRuntimeExecutable( executablePath: string, expectedDigest: string, agent: string, @@ -1246,23 +1241,31 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable must be a bounded executable file`, ); } - const hash = createHash("sha256"); - const buffer = Buffer.alloc(1024 * 1024); + const cachedDigest = await executableDigestCache.read(fileIdentity(before)); + let digest = cachedDigest; let position = 0; - try { - while (position < Number(before.size)) { - const { bytesRead } = await handle.read( - buffer, - 0, - Math.min(buffer.length, Number(before.size) - position), - position, - ); - if (bytesRead === 0) break; - hash.update(buffer.subarray(0, bytesRead)); - position += bytesRead; + if (digest === null) { + const hash = createHash("sha256"); + const buffer = Buffer.alloc(1024 * 1024); + try { + while (position < Number(before.size)) { + const { bytesRead } = await handle.read( + buffer, + 0, + Math.min(buffer.length, Number(before.size) - position), + position, + ); + if (bytesRead === 0) break; + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + } finally { + buffer.fill(0); } - } finally { - buffer.fill(0); + digest = `sha256:${hash.digest("hex")}`; + } + if (cachedDigest === null && position === Number(before.size) && digest === expectedDigest) { + await executableDigestCache.write(fileIdentity(before), digest); } const after = await handle.stat({ bigint: true }); const lexicalAfter = await lstat(executablePath, { bigint: true }).catch( @@ -1271,7 +1274,7 @@ async function openVerifiedRuntimeExecutable( const beforeIdentity = fileIdentity(before); const afterIdentity = fileIdentity(after); if ( - position !== Number(before.size) || + (cachedDigest === null && position !== Number(before.size)) || lexicalAfter === null || lexicalAfter.isSymbolicLink() || !lexicalAfter.isFile() || @@ -1283,7 +1286,6 @@ async function openVerifiedRuntimeExecutable( `ACPX ${agent} runtime executable changed while it was verified`, ); } - const digest = `sha256:${hash.digest("hex")}`; if (digest !== expectedDigest) { throw new Error(`ACPX ${agent} runtime executable digest mismatch`); } diff --git a/packages/paperclip-runner/src/executable-digest-cache.ts b/packages/paperclip-runner/src/executable-digest-cache.ts new file mode 100644 index 00000000000..b50b54d9059 --- /dev/null +++ b/packages/paperclip-runner/src/executable-digest-cache.ts @@ -0,0 +1,70 @@ +import { createHash, randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { lstat, mkdir, open, rename, unlink } from "node:fs/promises"; +import { tmpdir, userInfo } from "node:os"; +import { isAbsolute, join } from "node:path"; + +export interface ExecutableFileIdentity { + device: string; + inode: string; + size: string; + modifiedNanoseconds: string; + changedNanoseconds: string; +} + +export function createExecutableDigestCache(directory?: string) { + async function location(identity: ExecutableFileIdentity, create: boolean): Promise { + const user = userInfo(); + const temporaryRoot = tmpdir(); + if (!isAbsolute(directory ?? temporaryRoot)) throw new Error("Executable digest cache directory is unavailable"); + const base = join(temporaryRoot, `paperclip-runner-${user.uid}`); + const directories = directory ? [directory] : [base, join(base, "executable-digests-v1")]; + for (const [index, path] of directories.entries()) { + if (create) await mkdir(path, { mode: 0o700 }).catch(error => { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + }); + const metadata = await lstat(path); + if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.uid !== user.uid || + (metadata.mode & (index === directories.length - 1 ? 0o077 : 0o022)) !== 0) { + throw new Error("Executable digest cache directory is unavailable"); + } + } + const key = createHash("sha256").update(JSON.stringify([1, identity.device, identity.inode, identity.size, + identity.modifiedNanoseconds, identity.changedNanoseconds])).digest("hex"); + return join(directories.at(-1)!, `v1-${key}.json`); + } + + return { + async read(identity: ExecutableFileIdentity): Promise { + try { + const path = await location(identity, false); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isFile() || metadata.uid !== userInfo().uid || (metadata.mode & 0o077) !== 0 || + metadata.size < 1 || metadata.size > 1024) return null; + const entry: unknown = JSON.parse(await handle.readFile("utf8")); + if (typeof entry !== "object" || entry === null || !("version" in entry) || entry.version !== 1 || + !("digest" in entry) || typeof entry.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(entry.digest)) return null; + return entry.digest; + } finally { await handle.close(); } + } catch { return null; } + }, + async write(identity: ExecutableFileIdentity, digest: string): Promise { + let temporary: string | undefined; + try { + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) return; + const path = await location(identity, true); + temporary = `${path}.${randomUUID()}.tmp`; + const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + try { await handle.writeFile(JSON.stringify({ version: 1, digest })); } + finally { await handle.close(); } + await rename(temporary, path); + } catch { } finally { + if (temporary) await unlink(temporary).catch(() => undefined); + } + }, + }; +} + +export const executableDigestCache = createExecutableDigestCache(); diff --git a/scripts/reuse-canary-image.mjs b/scripts/reuse-canary-image.mjs new file mode 100644 index 00000000000..3a43524dd0b --- /dev/null +++ b/scripts/reuse-canary-image.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { resolveStandardImageDigest } from "./standard-image-contract.mjs"; + +export async function reuseCanaryImage(sha, { + resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }), + run = (command, args) => execFileSync(command, args, { stdio: "inherit" }), +} = {}) { + assert.match(sha ?? "", /^[a-f0-9]{40}$/); + const digest = await resolve(sha); + if (digest === null) return false; + assert.match(digest, /^sha256:[a-f0-9]{64}$/); + const image = `ghcr.io/paperclipai/paperclip@${digest}`; + await run("gh", ["attestation", "verify", `oci://${image}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]); + await run("docker", ["buildx", "imagetools", "create", "-t", + `ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]); + return true; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const reused = await reuseCanaryImage(process.env.GITHUB_SHA); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`); + const summary = reused + ? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge." + : "No full-SHA master image exists. Building both platforms for this canary."; + console.log(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); +} diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs new file mode 100644 index 00000000000..26dfbc83c5f --- /dev/null +++ b/scripts/reuse-canary-image.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { reuseCanaryImage } from "./reuse-canary-image.mjs"; + +const sha = "a".repeat(40); +const digest = `sha256:${"b".repeat(64)}`; + +test("reuses only an exact-source, master-signed multi-platform image by digest", async () => { + const calls = []; + const reused = await reuseCanaryImage(sha, { + resolve: async source => { assert.equal(source, sha); return digest; }, + run: async (...args) => calls.push(args), + }); + assert.equal(reused, true); + assert.deepEqual(calls, [ + ["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]], + ["docker", ["buildx", "imagetools", "create", "-t", + "ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]], + ]); +}); + +test("a missing master image enables the fallback without retagging", async () => { + assert.equal(await reuseCanaryImage(sha, { + resolve: async () => null, + run: () => assert.fail("A missing image must not be promoted"), + }), false); +}); + +test("registry and provenance failures stop promotion rather than bypass verification", async () => { + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => { throw new Error("HTTP 403"); }, + run: () => assert.fail("A registry failure must not be promoted"), + }), /HTTP 403/); + let commands = 0; + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => digest, + run: command => { + assert.equal(command, "gh"); + commands++; + throw new Error("Untrusted source or signature"); + }, + }), /Untrusted source/); + assert.equal(commands, 1); +}); + +test("invalid source refs and digests cannot reach external commands", async () => { + for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) { + await assert.rejects(reuseCanaryImage(invalid, { + resolve: () => assert.fail("Invalid source accepted"), + })); + } + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => "mutable:tag", + run: () => assert.fail("Invalid digest accepted"), + })); +}); + +test("canary reuse gates both native builds and still runs convergent promotion", () => { + const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); + const reuse = workflow.split(" reuse_canary_image:\n")[1].split(" build-and-push:\n")[0]; + assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); + assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); + const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; + assert.match(build, /needs: \[reuse_canary_image\]/); + assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); + assert.match(build, /platform: linux\/amd64/); + assert.match(build, /platform: linux\/arm64/); + const promotion = workflow.split(" promote_canary_channel:\n")[1]; + assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/); +}); diff --git a/scripts/standard-image-contract.mjs b/scripts/standard-image-contract.mjs index 62a8249e2da..5e0ae3cc24c 100644 --- a/scripts/standard-image-contract.mjs +++ b/scripts/standard-image-contract.mjs @@ -21,6 +21,7 @@ export function standardImageDigest(bytes) { /** Read registry bytes directly; CLI stdout formatting is not artifact content. */ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + missing = "error", } = {}) { assert.match(sha ?? "", /^[a-f0-9]{40}$/); const request = async (url, headers, consume, allowNotFound = false) => { @@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } if (!response.ok) { await response.body?.cancel(); + if (allowNotFound && response.status === 404 && missing === "return-null") return null; const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404); assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`); const retryAfter = Number(response.headers.get("retry-after")); @@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { }; const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json()); assert.ok(typeof token === "string" && token, "Missing public pull token"); - const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { + const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", }, async response => { assert.ok(response.body, "Standard image index has no body"); @@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } finally { await reader.cancel(); } return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") }; }, true); + if (manifest === null) return null; + const { raw, expected } = manifest; const digest = standardImageDigest(raw); assert.equal(digest, expected, "Registry digest does not match the index bytes"); return digest; diff --git a/scripts/standard-image-contract.test.mjs b/scripts/standard-image-contract.test.mjs index af2d4462977..d61f3fb862d 100644 --- a/scripts/standard-image-contract.test.mjs +++ b/scripts/standard-image-contract.test.mjs @@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a assert.equal(calls, expectedCalls); } }); + +test("optional lookup falls back only for a missing manifest", async () => { + for (const status of [401, 403, 404, 503]) { + let calls = 0; + const lookup = resolveStandardImageDigest("a".repeat(40), async url => { + if (url.includes("/token?")) return Response.json({ token: "public-fixture" }); + calls++; + return new Response(null, { status }); + }, { missing: "return-null", sleep: async () => {} }); + if (status === 404) assert.equal(await lookup, null); + else await assert.rejects(lookup, /Registry lookup failed/); + assert.equal(calls, status === 503 ? 5 : 1); + } +}); diff --git a/server/src/__tests__/attention-retention-work.test.ts b/server/src/__tests__/attention-retention-work.test.ts new file mode 100644 index 00000000000..f06355e1ae7 --- /dev/null +++ b/server/src/__tests__/attention-retention-work.test.ts @@ -0,0 +1,897 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql, type SQL } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox, + decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions, + issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db"; +import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js"; +import { budgetService } from "../services/budgets.js"; +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { attentionService } from "../services/attention.js"; +import { companyService } from "../services/companies.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js"; +import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js"; +import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { createAgentLifecycle } from "../services/agent-lifecycle.js"; +import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js"; +import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL"); +const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +(support.supported ? describe : describe.skip)("remaining attention retention", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, agentId: string, draining = false; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + const statements: string[] = []; + const parameterCounts: number[] = []; + beforeAll(async () => { + if (externalUrl) await applyPendingMigrations(externalUrl); + else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-"); + db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + db.$client.options.onnotice = () => {}; + db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); }; + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; + await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` }); + await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); + vi.setSystemTime(new Date("2026-11-01T12:00:00Z")); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.transaction(async tx => { + await tx.execute(sql`set local client_min_messages = warning`); + await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`); + await tx.execute(sql`drop function if exists attention_retention_fail()`); + await tx.execute(sql`truncate companies cascade`); + }); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const old = () => new Date(Date.now() - delay - 600_000); + async function insertIssue(status = "todo", values: Partial = {}, database: Db = db) { + const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status, + createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning(); + return row; + } + async function change(write: (tx: Db) => PromiseLike, scope = { companyId }) { + return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); }); + } + const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and( + eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id))) + .then(rows => rows[0] ?? null); + async function settle() { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 }); + } + async function quiet() { + await settle(); expect(coordinator!.nextWakeAt()).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + statements.length = 0; + await vi.advanceTimersByTimeAsync(300_000); + expect(statements).toEqual([]); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); + } + async function start(service = attentionRetentionService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); + await registerAttentionRetentionWork(coordinator, service, () => !draining).ready; + return service; + } + async function archived(kind: string, id: string) { + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 }); + } + async function loadAll(service = attentionRetentionService(db)) { + const entries = []; + for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; } + } + const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId, + authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true }, + actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } }); + + it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => { + await insertIssue("done"); + const ticks: number[] = []; + for (let tick = 0; tick < 3; tick++) { + statements.length = 0; + const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true }); + await decisionRetentionService(db).autoArchive({ companyId, items: feed.items }); + ticks.push(statements.length); + await vi.advanceTimersByTimeAsync(30_000); + } + expect(ticks.every(count => count > 0)).toBe(true); + process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n"); + await start(); await quiet(); + }); + + it("has no recurring SQL when empty or when terminal source history remains", async () => { + await insertIssue("done"); await insertIssue("cancelled"); + await start(); await quiet(); + }); + + it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => { + await start(); await quiet(); + const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + await settle(); + const [saved] = await db.select({ at: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id)); + const due = saved.at + delay; + expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore(); + await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet(); + const outbox = await db.select().from(decisionArchiveNotificationOutbox); + expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id }); + expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review"); + await coordinator!.stop(); coordinator = undefined; await start(); await quiet(); + }); + + it("registers the ordinary createChild blocker path inside its relation transaction", async () => { + const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId }); + await start(); await quiet(); + const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, { + title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId, + blockParentUntilDone: true, + }); + expect(parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id), + eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet(); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1); + }); + + it("batches a full page of maintained reviews and confirmations by company", async () => { + const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } }))); + statements.length = 0; + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" }); + expect(page.rows).toEqual([]); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(20); + expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1); + await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId)); + statements.length = 0; + const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" }); + expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1); + }); + + it("batches decision and recovery source links and rechecks a late hidden origin", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning(); + const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId, + originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose", + options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning(); + const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId, + sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", + nextAction: "Choose", updatedAt: old() }))).returning(); + const service = attentionRetentionService(db); + const pages = []; + for (const sourceKind of ["decision", "recovery_action"] as const) { + statements.length = 0; + const page = await service.load({ companyId, sourceKind }); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(3); + pages.push(page.rows!); + } + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id)); + const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!; + const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!; + expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull(); + expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => { + const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: old() }))).returning(); + const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId, + status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning(); + await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id, + seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" }))); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const }; + statements.length = 0; + const page = await service.load(scope); + expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false); + expect(statements.length).toBeLessThanOrEqual(6); + const hiddenRun = runs[98], supersededRun = runs[99]; + const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!; + const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!; + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id)); + await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded", + contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() }); + expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull(); + expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows).toHaveLength(98); + expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false); + }); + + it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => { + const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(), + unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning(); + const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId, + title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning(); + await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId, + issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" }))); + await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`); + const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const }; + statements.length = 0; + const page = await service.load(scope); + const queries = [...statements]; + expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false); + const retentionReads = queries.filter(query => query.includes('from "decision_retention"')); + expect(retentionReads).toHaveLength(200); + expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30); + const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!; + expect(terminal.sourceId).toBe(terminals[99].id); + const [expected] = await db.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`); + expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due)); + await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id)); + expect(await service.check(terminal)).toBeNull(); + expect(await state("blocker_attention", terminals[99].id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + expect((await service.load(scope)).complete).toBe(true); + const refreshed = await service.load(scope); + expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false); + }); + + it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => { + const root = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId }); + const heavy = await insertIssue("blocked", { updatedAt: new Date() }); + const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES; + const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId, + title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning(); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" }, + { companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" }, + ...heavyNodes.map((node, index) => ({ companyId, issueId: node.id, + relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })), + ]); + const owner = issueService(db); + const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!; + const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!; + expect(expected.terminalBlockerIssueId).toBe(terminal.id); + expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId); + for (const input of [[root, heavy], [heavy, root]]) { + const batch = await owner.listBlockerAttentionBatch(companyId, input); + if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected); + if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false); + } + const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(entry.dueAt).toBeLessThanOrEqual(Date.now()); + if (limit === "depth") expect(entry.sourceId).toBe(terminal.id); + await attentionRetentionService(db).check(entry); + expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1); + }); + + it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => { + const root = await insertIssue("blocked"); + const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({ + companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress" + : index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning(); + const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2]; + const cycle = await insertIssue("blocked"); + const leaf = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + ...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })), + { companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" }, + { companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" }, + ]); + const owner = issueService(db), roots = [root, overlapping, cycle]; + const singles = new Map(); + for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id)); + expect(singles.get(root.id).blockingTreeLive).toBe(false); + expect(singles.get(overlapping.id).blockingTreeLive).toBe(true); + for (const input of [roots, [...roots].reverse()]) { + const batch = await owner.listBlockerAttention(companyId, input); + for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id)); + } + }); + + it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 70) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 10) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues branch cross join generate_series(1, 99) n + where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`); + const completed = await insertIssue("blocked"); + const shallow = await insertIssue("blocked", { updatedAt: new Date() }); + const middle = await insertIssue("todo", { updatedAt: new Date() }); + const overdue = await insertIssue("backlog", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values([ + { companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" }, + { companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" }, + ]); + const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked"))); + parameterCounts.length = 0; + const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => { + throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`); + }); + expect(result.attention.size).toBe(72); + expect(result.deferredRootIds.has(completed.id)).toBe(false); + expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id); + expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(result.deferredRootIds.has(shallow.id)).toBe(true); + const service = attentionRetentionService(db); + const page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!; + expect(immediate.dueAt).toBeLessThanOrEqual(Date.now()); + await service.check(immediate); + expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date); + }, 120_000); + + it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => { + const root = await insertIssue("blocked"); + const live = await insertIssue("in_progress", { assigneeAgentId: agentId }); + await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" }); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 3000) n`); + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await state("blocker_attention", root.id)).toBeNull(); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => { + const root = await insertIssue("blocked"); + await db.execute(sql`with blockers as ( + insert into issues (id, company_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from generate_series(1, 1999) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`); + const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId }); + if (kind === "run") { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id)); + } else if (kind === "confirmation") { + await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } }); + } + parameterCounts.length = 0; + const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]); + expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES); + expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2); + expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6); + expect(parameterCounts.length).toBeLessThanOrEqual(100); + expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100); + expect(batch.deferredRootIds.has(root.id)).toBe(true); + expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true); + const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" }); + const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!; + expect(await service.check(entry)).toBeNull(); + expect(await db.select().from(decisionRetention)).toHaveLength(0); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => { + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at) + select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz + from generate_series(1, 4) n`); + await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at) + select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz + from issues root cross join generate_series(1, 600) n + where root.company_id = ${companyId}::uuid and root.status = 'blocked'`); + await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep) + select company_id, 'blocker_attention', id::text, updated_at, true from issues + where company_id = ${companyId}::uuid and status = 'backlog'`); + const service = attentionRetentionService(db), check = vi.spyOn(service, "check"); + await start(service); await quiet(); + expect(check).toHaveBeenCalledTimes(4); + expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + }); + + it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => { + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" }); + await db.execute(sql`with parents as ( + insert into issues (id, company_id, title, status) + select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo' + from generate_series(1, 2100) n returning id) + insert into issue_relations (company_id, issue_id, related_issue_id, type) + select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + try { + await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` }); + } finally { release(); unsubscribe(); } + expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8); + expect(hints).toHaveLength(8); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })])); + await settle(); + expect(load.mock.calls.length).toBeLessThanOrEqual(3); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + let query!: Parameters[0], returned = 0; + await db.transaction(async tx => { + const execute = tx.execute.bind(tx); + vi.spyOn(tx, "execute").mockImplementation(async statement => { + const result = await execute(statement); + if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) { + query = statement; returned = result.length; + } + return result; + }); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id }); + }); + expect(returned).toBe(101); + const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`); + const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan; + expect(plan["Node Type"]).toBe("Limit"); + expect(plan["Actual Rows"]).toBe(101); + const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); }; + collect(plan); + const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union"); + const relations = nodes.filter(node => node["Relation Name"] === "issue_relations"); + expect(recursive).toHaveLength(1); + expect(recursive[0]["Actual Rows"]).toBe(101); + expect(relations.length).toBeGreaterThan(0); + expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true); + process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n"); + }); + + it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => { + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id) + select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n, + case when n = 150 then 'in_review' else 'todo' end, + case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end, + case when n = 150 then 'board' else null end + from generate_series(0, 150) n`); + await db.execute(sql`update issues child set parent_id = parent.id from issues parent + where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id + and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`); + const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]), + db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]); + await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" }); + await settle(); + expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true); + expect(await state("review", far.id)).toBeNull(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + await archived("review", far.id); + await quiet(); + }); + + it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => { + const a = await insertIssue(), b = await insertIssue(), c = await insertIssue(); + await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id)); + await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id)); + await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id)); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id })); + const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown }; + wide.ignored = wide; + await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); + } finally { unsubscribe(); } + expect(hints).toHaveLength(3); + expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })])); + }); + + it("canonicalizes wide circular notification variables to the supported identity fields", async () => { + const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = []; + const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown }; + wide.ignored = wide; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]); + }); + + it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => { + const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId }); + const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) }); + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" }); + const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning(); + await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true }))); + const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load"); + await start(service); await settle(); load.mockClear(); + let release!: () => void, entered = false; + const gate = new Promise(resolve => { release = resolve; }); + load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; }); + statements.length = 0; + await issueService(db).update(row.id, { title: "Held mutation" }); + await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 }); + await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` }))); + release(); await settle(); + for (let index = 0; index < 8; index++) { + await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle(); + } + await archived("review", urgent.id); + const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId); + expect(routine.length).toBeGreaterThan(0); + expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true); + const scans = statements.filter(query => query.includes("with source_candidates")); + expect(scans.length).toBeGreaterThan(0); + expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true); + expect(await state("review", row.id)).toBeNull(); + }); + + it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => { + const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" }); + const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning(); + const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning(); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning(); + await start(); await quiet(); + await issueService(db).update(origin.id, { hiddenAt: null }); + for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id); + await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + }); + + it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => { + const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" }); + const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" }); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear(); + await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle(); + const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : [])); + expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id])); + load.mockClear(); + await issueService(db).update(terminal.id, { hiddenAt: null }); await settle(); + expect(await state("blocker_attention", terminal.id)).toBeNull(); + const visible = await service.load({ companyId, issueId: newParent.id }); + expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true); + expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true); + const next = await insertIssue("backlog", { assigneeAgentId: agentId }); + await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id); + }); + + it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => { + const root = await insertIssue("blocked"); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) }); + await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull(); + await issueService(db).update(incident.id, { hiddenAt: new Date() }); + await archived("blocker_attention", terminal.id); await quiet(); + }); + + it("resolves opaque run issue references through trusted company issue rows", async () => { + const issue = await insertIssue(); + const [valid, opaque] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } }, + ]).returning(); + const hints: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => { + if (event === "settled") hints.push(...values ?? []); + }); + try { + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id })); + await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id })); + } finally { unsubscribe(); } + expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]); + }); + + it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => { + const source = await insertIssue("blocked", { responsibleUserId: "board" }); + const service = issueService(db); + const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board", + createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation, + originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` }); + await service.update(source.id, { blockedByIssueIds: [incident.id] }); + await service.update(incident.id, { status: "done" }); + expect((await service.getById(incident.id))?.status).toBe("done"); + expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]); + }); + + it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => { + const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" }); + const parent = await insertIssue("todo", { responsibleUserId: "board" }); + const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery, + originId: review.id, assigneeUserId: "board", responsibleUserId: "board" }); + await issueService(db).update(recovery.id, { parentId: parent.id }); + expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled"); + await start(); await quiet(); + await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" }); + await settle(); + expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date); + expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1); + }); + + it("scopes ordinary execution-control writes to their company", async () => { + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" }); + await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany, + title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + for (let count = 0; count < 3; count++) { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, + tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId }); + await quiet(); + } + expect(load).toHaveBeenCalled(); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => { + const service = budgetService(db); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100, + warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board"); + const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load"); + await start(retention); await quiet(); load.mockClear(); + const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", + billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20, + costCents: 90, occurredAt: new Date() }).returning(); + await service.evaluateCostEvent(event); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId)); + expect(incident).toMatchObject({ status: "open", thresholdType: "soft" }); + expect(await state("budget_alert", incident.id)).toBeNull(); + await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board"); + await quiet(); + expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true); + }); + + it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => { + const origin = await insertIssue(); + const review = await insertIssue("in_review", { assigneeUserId: "board" }); + const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() }); + await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning(); + await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" }); + await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, + title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }); + await db.insert(issueThreadInteractions).values([ + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() }, + { companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() }, + { companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }, + ]); + await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() }); + await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort(); + expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected); + expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`); + await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet(); + }); + + it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true }); + await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull(); + await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false }); + await archived("review", row.id); await quiet(); + await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet(); + }); + + it("moves deadlines on source activity and cancels when the source leaves attention", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() }); + await start(); await settle(); const first = coordinator!.nextWakeAt()!; + await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id))); + await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first); + await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull(); + }); + + it("preserves outer transaction ownership and first writes during drain", async () => { + await start(); await quiet(); draining = true; + let id: string | undefined; + await db.transaction(async outer => { + await outer.transaction(async nested => { + await notifyAttentionRetentionWork(nested, { companyId }); + id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id; + }); + await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull(); + }); + await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); }); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1); + }); + + it("isolates companies, catches up at activation, and bounds retained-history pages", async () => { + const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" }); + const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" }); + await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() }))); + const service = attentionRetentionService(db); const page = await service.load({ companyId }); + expect(page).toEqual({ rows: [], complete: true }); + await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0); + await companyService(db).update(other, { status: "active" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); }); + expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet(); + }); + it("preserves terminal blocker source identity and its own activity deadline", async () => { + const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() }); + const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) }); + await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const entries = await loadAll(); + const expected = feed.items.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort(); + expect(entries.filter(item => item.sourceKind === "blocker_attention") + .map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected); + expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`); + await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet(); + expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt); + expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt); + }); + + it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => { + const origin = await insertIssue(); + const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId }); + const hidden = await insertIssue("todo", { hiddenAt: old() }); + const [paused, manager, unhealthy] = await db.insert(agents).values([ + { companyId, name: "Paused", role: "engineer", status: "paused" }, + { companyId, name: "Terminated", role: "manager", status: "terminated" }, + { companyId, name: "Unhealthy", role: "engineer", status: "active" }, + ]).returning(); + await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id)); + const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() }; + const [older, newest, independent] = await db.insert(issueThreadInteractions).values([ + common, { ...common, createdAt: new Date(old().getTime() + 1000) }, + { ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } }, + { ...common, createdAt: new Date(), addresseeUserId: "board" }, + { ...common, addresseeAgentId: agentId }, + { ...common, issueId: hidden.id }, + { ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } }, + { ...common, issueId: conversation.id, addresseeAgentId: paused.id }, + { ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } }, + ] as typeof issueThreadInteractions.$inferInsert[]).returning(); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort(); + expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected); + expect(expected).not.toContain(`issue_thread_interaction:${older.id}`); + expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`); + await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet(); + await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id))); + await archived("issue_thread_interaction", older.id); await quiet(); + }); + + it("excludes maintained reviews until a pending approval creates human attention", async () => { + const row = await insertIssue("in_review", { assigneeAgentId: agentId }); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id, + kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning(); + expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false); + const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning(); + await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id }); + const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true }); + expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true); + await start(); await archived("review", row.id); await quiet(); + expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + }); + + it("reactivates an exhausted failed run through the ordinary durable event producer", async () => { + const origin = await insertIssue(); + const [run, newer] = await db.insert(heartbeatRuns).values([ + { companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }, + { companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() }, + ]).returning(); + await start(); await quiet(); + await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" }); + await quiet(); expect(await state("failed_run", run.id)).toBeNull(); + await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id))); + await archived("failed_run", run.id); await quiet(); + }); + + it("cancels an agent error deadline through ordinary clearError", async () => { + await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId)); + await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); + await createAgentLifecycle(db).clearError(agentId); await quiet(); + expect(await state("agent_error_alert", agentId)).toBeNull(); + }); + + it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => { + await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))); + const service = attentionRetentionService(db); + const load = vi.spyOn(service, "load"); + await start(service); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 }); + await quiet(); + const scopes = load.mock.calls.map(([scope]) => scope); + expect(scopes.filter(scope => !scope)).toHaveLength(3); + expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0); + expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225); + expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225); + }); + + it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => { + const row = await insertIssue("in_review", { assigneeUserId: "board" }); + await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`); + await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`); + await start(); expect(errors).toHaveLength(1); errors.length = 0; + expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0); + draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`); + statements.length = 0; await vi.advanceTimersByTimeAsync(120_000); + expect(statements.length).toBeLessThanOrEqual(1); + expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true); + statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet(); + await coordinator!.stop(); coordinator = undefined; + let release!: () => void; let scanned!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { scanned = resolve; }); + const service = attentionRetentionService(db), original = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; }); + const starting = start(service); await entered; + const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx)); + release(); await starting; await archived("review", next.id); await quiet(); + }); + + it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => { + const origin = await insertIssue(); await start(); await quiet(); + const service = issueThreadInteractionService(db); + const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" }); + await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull(); + await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet(); + }); + +}); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index a024ed421d7..5af78c2cb8b 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => { beforeAll(async () => { if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-"); - db = createDb(externalUrl ?? tempDb!.connectionString); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 30_000); afterEach(async () => { diff --git a/server/src/__tests__/budgets-history.test.ts b/server/src/__tests__/budgets-history.test.ts new file mode 100644 index 00000000000..5b106329d72 --- /dev/null +++ b/server/src/__tests__/budgets-history.test.ts @@ -0,0 +1,109 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; +import { budgetService } from "../services/budgets.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() })); +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("budget history for deleted scopes", () => { + let db: ReturnType; + let database: Awaited>; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-"); + db = createDb(database.connectionString); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterEach(async () => { + await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents); + await db.delete(projects); await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning(); + await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() }); + return { companyId, agentId, policy: policy! }; + } + + it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => { + const { companyId, policy } = await fixture(), deletedId = randomUUID(); + if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" }); + else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" }); + const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning(); + await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }); + if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId)); + else await db.delete(projects).where(eq(projects.id, deletedId)); + const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents); + + const overview = await budgetService(db).overview(companyId); + expect(overview.policies).toHaveLength(2); + expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 }); + expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 }); + expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]); + expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore); + expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore); + expect(await db.select().from(costEvents)).toEqual(costsBefore); + }); + it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => { + const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db); + await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found"); + await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found"); + for (const scopeType of ["agent", "project"] as const) { + await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId, + windowKind: "calendar_month_utc", amount: 100 }).returning(); + const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning(); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }; + await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board")) + .rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before); + const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board"); + expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" }); + const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!; + const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id)); + expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) }); + expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" }); + expect(await db.select().from(budgetPolicies)).toEqual(before.policies); + expect(await db.select().from(agents)).toEqual(before.agents); + expect(await db.select().from(costEvents)).toEqual(before.costs); + expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false); + } + }); + + it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => { + const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID(); + await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` }); + await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" }); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"), + thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning(); + const service = budgetService(db); + await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found"); + for (const scopeId of [foreignAgentId, randomUUID()]) { + await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id)); + const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }; + for (const action of ["raise_budget_and_resume", "keep_paused"] as const) { + await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found"); + expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents), + agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before); + } + } + }); + +}); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index fcaf642aa84..8b10fe2eec7 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref, versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id)); }); - await expect.poll(async () => ((await f.checkpoint()).value as Record).credentialFingerprint, - { timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint); - expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" }); + // Credential synchronization and epoch initialization commit separately. + // Require the final epoch receipt, not just the intermediate fingerprint. + await expect.poll(async () => { + const value = (await f.checkpoint()).value as Record; + return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string" + && value.credentialFingerprint !== oldWindow.credentialFingerprint }; + }, { timeout: 5000 }).toMatchObject({ + fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized", + }); expect(f.requests).toEqual([]); } finally { await coordinator.stop(); } }); diff --git a/server/src/__tests__/company-portability.test.ts b/server/src/__tests__/company-portability.test.ts index fdf00b828af..9b7cfb3a2a9 100644 --- a/server/src/__tests__/company-portability.test.ts +++ b/server/src/__tests__/company-portability.test.ts @@ -4168,6 +4168,7 @@ describe("company portability", () => { const insertedRelationValues: Array> = []; const monitorUpdates: Array> = []; const db = { + transaction: async (write: (tx: unknown) => Promise) => write(db), insert: () => ({ values: (rows: Array>) => ({ onConflictDoNothing: async () => { @@ -4184,6 +4185,7 @@ describe("company portability", () => { }), // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; + db.transaction = async (write: (tx: typeof db) => Promise) => write(db); return { db, insertedRelationValues, monitorUpdates }; } diff --git a/server/src/__tests__/company-search-service.test.ts b/server/src/__tests__/company-search-service.test.ts index 5d9e53260fb..7dffe3d6a90 100644 --- a/server/src/__tests__/company-search-service.test.ts +++ b/server/src/__tests__/company-search-service.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -28,12 +29,13 @@ import { import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js"; import { issueService } from "../services/issues.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { console.warn( - `Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`, + `Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`, ); } @@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + // External suites can reuse this dedicated database sequentially. Match + // the fresh embedded fixture before this suite creates its own rows. + if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`); svc = companySearchService(db); await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm")); }, 20_000); @@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/decision-effect-recovery-work.test.ts b/server/src/__tests__/decision-effect-recovery-work.test.ts index 6c1a17d2b39..8a7d4e300c8 100644 --- a/server/src/__tests__/decision-effect-recovery-work.test.ts +++ b/server/src/__tests__/decision-effect-recovery-work.test.ts @@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId, updatedAt, inputValues: {}, metadata: { continuationPending: true }, - }).where(eq(decisions.id, id))); + }).where(eq(decisions.id, id)), { companyId }); async function start() { coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) }); @@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes } else { await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running", chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000), - metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id))); + metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId }); } await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet(); const [retained] = await db.select().from(decisionRetention); diff --git a/server/src/__tests__/decisions-service.test.ts b/server/src/__tests__/decisions-service.test.ts index 7fc2cc50c0d..999bde8d60e 100644 --- a/server/src/__tests__/decisions-service.test.ts +++ b/server/src/__tests__/decisions-service.test.ts @@ -793,7 +793,7 @@ describePg("decisionService", () => { expect((await service().get(id))?.metadata.continuationPending).toBe(false); } const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions) - .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id))); + .set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId }); it("coalesces sequential tenant writes without dropping pending recovery", async () => { const row = await createCommentDecision(); diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts new file mode 100644 index 00000000000..914f33fe609 --- /dev/null +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -0,0 +1,427 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; +import { accessService } from "../services/access.js"; +import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js"; +import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js"; +import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; +import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; +import { issueTreeControlService } from "../services/issue-tree-control.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js"; +import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { requestTaskRetry } from "../services/task-retry.js"; +import { issueService } from "../services/issues.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +vi.setConfig({ testTimeout: 90_000 }); +describe("deferred receipt durable dispatch", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType; + let drain = false; + const errors: unknown[] = []; + const passes = new Set>(); + beforeAll(async () => { + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + }); + beforeEach(async () => { + await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0; + heartbeat = heartbeatService(db); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, + onError: error => errors.push(error) }); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = Promise.resolve(task.run(signal)); passes.add(pass); + void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass; + } }); + }); + afterEach(async () => { + await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); + vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function fixture() { + const companyId = randomUUID(), actorId = "deferred-receipt-user"; + await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`, + createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`, + defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false }); + await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle", + adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning(); + const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning(); + const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id, + status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed", + contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning(); + return { companyId, agent, issue, source, actorId }; + } + async function quiet() { + await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 }); + const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute"); + const selected = reads.mock.calls.length, executed = executes.mock.calls.length; + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); + } + async function finish(issueId: string) { + await issueService(db).update(issueId, { status: "done" }); + await vi.waitFor(async () => { + await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + }, { timeout: 15_000 }); + const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry")); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (saved.length) { + const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); + while (Date.now() <= due) { + await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1)); + await Promise.allSettled([...passes]); await immediate(); + } + } + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0); + }); + } + it("is quiet after empty reconstruction and terminal receipt history", async () => { + const f = await fixture(); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet(); + }); + it("dispatches an ordinary committed task Retry into saved process output", async () => { + const f = await fixture(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + expect(receipt.accepted).toBe(true); + let runId: string | undefined; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!; + }, { timeout: 15_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + await finish(f.issue.id); + const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`)); + expect(linked).toHaveLength(2); + expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId])); + expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`); + expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced"); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1); + await quiet(); + }); + it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => { + const f = await fixture(), updatedAt = new Date(), native = kind === "native claim"; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt, + payload: native ? {} : { executionWait: { reason: "recovery" } }, + requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => { + await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!)); + }); + await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check, + schedulingSuppressed: async () => false }, () => !drain).ready; + const due = updatedAt.getTime() + (native ? 60_000 : 30_000); + expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }))); + await quiet(); + }); + it("reconciles the first drained producer and waits for resume without domain SQL", async () => { + const f = await fixture(); drain = true; + const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative, + schedulingSuppressed: async () => false }, () => !drain).ready; + await db.transaction(async tx => { + await notifyDeferredExecutionWork(tx, { companyId: f.companyId }); + await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand", + status: "deferred_issue_execution", reason: "task_retry" }); + }); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + await vi.advanceTimersByTimeAsync(1000); + expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + drain = false; await vi.advanceTimersByTimeAsync(30_000); + await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet(); + }); + it.each([ + { name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } }, + { name: "interaction", payload: { mutation: "interaction" } }, + { name: "execution wait", payload: { executionWait: { reason: "recovery" } } }, + { name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } }, + ])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => { + const f = await fixture(), foreign = await fixture(); + const resume = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(), + schedulingSuppressed: async () => false }, () => !drain).ready; + const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload }).returning(); + let requestId: string; + await db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyDeferredExecutionWork(nested, { companyId: f.companyId }); + const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning(); + requestId = row.id; + }); + expect(resume).not.toHaveBeenCalled(); + }); + await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution"); + await quiet(); + }); + it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => { + const f = await fixture(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution" }).returning(); + await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } }) + .where(eq(heartbeatRuns.id, f.source.id)); + const resume = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(), + schedulingSuppressed: async () => false }, () => !drain).ready; + expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })); + await quiet(); + }); + it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => { + const f = await fixture(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, + uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning(); + const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id, + provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning(); + const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000); + await db.insert(agentWakeupRequests).values([ + { id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry", + status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id, + taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id, + actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } }, + { id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution", + payload: { issueId: f.issue.id, taskRetryRequestId: intentId } }, + ]); + await db.transaction(async tx => { + await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId }); + await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id, + providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(), + payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } }); + }); + await vi.waitFor(async () => { + const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId)); + expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime()); + }, { timeout: 5_000 }); + expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed"); + await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId })); + await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status) + .toBe("issued"), { timeout: 5_000 }); + await quiet(); + }); + it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => { + const f = await fixture(), initialAt = new Date(); + const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({ + companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId, + }))).returning(); + const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId }, + _paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } }; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps); + await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope), + settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready; + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now()); + const movedAt = new Date(); + const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {}, + deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} }); + await writer.withLockedQueue({ issue: f.issue, queueId: wake.id, + actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } }, + async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload, + ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); }); + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000); + await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id)); + await quiet(); + }); + it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => { + const f = await fixture(), tree = issueTreeControlService(db); + const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId }; + const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + expect(request.requestId).toBeTruthy(); + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" }); + expect(wake.runId).toBeNull(); + }, { timeout: 5_000 }); + await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 }); + await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" }); + let runId: string | null = null; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId; + }, { timeout: 40_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1); + await finish(f.issue.id); await quiet(); + }); + it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => { + const f = await fixture(), initialAt = new Date(); + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt, + requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {}, + dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready; + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now()); + const movedAt = new Date(); + await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id)); + await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000); + await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now()); + await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet(); + }); + it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => { + const f = await fixture(); let reason = "operator_pause"; + const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => { + await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason }); + }); + await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {}, + schedulingSuppressed: async () => false }, () => !drain).ready; + const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + for (const nextReason of ["operator_pause", "budget_pause"]) { + reason = nextReason; + await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason }); + const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); + const due = current.updatedAt.getTime() + 30_000; + await vi.advanceTimersByTimeAsync(1_000); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due)); + const checked = resume.mock.calls.length; + await vi.advanceTimersByTimeAsync(due - Date.now()); + await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1)); + await quiet(); + } + expect(resume).toHaveBeenCalledTimes(2); + }); + it("reactivates a saved Retry through ordinary restore cancelled task", async () => { + const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId }; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + const tree = issueTreeControlService(db); + const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor }); + await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id); + const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor }); + const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor }); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull(); + await commitEligibility(); + let runId: string | null = null; + await vi.waitFor(async () => { + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)); + expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId; + }, { timeout: 15_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1); + await finish(f.issue.id); await quiet(); + }); + it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => { + const f = await fixture(), departing = `departing-${randomUUID()}`; + const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`, + createdAt: new Date(), updatedAt: new Date() }); + const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing, + principalType: "user", status: "active", membershipRole: "member" }).returning(); + await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null }); + const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0]; + expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null }); + const deferred = vi.fn(), native = vi.fn(); + const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred), + subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)]; + try { + await expect(db.transaction(async tx => { + await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id, + { reassignment: { assigneeAgentId: f.agent.id } }); + expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled(); + throw new Error("roll back member archive"); + })).rejects.toThrow("roll back member archive"); + expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts); + await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents); + deferred.mockClear(); native.mockClear(); + await db.transaction(async tx => { + await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id, + { reassignment: { assigneeAgentId: f.agent.id } }); + expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled(); + }); + expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1); + expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({ + assigneeUserId: null, assigneeAgentId: f.agent.id, + }); + expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived"); + expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null }); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0); + } finally { for (const unsubscribe of unsubscribes) unsubscribe(); } + await quiet(); + }); + it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => { + const f = await fixture(); const dispatchNative = vi.fn(async () => {}); + await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative, + schedulingSuppressed: async () => false }, () => !drain).ready; + await expect(db.transaction(async tx => { + await notifyNativeStatusWakeWork(tx, { companyId: f.companyId }); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + await db.transaction(async tx => { + await expect(tx.transaction(async nested => { + await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback"); + })).rejects.toThrow("nested rollback"); + }); + await quiet(); expect(dispatchNative).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/__tests__/execution-control-work.test.ts b/server/src/__tests__/execution-control-work.test.ts index 0046d8574aa..14cd9456eb5 100644 --- a/server/src/__tests__/execution-control-work.test.ts +++ b/server/src/__tests__/execution-control-work.test.ts @@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => { const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000); for (const at of [due, earlier]) { await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await settle(); expect(c.nextWakeAt()).toBe(at.getTime()); } await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns) - .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id))); + .set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId }); await quiet(c); }); it("delivers a reconciled continuation when marking and resolving commit together", async () => { diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts new file mode 100644 index 00000000000..8aab42b35b1 --- /dev/null +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -0,0 +1,506 @@ +import { createServer } from "node:http"; +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, approvals, applyPendingMigrations, databaseWorkPending, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, externalObjectMentions, issues, issueComments, plugins } from "@paperclipai/db"; +import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { documentService } from "../services/documents.js"; +import { budgetService } from "../services/budgets.js"; +import { pluginRegistryService } from "../services/plugin-registry.js"; +import { companyService } from "../services/companies.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; + +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECT_REFRESH_TEST_DATABASE_URL"); +const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const terminal: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "done", statusCategory: "succeeded", statusTone: "success", isTerminal: true, +} }; +const active: ExternalObjectResolveResult = { ok: true, snapshot: { + statusKey: "open", statusCategory: "open", statusTone: "neutral", isTerminal: false, ttlSeconds: 60, +} }; + +(support.supported ? describe : describe.skip)("external object refresh work (real PostgreSQL, simulated provider)", () => { + let db: ReturnType; + let temp: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let admitted = true; + let reconcile = true; + let enabled = true; + const provider = vi.fn(async (): Promise => terminal); + const errors = vi.fn(); + const companyIds: string[] = []; + beforeAll(async () => { + if (url) await applyPendingMigrations(url); + if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-"); + db = createDb(url ?? temp!.connectionString, { maxConnections: 6 }); + await db.execute(sql`truncate table ${companies} cascade`); + }, 90_000); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + for (const id of companyIds.splice(0)) { + await db.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id)); + await db.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id)); + await db.delete(approvals).where(eq(approvals.companyId, id)); + await db.delete(costEvents).where(eq(costEvents.companyId, id)); + await db.delete(activityLog).where(eq(activityLog.companyId, id)); + await db.delete(issues).where(eq(issues.companyId, id)); + await db.delete(externalObjects).where(eq(externalObjects.companyId, id)); + await db.delete(companies).where(eq(companies.id, id)); + } + await db.delete(plugins); + admitted = true; reconcile = true; enabled = true; errors.mockClear(); + provider.mockReset().mockResolvedValue(terminal); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temp?.cleanup(); }); + + function service() { + return externalObjectService(db, { github: false, enabled: () => enabled, + resolvers: [{ providerKey: "url", resolve: provider }] }); + } + async function start(svc = service()) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, + canReconcile: () => reconcile, onError: errors }); + const worker = svc.registerRefreshWork(coordinator, () => admitted); + await worker.ready; + return worker; + } + async function fixture(status = "active") { + const id = randomUUID(); companyIds.push(id); + await db.insert(companies).values({ id, name: "Refresh test", status, + issuePrefix: `E${id.replaceAll("-", "").slice(0, 8)}` }); + const [issue] = await db.insert(issues).values({ companyId: id, title: "Reference", + description: `https://example.com/${id}`, status: "todo" }).returning(); + return { companyId: id, issueId: issue!.id }; + } + async function row(companyId: string) { + return (await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)))[0]!; + } + async function settle(predicate: () => Promise | boolean) { + for (let i = 0; i < 200; i++) { + await vi.advanceTimersByTimeAsync(0); await delay(5); + if (await predicate()) return; + } + throw new Error("refresh worker did not settle"); + } + async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); } + function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); } + async function change(companyId: string, patch: Partial) { + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, companyId); + await tx.update(externalObjects).set(patch).where(eq(externalObjects.companyId, companyId)); + }); + } + + it("activates saved work when the ordinary feature and company settings enable it", async () => { + clock(); const settings = instanceSettingsService(db); + await settings.updateExperimental({ enableExternalObjects: true }); + const svc = externalObjectService(db, { github: false, + enabled: async () => (await settings.getExperimental()).enableExternalObjects, + resolvers: [{ providerKey: "url", resolve: provider }] }); + const f = await fixture("paused"); await svc.syncIssue(f.issueId); + await settings.updateExperimental({ enableExternalObjects: false }); + await start(svc); await quiet(); + await companyService(db).update(f.companyId, { status: "active" }); await quiet(); + expect(provider).not.toHaveBeenCalled(); + await settings.updateExperimental({ enableExternalObjects: true }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("settles empty, configured-idle and retained terminal history without recurring SQL or provider calls", async () => { + clock(); + const svc = service(); await start(svc); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.isTerminal === true); await quiet(); + expect((await row(f.companyId)).statusKey).toBe("done"); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(15 * 60_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + expect(errors).not.toHaveBeenCalled(); + }); + + it("registers exact future deadlines, preempts them on earlier writes and cancels terminal work", async () => { + clock(); provider.mockResolvedValue(active); + const svc = service(); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + const earlier = new Date(Date.now() + 1000); await change(f.companyId, { nextRefreshAt: earlier }); + await settle(() => coordinator!.nextWakeAt() === earlier.getTime()); + await vi.advanceTimersByTimeAsync(999); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue(terminal); await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + await vi.advanceTimersByTimeAsync(120_000); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("keeps disabled and paused company work passive and reconstructs on eligibility mutation", async () => { + clock(); enabled = false; + const svc = service(); await start(svc); await quiet(); + const f = await fixture("paused"); + enabled = true; await svc.syncIssue(f.issueId); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).not.toHaveBeenCalled(); + await db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, f.companyId); + await tx.update(companies).set({ status: "active" }).where(eq(companies.id, f.companyId)); + }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("waits for the outer commit and leaves rolled-back ingestion quiet", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + const f = await fixture(); + await expect(db.transaction(async tx => { + await svc.syncIssue(f.issueId, tx); throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect(await row(f.companyId)).toBeUndefined(); expect(provider).not.toHaveBeenCalled(); + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + let nestedDone = false; + const commit = db.transaction(async tx => { + await tx.transaction(async nested => { await svc.syncIssue(f.issueId, nested); }); + nestedDone = true; await held; + }); + await settle(() => nestedDone); await vi.advanceTimersByTimeAsync(60_000); + expect(provider).not.toHaveBeenCalled(); release(); await commit; + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("retains first work during drain and recovers a provider failure after restart", async () => { + clock(); const svc = service(); await start(svc); await quiet(); + admitted = false; const f = await fixture(); await svc.syncIssue(f.issueId); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(90_000); expect(provider).not.toHaveBeenCalled(); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); expect(selects).not.toHaveBeenCalled(); + admitted = true; provider.mockRejectedValueOnce(new Error("simulated transport failure")); + await vi.advanceTimersByTimeAsync(30_000); await settle(() => errors.mock.calls.length === 1); + const claimed = await row(f.companyId); expect(claimed.refreshToken).not.toBeNull(); + await coordinator!.stop(); coordinator = undefined; await start(svc); + const due = claimed.refreshStartedAt!.getTime() + 300_000; + expect(coordinator!.nextWakeAt()).toBe(due); + await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("leaves unsupported URLs passive and cancels orphan work until ordinary reattachment", async () => { + clock(); const unsupported = externalObjectService(db, { github: false }); + await start(unsupported); const f = await fixture(); await unsupported.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); + expect((await row(f.companyId)).lastResolvedAt).toBeNull(); + await coordinator!.stop(); coordinator = undefined; + provider.mockResolvedValue(active); const svc = service(); await start(svc); + await settle(async () => (await row(f.companyId)).lastResolvedAt !== null); + await db.update(issues).set({ description: null }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await quiet(); + selects.mockClear(); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await db.update(issues).set({ description: `https://example.com/${f.companyId}` }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("activates a newly ready plugin and fences a result after company permission is withdrawn", async () => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.refresh-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Refresh fixture", + description: "Simulated external object provider", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "tracker", displayName: "Tracker", objectTypes: ["ticket"], urlPatterns: ["https://example.com/:id"] }], + }; + const plugin = await registry.install({ packageName: "refresh-fixture" }, manifest); + let finish!: (result: ExternalObjectResolveResult) => void; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + if (calls.mock.calls.filter(call => call[1] === "resolveExternalObject").length === 1) + return new Promise(resolve => { finish = resolve; }); + return terminal; + }); + const svc = externalObjectService(db, { github: false, + pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + detectors: [{ key: "fixture", detect: ({ urls }) => urls.map(canonical => ({ canonical, + detectorKey: "fixture", providerKey: "tracker", objectType: "ticket", externalId: canonical.canonicalIdentityHash })) }], + }); + await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); await quiet(); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + await registry.updateStatus(plugin!.id, { status: "ready" }); + await settle(() => typeof finish === "function"); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + finish(terminal); + await settle(async () => (await row(f.companyId)).refreshToken === null); await quiet(); + expect((await row(f.companyId)).isTerminal).toBe(false); + selects.mockClear(); calls.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled(); + calls.mockImplementation(async () => terminal); + await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: true, settingsJson: {} }); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(calls).toHaveBeenCalledOnce(); + }); + + it.each(["company setting", "status", "uninstall", "manifest", "other company"] as const)( + "revalidates the selected plugin after %s changes even while the built-in fallback remains available", async change => { + clock(); const registry = pluginRegistryService(db); + const manifest: PaperclipPluginManifestV1 = { + id: `paperclip.selected-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Selected provider", + description: "Selected resolver fixture", author: "Paperclip", categories: ["connector"], + capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" }, + objectReferences: [{ providerKey: "github", displayName: "GitHub", objectTypes: ["pull_request"], urlPatterns: ["https://github.com/:owner/:repo/pull/:id"] }], + }; + const plugin = await registry.install({ packageName: "selected-provider-fixture" }, manifest); + await registry.updateStatus(plugin!.id, { status: "ready" }); + let finishPlugin!: (result: ExternalObjectResolveResult) => void; + const pluginResult = new Promise(resolve => { finishPlugin = resolve; }); + let selected = false; + const calls = vi.fn(async (_id: string, method: string) => { + if (method === "detectExternalObjects") return { detections: [] }; + selected = true; return pluginResult; + }); + let finishFallback!: () => void; + const fallbackResult = new Promise(resolve => { finishFallback = resolve; }); + const fallback = vi.fn(async () => { + await fallbackResult; + return new Response(JSON.stringify({ state: "closed", merged: true, title: "Allowed fallback" }), { status: 200 }); + }); + const svc = externalObjectService(db, { pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager, + github: { fetch: fallback, credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }) } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => selected); + if (change === "company setting") await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} }); + else if (change === "status") await registry.updateStatus(plugin!.id, { status: "error" }); + else if (change === "uninstall") await registry.uninstall(plugin!.id, true); + else if (change === "manifest") await registry.update(plugin!.id, { manifest: { ...manifest, capabilities: [], objectReferences: [] } }); + else { + const other = await fixture(); + await registry.upsertCompanySettings(plugin!.id, other.companyId, { enabled: false, settingsJson: {} }); + } + finishPlugin({ ok: true, snapshot: { statusKey: "plugin_result", statusCategory: "succeeded", statusTone: "success", isTerminal: true } }); + if (change === "other company") { + await settle(async () => (await row(f.companyId))?.statusKey === "plugin_result"); await quiet(); + expect(fallback).not.toHaveBeenCalled(); + } else { + await settle(async () => fallback.mock.calls.length > 0 || Boolean((await row(f.companyId))?.lastResolvedAt)); + const beforeFallback = await row(f.companyId); + expect(beforeFallback.isTerminal).toBe(false); expect(beforeFallback.statusKey).toBeNull(); expect(beforeFallback.lastResolvedAt).toBeNull(); + expect(fallback).toHaveBeenCalledOnce(); + finishFallback(); + await settle(async () => (await row(f.companyId))?.statusKey === "merged"); await quiet(); + expect((await row(f.companyId)).displayTitle).toBe("acme/app#42: Allowed fallback"); + } + } finally { finishPlugin(terminal); finishFallback(); } + }); + + it("reactivates refresh after ordinary budget pause and budget raise", async () => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastResolvedAt !== null); + await db.insert(costEvents).values({ companyId: f.companyId, usageKind: "decision", provider: "fixture", + biller: "fixture", billingType: "metered_api", model: "simulated", costCents: 200, occurredAt: new Date() }); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, + amount: 100, hardStopEnabled: true }, null); + expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0]!.status).toBe("paused"); + await quiet(); + const reads = vi.spyOn(db, "select"); provider.mockClear(); + await vi.advanceTimersByTimeAsync(900_000); expect(reads).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + provider.mockResolvedValue(terminal); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 300 }, null); + await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it.each(["comment", "document"] as const)("keeps hard-deleted %s activation scoped to its retained mention owner", async kind => { + clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc); + const a = await fixture(); const b = await fixture(); await svc.syncIssue(b.issueId); + let id: string; + if (kind === "comment") { + const [comment] = await db.insert(issueComments).values({ companyId: a.companyId, issueId: a.issueId, + body: `https://example.com/${a.companyId}` }).returning(); + id = comment!.id; await svc.syncComment(id); + } else { + const result = await documentService(db).upsertIssueDocument({ issueId: a.issueId, + key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` }); + id = result.document.id; await svc.syncDocument(id); + } + await settle(async () => { + const left = await row(a.companyId), right = await row(b.companyId); + return Boolean(left?.lastResolvedAt && right?.lastResolvedAt); + }); + const expectedDeadline = Math.min((await row(a.companyId)).nextRefreshAt!.getTime(), (await row(b.companyId)).nextRefreshAt!.getTime()); + await settle(() => coordinator!.nextWakeAt() === expectedDeadline && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + if (kind === "comment") { await db.delete(issueComments).where(eq(issueComments.id, id)); await svc.syncComment(id); } + else { await documentService(db).deleteIssueDocument(a.issueId, "notes"); await svc.syncDocument(id); } + await settle(() => reads.length > 0); + for (const read of reads) { + expect(read.query).toContain('"external_objects"."company_id" ='); + expect(read.params).toContain(a.companyId); expect(read.params).not.toContain(b.companyId); + } + } finally { db.$client.options.debug = debug; } + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => { + clock(); admitted = false; reconcile = false; + const svc = service(); const selects = vi.spyOn(db, "select"); + await start(svc); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); + const f = await fixture(); await svc.syncIssue(f.issueId); + selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled(); + admitted = true; reconcile = true; + await vi.advanceTimersByTimeAsync(30_000); + await settle(async () => (await row(f.companyId))?.isTerminal); await quiet(); + expect(provider).toHaveBeenCalledOnce(); + }); + + it("renews an active provider lease so another service cannot duplicate its action", async () => { + clock(); const svc = service(); await start(svc); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(() => provider.mock.calls.length === 1); + const claimed = await row(f.companyId); + await vi.advanceTimersByTimeAsync(60_000); + await settle(async () => (await row(f.companyId)).refreshStartedAt!.getTime() > claimed.refreshStartedAt!.getTime()); + const competing = await service().refreshObject(claimed.id, { companyId: f.companyId, + force: true, now: new Date(claimed.refreshStartedAt!.getTime() + 301_000) }); + expect(competing.reason).toBe("refresh_in_progress"); expect(provider).toHaveBeenCalledOnce(); + finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet(); + }); + + it("uses durable provider retry deadlines and cancels the observed result on shutdown without overlap", async () => { + clock(); const svc = service(); await start(svc); + provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 }); + const f = await fixture(); await svc.syncIssue(f.issueId); + await settle(async () => (await row(f.companyId))?.lastErrorCode === "slow_down"); + const saved = await row(f.companyId); + await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime()); + expect(saved.refreshToken).toBeNull(); + await vi.advanceTimersByTimeAsync(saved.nextRefreshAt!.getTime() - Date.now() - 1); + expect(provider).toHaveBeenCalledOnce(); + let finish!: (result: ExternalObjectResolveResult) => void; + provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; })); + await vi.advanceTimersByTimeAsync(1); await settle(() => provider.mock.calls.length === 2); + await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0); + expect(provider).toHaveBeenCalledTimes(2); + let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; }); + await stop; expect(stopped).toBe(true); coordinator = undefined; + const cancelled = await row(f.companyId); expect(cancelled.refreshToken).toBeNull(); expect(cancelled.isTerminal).toBe(false); + finish(terminal); await delay(10); + expect((await row(f.companyId)).isTerminal).toBe(false); + const selects = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(600_000); + expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2); + }); + + it("cancels an actual stalled GitHub HTTP body, retains due debt and resumes after restart", async () => { + clock(); let stall = true, started = false, closed = false; + const server = createServer((request, response) => { + if (request.url?.includes("/pulls/")) { + if (stall) { + started = true; response.on("close", () => { closed = true; }); + response.writeHead(200, { "content-type": "application/json" }); response.write('{"state":'); + } else response.end(JSON.stringify({ state: "closed", merged: true, title: "Done" })); + } else response.end(JSON.stringify({ private: false })); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing HTTP test port"); + const svc = externalObjectService(db, { github: { + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }), + } }); + try { + await start(svc); const f = await fixture(); + await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId)); + await svc.syncIssue(f.issueId); await settle(() => started); + await coordinator!.stop(); coordinator = undefined; + await settle(() => closed); + const cancelled = await row(f.companyId); + expect(cancelled.refreshToken).toBeNull(); expect(cancelled.lastResolvedAt).toBeNull(); + expect(cancelled.nextRefreshAt!.getTime()).toBeLessThanOrEqual(Date.now()); + stall = false; await start(svc); + await settle(async () => Boolean((await row(f.companyId))?.isTerminal)); await quiet(); + } finally { await coordinator?.stop(); coordinator = undefined; server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); } + }); + + it("loads only the changed identity and objects after ingestion into ten thousand future objects", async () => { + clock(); const svc = service(); const f = await fixture(); + const future = new Date(Date.now() + 86_400_000); + const ids = Array.from({ length: 10_000 }, () => randomUUID()); + for (let offset = 0; offset < ids.length; offset += 500) { + const batch = ids.slice(offset, offset + 500); + await db.insert(externalObjects).values(batch.map(id => ({ id, companyId: f.companyId, + providerKey: "url", objectType: "link", externalId: id, nextRefreshAt: future }))); + await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId, + sourceIssueId: f.issueId, sourceKind: "description" as const, objectId }))); + } + await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime()); + const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming", + description: "https://example.com/new-object" }).returning(); + const reads: Array<{ query: string; params: unknown[] }> = []; + const debug = db.$client.options.debug; + db.$client.options.debug = (_connection, query, params) => { + if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"')) + reads.push({ query, params: [...params] }); + }; + try { + await svc.syncIssue(incoming!.id); + await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime() && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh)); + expect(reads.length).toBeGreaterThan(0); expect(reads.length).toBeLessThanOrEqual(6); + for (const read of reads) expect(read.query).toMatch(/"external_objects"\."(?:id|external_id)" =/); + } finally { db.$client.options.debug = debug; } + }); + + it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => { + clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused"); + for (const companyId of [b.companyId, inactive.companyId]) { + await db.insert(externalObjects).values(Array.from({ length: 255 }, (_, index) => ({ + companyId, providerKey: "url", objectType: "link", externalId: String(index), + sanitizedCanonicalUrl: `https://example.com/${companyId}/${index}`, canonicalIdentityHash: `${companyId}-${index}`, + nextRefreshAt: null, + }))); + } + await db.update(issues).set({ description: Array.from({ length: 255 }, (_, index) => + `https://example.com/${a.companyId}/${index}`).join(" ") }).where(eq(issues.id, a.issueId)); + await svc.syncIssue(a.issueId); + const future = new Date(Date.now() + 60_000); + await db.update(externalObjects).set({ nextRefreshAt: future }).where(eq(externalObjects.companyId, a.companyId)); + await svc.syncIssue(b.issueId); + await start(svc); await settle(() => provider.mock.calls.length === 1); + await settle(() => coordinator!.nextWakeAt() === future.getTime()); + expect((await row(a.companyId)).nextRefreshAt).toEqual(future); + const companiesSeen = provider.mock.calls.map(call => (call as unknown as [{ companyId: string }])[0].companyId); + expect(companiesSeen).toEqual([b.companyId]); + await change(a.companyId, { isTerminal: true }); await quiet(); + await vi.advanceTimersByTimeAsync(600_000); expect(provider).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/external-objects-service.test.ts b/server/src/__tests__/external-objects-service.test.ts index e9022599779..f0c6480a027 100644 --- a/server/src/__tests__/external-objects-service.test.ts +++ b/server/src/__tests__/external-objects-service.test.ts @@ -1,8 +1,11 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; import { companies, + applyPendingMigrations, createDb, activityLog, externalObjectMentions, @@ -11,6 +14,7 @@ import { issues, plugins, } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -26,7 +30,8 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -709,9 +714,10 @@ describeEmbeddedPostgres("externalObjectService", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); - db = createDb(tempDb.connectionString); - }, 20_000); + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); + }, 90_000); afterEach(async () => { await db.delete(activityLog); @@ -725,9 +731,22 @@ describeEmbeddedPostgres("externalObjectService", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); + async function runRefreshWork(service: ReturnType) { + const errors: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, + onError: error => errors.push(error) }); + try { + await service.registerRefreshWork(coordinator, () => true).ready; + expect(errors).toEqual([]); + } finally { + await coordinator.stop(); + } + } + async function createIssue(companyId = randomUUID()) { const issueId = randomUUID(); await db.insert(companies).values({ @@ -877,9 +896,9 @@ describeEmbeddedPostgres("externalObjectService", () => { expect(object.nextRefreshAt).toBeInstanceOf(Date); - const refreshed = await svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + await runRefreshWork(svc); - expect(refreshed).toHaveLength(1); + expect((await db.select().from(externalObjects))[0]?.lastResolvedAt).toBeInstanceOf(Date); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -916,15 +935,13 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(svc); await resolverStarted; const manualRefresh = svc.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const [dueResults, manualResult] = await Promise.all([dueRefresh, manualRefresh]); + const [, manualResult] = await Promise.all([dueRefresh, manualRefresh]); - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualResult.refreshed).toBe(true); expect(manualResult.object.statusLabel).toBe("Open"); expect(resolve).toHaveBeenCalledTimes(1); @@ -964,14 +981,12 @@ describeEmbeddedPostgres("externalObjectService", () => { await scheduledService.syncIssue(issueId); const object = await db.select().from(externalObjects).then((rows) => rows[0]!); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000)); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const manualRefresh = await manualService.refreshObject(object.id, { companyId, force: true }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; - expect(dueResults).toHaveLength(1); - expect(dueResults[0]?.refreshed).toBe(true); expect(manualRefresh.refreshed).toBe(false); expect(manualRefresh.reason).toBe("refresh_in_progress"); expect(manualRefresh.object).not.toHaveProperty("refreshToken"); @@ -1027,7 +1042,7 @@ describeEmbeddedPostgres("externalObjectService", () => { const object = await db.select().from(externalObjects).then((rows) => rows[0]!); const claimedAt = new Date(Date.now() + 1_000); - const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, claimedAt); + const dueRefresh = runRefreshWork(scheduledService); await resolverStarted; const replacementResult = await replacementService.refreshObject(object.id, { companyId, @@ -1035,11 +1050,10 @@ describeEmbeddedPostgres("externalObjectService", () => { now: new Date(claimedAt.getTime() + 301_000), }); finishResolver(); - const dueResults = await dueRefresh; + await dueRefresh; const finalObject = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(replacementResult.refreshed).toBe(true); - expect(dueResults[0]).toMatchObject({ refreshed: false, reason: "refresh_superseded" }); expect(finalObject.statusLabel).toBe("Closed"); expect(finalObject.refreshStartedAt).toBeNull(); expect(finalObject.refreshToken).toBeNull(); @@ -1073,9 +1087,7 @@ describeEmbeddedPostgres("externalObjectService", () => { await svc.syncIssue(active.issueId); await svc.syncIssue(paused.issueId); - const result = await svc.refreshDueObjectsForActiveCompanies(50, new Date(Date.now() + 1_000)); - - expect(result).toEqual({ companies: 1, checked: 1, refreshed: 1 }); + await runRefreshWork(svc); expect(resolve).toHaveBeenCalledTimes(1); const rows = await db.select().from(externalObjects); const activeObject = rows.find((row) => row.companyId === active.companyId); @@ -1134,9 +1146,8 @@ describeEmbeddedPostgres("externalObjectService", () => { .set({ nextRefreshAt: new Date(0) }) .where(eq(externalObjects.id, object.id)); - const refreshed = await svc.refreshDueObjects(companyId); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolve).toHaveBeenCalledTimes(1); }); @@ -1179,7 +1190,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("re-checks previously frozen GitHub not-found objects so they can self-heal", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); const inserted = await db.insert(externalObjects).values({ @@ -1220,9 +1231,10 @@ describeEmbeddedPostgres("externalObjectService", () => { const svc = externalObjectService(db, { resolvers: [resolver], github: false }); expect(object.isTerminal).toBe(true); - const refreshed = await svc.refreshDueObjects(companyId); - expect(refreshed).toHaveLength(1); - expect(refreshed[0]?.refreshed).toBe(true); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); const updated = await db.select().from(externalObjects).then((rows) => rows[0]!); expect(updated.statusLabel).toBe("Open"); @@ -1231,7 +1243,7 @@ describeEmbeddedPostgres("externalObjectService", () => { }); it("does not re-check probe-verified GitHub not-found objects (deleted references are terminal)", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1268,14 +1280,16 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); it("does not re-check terminal objects that are not GitHub not-found verdicts", async () => { - const { companyId } = await createIssue(); + const { companyId, issueId } = await createIssue(); const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42"); if (!canonical) throw new Error("expected canonical url"); await db.insert(externalObjects).values({ @@ -1311,9 +1325,11 @@ describeEmbeddedPostgres("externalObjectService", () => { }; const svc = externalObjectService(db, { resolvers: [resolver], github: false }); - const refreshed = await svc.refreshDueObjects(companyId); + const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)); + await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId, + sourceKind: "description", objectId: linked!.id }); + await runRefreshWork(svc); - expect(refreshed).toEqual([]); expect(resolver.resolve).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/git-restore-point-ledger.test.ts b/server/src/__tests__/git-restore-point-ledger.test.ts index 8b2e894dc44..0f30dea022b 100644 --- a/server/src/__tests__/git-restore-point-ledger.test.ts +++ b/server/src/__tests__/git-restore-point-ledger.test.ts @@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => { } = {}) { const companyId = input.companyId ?? randomUUID(); if (!input.companyId) { - await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` }); + await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` }); } const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); diff --git a/server/src/__tests__/github-connection-events.test.ts b/server/src/__tests__/github-connection-events.test.ts index 68900aaa0db..bef7020d59c 100644 --- a/server/src/__tests__/github-connection-events.test.ts +++ b/server/src/__tests__/github-connection-events.test.ts @@ -1,3 +1,4 @@ +import { externalObjectService } from "../services/external-objects.js"; import { startToolHealthWorker } from "./helpers/tool-health-worker.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; @@ -179,6 +180,19 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { expect(connector.leaseEvents).toHaveBeenCalledTimes(5); } + let releaseRefresh!: () => void; + let markStarted!: () => void; + const started = new Promise(resolve => { markStarted = resolve; }); + const held = new Promise(resolve => { releaseRefresh = resolve; }); + const resolver = externalObjectService(db, { github: false, resolvers: [{ + providerKey: "github", resolve: async () => { + markStarted(); await held; + return { ok: true, snapshot: { statusKey: "open", statusCategory: "open", statusTone: "info", isTerminal: false } }; + }, + }] }); + const oldRefresh = resolver.refreshObject(externalObjectId, { companyId, force: true }); + await started; + const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined); const outcome = await service.pollOnce({ forSetup: activeSetup }); const [processingReceipt] = await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id)); @@ -188,6 +202,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed", }, "GitHub connector event retained for retry"]]); warning.mockRestore(); + releaseRefresh(); + await expect(oldRefresh).resolves.toMatchObject({ refreshed: false, reason: "refresh_superseded" }); expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, confirmation!.id)))[0]!.status).toBe("accepted"); const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId)); expect(snapshot).toMatchObject({ @@ -196,6 +212,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => { statusCategory: "succeeded", statusTone: "success", isTerminal: true, + refreshToken: null, + refreshStartedAt: null, remoteVersion: "2026-09-04T11:59:01.000Z", data: expect.objectContaining({ marker: "preserved", merged: true, headRef: "feature", baseSha: "b".repeat(40) }), }); diff --git a/server/src/__tests__/heartbeat-execution-configuration.test.ts b/server/src/__tests__/heartbeat-execution-configuration.test.ts index 509a9cfa7e6..2cd52f84849 100644 --- a/server/src/__tests__/heartbeat-execution-configuration.test.ts +++ b/server/src/__tests__/heartbeat-execution-configuration.test.ts @@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { aiConnectionService } from "../services/ai-connections.js"; import * as aiRuntime from "../services/ai-connection-runtime.js"; +import * as connectorRuntime from "../services/connector-runtime.js"; +import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js"; import * as sentry from "../sentry.js"; import { waitForPendingRunFailureReports } from "../services/run-failure-report.js"; import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const skillFailures = vi.hoisted(() => new Map()); +const skillWaits = vi.hoisted(() => new Map Promise>()); describe("Runner home mode during recovery preparation", () => { it.each([ @@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => { return { ...service, withRuntimeSkillEntries: async (companyId: string, options: Parameters[1], consume: (entries: Awaited>) => T | Promise) => { + await skillWaits.get(companyId)?.(); const failure = skillFailures.get(companyId); if (failure) throw new Error(failure); return service.withRuntimeSkillEntries(companyId, options, consume); @@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), }); }, 60_000); - afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); }); + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); }); afterAll(async () => { await heartbeat?.drainActiveRunExecutions(); unregisterServerAdapter(adapterType); @@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } }); }, 60_000); + it("overlaps skills with assignments and dependent connection instructions before composition", async () => { + const f = await seed(); + const skillGate = deferred(); + skillWaits.set(f.companyId, () => skillGate.promise); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments"); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun"); + const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" }); + const opening = configured.prepare(); + try { + await vi.waitFor(() => { + expect(assignments).toHaveBeenCalledOnce(); + expect(instructions).toHaveBeenCalledOnce(); + }); + } finally { + skillGate.resolve(); + } + expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } }); + }); + + it("settles concurrent configuration reads before exposing the first preparation failure", async () => { + const f = await seed(); + const assignmentGate = deferred>>(); + const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise); + const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure")); + skillFailures.set(f.companyId, "first skill failure"); + const configured = await configuration(f, { provider: "codex", homeMode: "isolated" }); + let settled = false; + const opening = configured.prepare(); + void opening.then(() => { settled = true; }, () => { settled = true; }); + try { + await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce()); + await Promise.resolve(); + expect(settled).toBe(false); + expect(instructions).not.toHaveBeenCalled(); + } finally { + assignmentGate.resolve([]); + } + await expect(opening).rejects.toThrow("first skill failure"); + }); it("does not install new environment declarations when recovering a saved skill selection", async () => { const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`; @@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12); const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_") && span.attributes["paperclip.task.run.run_id"] === runIdHash); - expect(steps.map(span => span.name)).toEqual([ + expect(steps.map(span => span.name).sort()).toEqual([ "configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime", "configuration_skills", "configuration_connector_assignments", "configuration_connector_skills", "configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity", - ].map(step => `heartbeat.prepare.${step}`)); + ].map(step => `heartbeat.prepare.${step}`).sort()); for (const span of steps) { expect(span.status).toEqual({ code: 0 }); expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); @@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", ( && span.attributes["paperclip.task.run.run_id"] === runIdHash); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0); expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 }); - expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false); + expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true); for (const span of steps) { expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash }); expect(span.events).toEqual([]); diff --git a/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts new file mode 100644 index 00000000000..f3398d828f8 --- /dev/null +++ b/server/src/__tests__/heartbeat-run-dispatch-lifecycle.test.ts @@ -0,0 +1,127 @@ +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; +import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js"; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { + for (const stop of cleanup.splice(0)) await stop(); + vi.useRealTimers(); +}); +function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) { + vi.useFakeTimers(); + vi.setSystemTime(1_000_000); + const dialect = new PgDialect(); + const execute = vi.fn(async (query: any) => { + if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; + throw new Error("unexpected database query"); + }); + const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise) => Promise } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise): Promise { + return callback({ execute, transaction: db.transaction }); + } }); + const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false, + draining: input.draining ?? false, standby: false, fail: false }; + const promote = vi.fn(async () => { + if (state.fail) throw new Error("database unavailable"); + if (state.due !== null && state.due <= Date.now()) state.due = null; + }); + const next = vi.fn(async () => state.due); + const dispatch = vi.fn(async (_scope?: { companyId: string }) => {}); + const coordinator = createDeliveryWorkCoordinator({ owner: db, + canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() }); + const worker = registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries: promote, nextScheduledRetryAt: next, + resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed, + }, () => !state.draining && !state.standby); + cleanup.push(() => coordinator.stop()); + return { db, state, promote, next, dispatch, coordinator, worker, + notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)), + settle: () => vi.advanceTimersByTimeAsync(1_000) }; +} + +describe("run dispatch obligations", () => { + it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => { + const f = fixture({ suppressed }); await f.worker.ready; + const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]; + expect(f.coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls); + expect(f.db.execute).not.toHaveBeenCalled(); + }); + it("restores an exact deadline and does no earlier recovery work", async () => { + const f = fixture({ due: 1_010_000 }); await f.worker.ready; + expect(f.coordinator.nextWakeAt()).toBe(1_010_000); + await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2); + expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.dispatch).toHaveBeenCalledOnce(); + }); + it("rearms earlier work and cancels a future deadline on a committed producer change", async () => { + const f = fixture({ due: 1_600_000 }); await f.worker.ready; + f.state.due = 1_005_000; await f.notify(); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBe(1_005_000); + f.state.due = null; await f.notify(); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const before = f.next.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before); + }); + it("batches repeated tenant writes and passes only their company to queued dispatch", async () => { + const f = fixture(); await f.worker.ready; + for (let index = 0; index < 100; index++) { + const source = { companyId: "company-a", issueId: `issue-${index % 2}`, + runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" }; + await f.notify(source); + await vi.advanceTimersByTimeAsync(10); + } + expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2); + expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2); + await f.settle(); + expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" }); + for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" }); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const count = f.dispatch.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count); + }); + it("remembers the first producer arriving during drain without owner SQL", async () => { + const f = fixture(); await f.worker.ready; f.state.draining = true; + await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000); + expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(0); + f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("releases a failed producer hold during drain and retains recovery for resume", async () => { + const f = fixture(); await f.worker.ready; + f.state.fail = true; await f.notify(); await f.settle(); + expect(f.promote).toHaveBeenCalledTimes(2); + f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000); + expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(0); + f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("preserves a notification during a nonoverlapping pass", async () => { + const f = fixture(); await f.worker.ready; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + f.promote.mockImplementationOnce(() => held); + await f.notify(); await f.settle(); + await f.notify(); await f.settle(); + expect(f.promote).toHaveBeenCalledTimes(2); + release(); await f.settle(); + await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => { + const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready; + await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled(); + f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000); + expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull(); + f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop(); + await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2); + }); +}); diff --git a/server/src/__tests__/heartbeat-run-dispatch-work.test.ts b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts new file mode 100644 index 00000000000..bcc8ed23972 --- /dev/null +++ b/server/src/__tests__/heartbeat-run-dispatch-work.test.ts @@ -0,0 +1,238 @@ +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { agentService } from "../services/agents.js"; +import { budgetService } from "../services/budgets.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { notifyRunDispatch } from "../services/run-dispatch-notifications.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => { + let database: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType | undefined; + let worker: ReturnType["registerRunDispatchWork"]> | undefined; + let reads: ReturnType; + let draining = false; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + }, 90_000); + beforeEach(async () => { + await db.execute(sql`TRUNCATE companies CASCADE`); + heartbeat = heartbeatService(db); draining = false; errors.length = 0; + reads = vi.spyOn(db, "select"); + }); + afterEach(async () => { + await worker?.stop(); await coordinator?.stop(); + await heartbeat.drainActiveRunExecutions(); + worker = undefined; coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, + canReconcile: () => true, onError: error => errors.push(error) }); + worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining); + await worker.ready; + } + async function fixture() { + const companyId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } }, + }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed", + responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(), + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {}, + }).returning(); + return { companyId, agentId: agent.id, runId: run.id }; + } + async function quiet() { + await heartbeat.drainActiveRunExecutions(); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 }); + const count = reads.mock.calls.length; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(reads).toHaveBeenCalledTimes(count); + } finally { vi.useRealTimers(); } + } + async function status(id: string) { + return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]); + } + async function completed(id: string) { + await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 }); + await heartbeat.drainActiveRunExecutions(); + const completed = await status(id); + expect(completed.status).toBe("succeeded"); + expect(completed.stdoutExcerpt).toContain("dispatch-proof"); + } + it("keeps fresh empty, configured idle and retained terminal history quiet", async () => { + await start(); await quiet(); + await fixture(); await worker!.stop(); await coordinator!.stop(); + await start(); await quiet(); + }); + it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => { + const f = await fixture(); await start(); await quiet(); + const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 }); + expect(result.outcome).toBe("scheduled"); + if (result.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 }); + expect((await status(result.run.id)).status).toBe("scheduled_retry"); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(Date.now()).toBeLessThan(result.dueAt.getTime()); + expect((await status(result.run.id)).status).toBe("scheduled_retry"); + await completed(result.run.id); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1); + await quiet(); + }); + it("reconstructs an overdue retry after process restart without a periodic tick", async () => { + const f = await fixture(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + heartbeat = heartbeatService(db); + await start(); await completed(scheduled.run.id); await quiet(); + }); + it("cancels a future retry and removes its old deadline", async () => { + const f = await fixture(); await start(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 }); + await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true }); + await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled"); + }); + it("retains the first retry written during drain and catches it up after resume", async () => { + const f = await fixture(); await start(); await quiet(); draining = true; + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + const count = reads.mock.calls.length; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await vi.advanceTimersByTimeAsync(90_000); + expect(reads).toHaveBeenCalledTimes(count); + expect(idleWorkSnapshot().active).toBe(0); + } finally { vi.useRealTimers(); } + expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); + draining = false; worker!.wake(); + await completed(scheduled.run.id); await quiet(); + }); + it("uses a committed budget raise to resume a queued run in the company it paused", async () => { + const f = await fixture(); + const budgets = budgetService(db); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100, + metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user"); + const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId, + provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0, + costCents: 125, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost); + expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused"); + const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning(); + await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued"); + await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000, + metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user"); + await completed(queued.id); await quiet(); + }); + it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => { + const f = await fixture(); await start(); await quiet(); + await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId)); + const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning(); + let writing: ReturnType["clearError"]> | undefined; + try { + await db.transaction(async blocker => { + await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update"); + writing = agentService(db).clearError(f.agentId); + void writing.catch(() => {}); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false); + expect((await status(queued.id)).status).toBe("queued"); + expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error"); + }); + } finally { await writing; } + await completed(queued.id); await quiet(); + }); + it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => { + const f = await fixture(); + const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 }); + if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation"); + await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() }, + cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id)); + await start(); await quiet(); + expect((await status(scheduled.run.id)).status).toBe("scheduled_retry"); + expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] }); + await quiet(); + }); + it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => { + const f = await fixture(); await start(); await quiet(); + await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId)); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { + await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status"); + expect(coordinator!.nextWakeAt()).toBeNull(); + const count = reads.mock.calls.length; + await vi.advanceTimersByTimeAsync(60_000); + expect(reads).toHaveBeenCalledTimes(count); + } finally { vi.useRealTimers(); } + expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error"); + await quiet(); + }); + it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => { + const selected = await fixture(), foreign = await fixture(); await start(); + const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 }); + const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 }); + if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries"); + const other = heartbeatService(db); + const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const competingWorker = other.registerRunDispatchWork(competing, () => true); + try { + await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions(); + expect((await status(later.run.id)).status).toBe("scheduled_retry"); + expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1); + await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true }); + await quiet(); + } finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); } + }); + it("holds nested producer work until outer commit and rolls back without a successor", async () => { + const f = await fixture(); await start(); await quiet(); + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + let queuedId = ""; + const writing = db.transaction(async tx => { + await tx.transaction(async nested => { + await notifyRunDispatch(nested); + const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning(); + queuedId = row.id; + }); + await held; + }); + await vi.waitFor(() => expect(queuedId).not.toBe("")); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(await status(queuedId)).toBeUndefined(); + release(); await writing; await completed(queuedId); await quiet(); + await expect(db.transaction(async tx => { + await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); }); + throw new Error("rollback outer"); + })).rejects.toThrow("rollback outer"); + await quiet(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2); + }); +}); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index 65191c24d2f..c4f0772ad47 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { await heartbeat.resumeQueuedRuns(); const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); expect(delivered.runId).toBeTruthy(); + // The receipt can commit before execution enters liveRunExecutions. + // Observe actual adapter admission before awaiting its drain. + await vi.waitFor(async () => { + const current = await heartbeat.getRun(delivered.runId!); + expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({ + status: current?.status, errorCode: current?.errorCode, + })).toBe(1); + }, { timeout: 5_000 }); await heartbeat.waitForRunExecutionDrain(delivered.runId!); expect(countExecuteCallsForRun(delivered.runId!)).toBe(1); expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] }); diff --git a/server/src/__tests__/heartbeat-task-context.test.ts b/server/src/__tests__/heartbeat-task-context.test.ts index 5449d7def6c..e073db37676 100644 --- a/server/src/__tests__/heartbeat-task-context.test.ts +++ b/server/src/__tests__/heartbeat-task-context.test.ts @@ -1,10 +1,11 @@ import { createHash, randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared"; import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js"; import { heartbeatService } from "../services/heartbeat.js"; +import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js"; import { documentService } from "../services/documents.js"; import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js"; import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js"; @@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s describePostgres("heartbeat task context delivered to an adapter", () => { const adapterType = "task_context_test"; const captured = new Map(); + const inspectPreparation = new Map void>(); let temporary: Awaited>; let db: ReturnType; let heartbeat: ReturnType; @@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => { type: adapterType, supportsLocalAgentJwt: false, execute: async (input) => { + inspectPreparation.get(input.agent.id)?.(); captured.set(input.runId, input); return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} }; }, @@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => { }); }); + it("reads selected comments once for both wake projections", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ + id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user", + body: "selected wake canary", + }); + const select = vi.spyOn(db, "select"); + inspectPreparation.set(f.agentId, () => { + const commentReads = select.mock.calls.filter(([projection]) => projection + && projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt); + expect(commentReads).toHaveLength(1); + }); + try { + const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] }); + } finally { + inspectPreparation.delete(f.agentId); + select.mockRestore(); + } + }); + + it("cancels deleted-only queued comments before adapter dispatch", async () => { + const f = await seed(), commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() }); + const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented", + payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const finished = await heartbeat.getRun(run!.id); + expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" }); + expect(captured.has(run!.id)).toBe(false); + }); + + it("preserves deleted comment metadata without delivering its body", async () => { + const f = await seed(); + const commentId = randomUUID(); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(), + presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null }); + expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] }); + expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary"); + }); + + it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => { + const f = await seed(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + const context: Record = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } }; + const pinned = vi.fn(async () => null); + const preparing = prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef, issueContext: null, context, taskSession: null }, + policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned }, + }); + if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable"); + else await preparing; + expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0); + expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) }); + }); + + it("retains the context issue scope when no issue reference is available", async () => { + const f = await seed(); + const otherIssueId = randomUUID(), commentId = randomUUID(); + await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 }); + await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId, + authorType: "user", authorUserId: "context-user", body: "other issue canary" }); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", issueId: f.issueId }).returning(); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const context: Record = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] }; + await prepareHeartbeatTaskContext(db, { + run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null }, + policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, + services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null }, + }); + expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary"); + expect(context.paperclipWakeComment).toBeUndefined(); + }); + it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => { const f = await seed(); const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId }); diff --git a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts index 52d434a3078..484cf55e8ff 100644 --- a/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts +++ b/server/src/__tests__/heartbeat-workspace-branch-containment.test.ts @@ -1,3 +1,5 @@ +import { subscribeDatabaseWork } from "@paperclipai/db"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { execFile } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; @@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { const tempRoots: string[] = []; beforeAll(async () => { - if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl); + if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 }); else { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-"); db = createDb(tempDb.connectionString); @@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => { }, }); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); const heartbeat = heartbeatService(db); - const run = await heartbeat.wakeup(agentId, { - source: "assignment", - triggerDetail: "system", - reason: "issue_assigned", - payload: { issueId }, - contextSnapshot: { issueId, wakeReason: "issue_assigned" }, - }); - + let run; + try { + run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned", + payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } }); + } finally { unsubscribeRoutine(); } + expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled"); expect(run).toBeNull(); expect(adapterExecute).not.toHaveBeenCalled(); diff --git a/server/src/__tests__/historical-cleanup-verification.test.ts b/server/src/__tests__/historical-cleanup-verification.test.ts index 37bc2dd6692..1269c02f1a2 100644 --- a/server/src/__tests__/historical-cleanup-verification.test.ts +++ b/server/src/__tests__/historical-cleanup-verification.test.ts @@ -1,8 +1,9 @@ -import { randomUUID } from "node:crypto"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db"; import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests (support.supported ? describe : describe.skip)("historical cleanup verification", () => { let db: ReturnType, database: Awaited>; beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000); - afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); + afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true); const state = async () => (await read()).backgroundWork; @@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }), }); } + it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => { + const f = await seed(adHoc); + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value); + let absent = false; + const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => { + if (!absent) return Response.json({ absent: false }); + const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000); + expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id, + leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId }); + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1", + sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url"); + return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` }); + }); + vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db); + const service = historicalCleanupVerificationService(db, { + inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined, + }); + expect(await state()).toBe("present"); + expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 }); + expect(await state()).toBe("present"); + absent = true; + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); + expect(await state()).toBe("none"); + const [current] = await db.select().from(environmentLeases); + expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId }); + expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination"); + expect(await service.sweep()).toMatchObject({ checked: 0 }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => { + await seed(true); + for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture", + PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value); + const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl); + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null, + inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }), + }] }); + const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined }); + expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 }); + expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled(); + }); it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => { const f = await seed(adHoc); expect(await state()).toBe("present"); const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 }); diff --git a/server/src/__tests__/issue-blocker-attention.test.ts b/server/src/__tests__/issue-blocker-attention.test.ts index 1e6c00631e7..4f1a0e71573 100644 --- a/server/src/__tests__/issue-blocker-attention.test.ts +++ b/server/src/__tests__/issue-blocker-attention.test.ts @@ -8,6 +8,7 @@ import { approvals, companies, createDb, + applyPendingMigrations, heartbeatRuns, issueApprovals, issueRelations, @@ -21,7 +22,10 @@ import { import { issueService } from "../services/issues.js"; import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); - db = createDb(tempDb.connectionString); + if (externalUrl) await applyPendingMigrations(externalUrl); + else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-"); + db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); }, 20_000); @@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/issue-recovery-deadlines.test.ts b/server/src/__tests__/issue-recovery-deadlines.test.ts new file mode 100644 index 00000000000..e23a672c0dc --- /dev/null +++ b/server/src/__tests__/issue-recovery-deadlines.test.ts @@ -0,0 +1,382 @@ +import { setImmediate as immediate } from "node:timers/promises"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { issueService } from "../services/issues.js"; +import { issueRecoveryActionService } from "../services/issue-recovery-actions.js"; +import { registerIssueRecoveryWork } from "../services/recovery/work.js"; +import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; +import { taskWatchdogService } from "../services/task-watchdogs.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; + +vi.setConfig({ testTimeout: 90_000 }); + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => { + let database: Awaited> | undefined; + let db: ReturnType; + let heartbeat: ReturnType; + let coordinator: ReturnType; + let worker: ReturnType["registerIssueStateRecoveryWork"]>; + let dispatch: ReturnType["registerRunDispatchWork"]>; + let cancellation: ReturnType["registerScheduledRetryCancellationWork"]>; + let draining = false; + const tasks = new Map number | null; hasPending: () => Promise }>(); + const errors: unknown[] = []; + const passes = new Set>(); + const extraStops: Array<() => Promise> = []; + const extraDeadlines: Array<() => number | null> = []; + beforeAll(async () => { + if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-"); + db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + }, 90_000); + beforeEach(async () => { + await db.execute(sql`TRUNCATE companies CASCADE`); + heartbeat = heartbeatService(db); draining = false; errors.length = 0; + }); + afterEach(async () => { + for (const stop of extraStops.splice(0)) await stop(); + extraDeadlines.length = 0; + await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop(); + await heartbeat.drainActiveRunExecutions(); + vi.useRealTimers(); vi.restoreAllMocks(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + tasks.clear(); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => { + tasks.set(queue, task); + return register(queue, { ...task, run: signal => { + const work = task.run(signal); passes.add(work); + void work.then(() => passes.delete(work), () => passes.delete(work)); + return work; + } }); + }; + worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining); + dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining); + cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining); + await Promise.all([worker.ready, dispatch.ready, cancellation.ready]); + } + async function compete() { + const other = heartbeatService(db); + const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const work = other.registerIssueStateRecoveryWork(competing, () => true); + extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready; + } + async function fixture() { + const companyId = randomUUID(); + await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`, + requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" }); + await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" }); + const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle", + adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() }, + runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } }, + }).returning(); + return { companyId, agentId: agent.id }; + } + async function settled() { + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const wake = coordinator.nextWakeAt(); + const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))), + runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) }); + expect(wake, detail).toBeNull(); + expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null)); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 40_000 }); + } + async function quiet() { + await settled(); + expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0); + const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute"); + const selected = reads.mock.calls.length, executed = executes.mock.calls.length; + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); } + finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); } + } + async function finish(issueId: string) { + await issueService(db).update(issueId, { status: "done" }); + // Closing one blocker can legitimately start its dependent. Wait for this + // task's dispatch and execution; quiet() verifies the whole fixture later. + await vi.waitFor(async () => { + await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); + expect(await db.select().from(heartbeatRuns).where(and( + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`, + sql`${heartbeatRuns.status} in ('queued', 'running')`, + ))).toHaveLength(0); + }, { timeout: 15_000 }); + const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + if (saved.length > 0) { + const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime())); + // Keep PostgreSQL I/O and coordinator settlement ahead of virtual time. + // A single leap can otherwise occur before the retry deadline is armed. + while (Date.now() <= due) { + await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1)); + await Promise.allSettled([...passes]); + await immediate(); + } + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...passes]); + await heartbeat.drainActiveRunExecutions(); + const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`)); + expect(remaining).toHaveLength(0); + }); + for (const retry of saved) { + const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id)); + expect(settledRetry.status).toBe("cancelled"); + } + } + } + async function completed(issueId: string) { + let runId: string | undefined; + await vi.waitFor(async () => { + const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`); + const run = runs.find(row => row.status === "running" || row.status === "succeeded"); + expect(run).toBeDefined(); runId = run!.id; + }, { timeout: 15_000 }); + await vi.waitFor(async () => { + await heartbeat.drainActiveRunExecutions(); + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded"); + }, { timeout: 15_000 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!)); + expect(run.stdoutExcerpt).toContain("recovery-proof"); + return run; + } + it("keeps empty, configured idle, and terminal history quiet", async () => { + await fixture(); await start(); await quiet(); + }); + it("creates a blocking child through the ordinary root service transaction", async () => { + const f = await fixture(); + const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" }); + const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true }); + expect(child.parentBlockerAdded).toBe(true); + expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId), + eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1); + }); + it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => { + const f = await fixture(); + if (mode === "committed") { await start(); await quiet(); } + const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId }); + const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] }); + const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" }); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id, + kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue", + fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} }); + if (mode === "startup") await start(); + await vi.waitFor(async () => { + expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId); + expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved"); + }, { timeout: 40_000 }); + await finish(blocker.id); await finish(dependent.id); await quiet(); + }); + it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => { + const f = await fixture(), foreign = await fixture(); + await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId)); + await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId, + title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId }))); + const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning(); + const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id, + kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue", + fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} }); + const scans = { blockers: 0, actions: 0, loads: 0 }; + const originalSelect = db.select.bind(db); + vi.spyOn(db, "select").mockImplementation(((fields?: Record) => { + const query = originalSelect(fields as never); const from = query.from.bind(query); + query.from = ((table: unknown) => { + if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++; + if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++; + if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++; + return from(table as never); + }) as typeof query.from; + return query; + }) as typeof db.select); + await start(); await settled(); + expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8); + expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved"); + const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning(); + const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id, + kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning(); + const before = { ...scans }; + for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(scans.loads).toBe(before.loads); + await settled(); + expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers); + expect(scans.loads).toBeLessThanOrEqual(before.loads + 4); + expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active"); + expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet(); + }); + it("limits an ordinary comment to its issue among one thousand open assignments", async () => { + const f = await fixture(); + const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({ + companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId, + }))).returning({ id: issues.id }); + const checked: string[] = []; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + const register = coordinator.register.bind(coordinator); + coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const pass = task.run(signal); passes.add(pass); + void pass.then(() => passes.delete(pass), () => passes.delete(pass)); + return pass; + } }); + // Observe the actual paged reconstruction boundary without executing one + // thousand adapters. The producer, transaction, coordinator and SQL are real; + // the other acceptance cases exercise the resulting recovery decisions. + worker = registerIssueRecoveryWork(db, coordinator, { + stranded: async scope => { checked.push(scope.issueId); }, + orphanBlockers: async () => {}, recoveryActions: async () => {}, + dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {}, + watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false, + }, () => true); + await worker.ready; + async function settleWork() { + for (let step = 0; step < 120; step++) { + await vi.advanceTimersByTimeAsync(1000); + await Promise.allSettled([...passes]); await immediate(); + if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return; + } + throw new Error("Recovery did not settle its committed work within two virtual minutes"); + } + await settleWork(); + expect(new Set(checked).size).toBe(1000); + checked.length = 0; + await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" }); + await settleWork(); + expect(checked).toEqual([rows[0].id]); + // Company eligibility changes still reconstruct all assignments. Narrowing + // comment hints must not hide unrelated work after an operator resumes it. + checked.length = 0; + await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId })); + await settleWork(); + expect(new Set(checked).size).toBe(1000); + await quiet(); + }); + it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => { + const f = await fixture(); await start(); await quiet(); + const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId }); + await completed(issue.id); + await finish(issue.id); await quiet(); + }, 90_000); + it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => { + const f = await fixture(); + const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" }); + const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId, + blockedByIssueIds: [blocker.id] }); + await start(); await quiet(); + await compete(); + await issueService(db).update(blocker.id, { status: "done" }); + const run = await completed(dependent.id); + expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved"); + await finish(dependent.id); await quiet(); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1); + }, 90_000); + it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" }); + await start(); await quiet(); + await compete(); + const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId }); + let reviewId: string | undefined; + await vi.waitFor(async () => { + const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id)); + expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!; + }, { timeout: 15_000 }); + await completed(reviewId!); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1); + await finish(reviewId!); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); + }, 90_000); + it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => { + const selected = await fixture(); await start(); await quiet(); + const foreign = await fixture(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId, + status: "failed", finishedAt: new Date() }).returning(); + const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done", + checkoutRunId: run.id }).returning(); + await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet(); + expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id); + await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop(); + heartbeat = heartbeatService(db); await start(); await quiet(); + expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull(); + }); + it("fences nested assignment writes until outer commit and ignores rollback", async () => { + const f = await fixture(); await start(); await quiet(); + let committedId = ""; + await db.transaction(async outer => { + await outer.transaction(async nested => { + const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested); + committedId = issue.id; + }); + await new Promise(resolve => setTimeout(resolve, 100)); + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + }); + await completed(committedId); await finish(committedId); await settled(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { + await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested); + }); + throw new Error("rollback control"); + })).rejects.toThrow("rollback control"); + await quiet(); + expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0); + }, 90_000); + it("retains the first committed assignment during drain and executes on resume", async () => { + const f = await fixture(); await start(); await quiet(); draining = true; + const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId }); + await new Promise(resolve => setTimeout(resolve, 100)); + const reads = vi.spyOn(db, "select"); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); } + finally { vi.useRealTimers(); reads.mockRestore(); } + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + draining = false; worker.wake(); dispatch.wake(); + await completed(issue.id); await finish(issue.id); await quiet(); + }, 90_000); + it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" }); + await start(); await quiet(); + await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId }); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 }); + expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0); + await taskWatchdogService(db).disableForIssue(f.companyId, source.id); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); + }); + it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => { + const f = await fixture(); + const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" }); + const lease = new Date(Date.now() + 5_000); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease, + contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning(); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id)); + await start(); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running"); + await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded"); + if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet(); + }, 90_000); +}); diff --git a/server/src/__tests__/issue-recovery-work-lifecycle.test.ts b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts new file mode 100644 index 00000000000..52f465daa8e --- /dev/null +++ b/server/src/__tests__/issue-recovery-work-lifecycle.test.ts @@ -0,0 +1,149 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js"; +import { registerIssueRecoveryWork } from "../services/recovery/work.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const stops: Array<() => Promise> = []; +afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); }); +function fixture() { + vi.useFakeTimers(); vi.setSystemTime(1_000_000); + const dialect = new PgDialect(); + const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> }; + const execute = vi.fn(async (query: any) => { + if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }]; + throw new Error("unexpected SQL"); + }); + const load = vi.fn((table: unknown, distinct = false) => distinct + ? table === issueRecoveryActions ? rows.actions : rows.blockers + : table === issueWatchdogs ? rows.watchdogs : rows.issues); + const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) }); + const db: any = installDatabaseWorkSignals({ execute, + select: () => query(), selectDistinct: () => query(true), + transaction: async (callback: (tx: any) => Promise) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }), + }); + const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false }; + const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => { + if (state.fail) throw new Error("database unavailable"); + if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline); + }); + const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {}); + const suppression = vi.fn(async () => state.suppressed); + const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; }, + locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; }, + schedulingSuppressed: suppression }; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby, + canReconcile: () => !state.standby, onError: vi.fn() }); + const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby); + stops.push(() => coordinator.stop()); + return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression, + notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })), + settle: () => vi.advanceTimersByTimeAsync(1_000) }; +} +describe("issue recovery obligation lifecycle", () => { + it("has no timer, SQL, or owner probe after empty reconstruction", async () => { + const f = fixture(); await f.worker.ready; + expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled(); + }); + it("reads scheduling suppression once per pass instead of once per issue", async () => { + const f = fixture(); + f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" }))); + await f.worker.ready; + expect(f.check).toHaveBeenCalledTimes(75); + expect(f.suppression).toHaveBeenCalledTimes(6); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => { + const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable")); + await f.worker.ready; const reads = f.suppression.mock.calls.length; + expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads); + await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => { + const f = fixture(); await f.worker.ready; + const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions }; + await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle(); + expect(f.coordinator.nextWakeAt()).toBeNull(); + const loads = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads); + }); + it("runs orphan blocker and action duties without assignment entries", async () => { + const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" }); + f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready; + expect(f.check).not.toHaveBeenCalled(); + expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" }); + expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" }); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); + f.state.deadline = 1_015_000; await f.worker.ready; + await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length; + for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); } + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked); + await vi.advanceTimersByTimeAsync(13_000); + expect(f.state.deadline).toBe(1_015_000); + expect(f.coordinator.nextWakeAt()).toBe(1_031_000); + await vi.advanceTimersByTimeAsync(16_000); + expect(f.load.mock.calls.length).toBeGreaterThan(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("retains exact grace and lease deadlines without early owner probes", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); + f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000; + await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000); + const checked = f.check.mock.calls.length; + await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked); + await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("releases settled producer holds during domain drain while coordinator remains open", async () => { + const f = fixture(); await f.worker.ready; f.state.draining = true; + f.rows.issues.push({ id: "issue", companyId: "selected" }); + await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length; + await vi.advanceTimersByTimeAsync(90_000); + expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBe(0); + f.state.draining = false; f.worker.wake(); await f.settle(); + expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("retries concrete failed entries and then disarms without reloading the company", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true; + await f.worker.ready; const loaded = f.load.mock.calls.length; + expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length; + f.state.fail = false; await vi.advanceTimersByTimeAsync(1); + expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps an incomplete watchdog subtree as concrete retry debt", async () => { + const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true; + await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000); + const loaded = f.load.mock.calls.length; f.state.incomplete = false; + await vi.advanceTimersByTimeAsync(30_000); + expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded); + expect(f.coordinator.nextWakeAt()).toBeNull(); + }); + it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => { + const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready; + let release!: () => void; const held = new Promise(resolve => { release = resolve; }); + f.check.mockImplementationOnce(() => held); + await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000); + expect(f.coordinator.nextWakeAt()).toBeNull(); + expect(f.check.mock.calls.length).toBeGreaterThan(3); + }); + it("retains startup work on standby and cancels future deadlines on shutdown", async () => { + const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); + await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled(); + f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle(); + expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop(); + await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce(); + }); +}); diff --git a/server/src/__tests__/issue-tree-control-service.test.ts b/server/src/__tests__/issue-tree-control-service.test.ts index 923f399d3bd..e98efb18309 100644 --- a/server/src/__tests__/issue-tree-control-service.test.ts +++ b/server/src/__tests__/issue-tree-control-service.test.ts @@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => { beforeAll(async () => { if (externalUrl) { await applyPendingMigrations(externalUrl); - db = createDb(externalUrl); + db = createDb(externalUrl, { maxConnections: 6 }); return; } tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-"); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index fb13ff43e82..0d28858b95c 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU beforeAll(async () => { if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-"); - db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString); + db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); svc = issueService(db); await ensureIssueRelationsTable(db); }, 20_000); diff --git a/server/src/__tests__/managed-skill-content.test.ts b/server/src/__tests__/managed-skill-content.test.ts index 210a88a8fd7..d617f8cdb0e 100644 --- a/server/src/__tests__/managed-skill-content.test.ts +++ b/server/src/__tests__/managed-skill-content.test.ts @@ -1,3 +1,5 @@ +import { createServer } from "node:http"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js"; import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js"; import { eq } from "drizzle-orm"; @@ -77,6 +79,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", () .toMatchObject({ content: "complete root guide" }); }); + it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => { + const companyId = await company(); + const objects = new Map(); + const server = createServer((request, response) => { + void (async () => { + const key = new URL(request.url!, "http://fixture").pathname; + if (request.method === "PUT") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + let body = Buffer.concat(chunks); + if (request.headers["content-encoding"]?.includes("aws-chunked")) { + const decoded: Buffer[] = []; + let offset = 0; + while (offset < body.length) { + const end = body.indexOf("\r\n", offset); + const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16); + if (size === 0) break; + decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2; + } + body = Buffer.concat(decoded); + } + objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return; + } + const body = objects.get(key); + if (!body) { response.writeHead(404).end(); return; } + response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' }); + response.end(request.method === "HEAD" ? undefined : body); + })().catch(() => { response.writeHead(500).end(); }); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address"); + const endpoint = `http://127.0.0.1:${address.port}`; + const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" }; + const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]])); + Object.assign(process.env, credentials); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({ + bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true, + }))); + const key = "cloud-runtime-config:v1"; + try { + await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" }); + const run = (phase: string) => promisify(execFile)(process.execPath, [ + "--import", fileURLToPath(import.meta.resolve("tsx")), + fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)), + `--phase=${phase}`, `--company=${companyId}`, "--limit=50", + ], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString, + PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture", + PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" }, + timeout: 60_000, maxBuffer: 1024 * 1024 }); + await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration"); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0); + await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } }); + const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!); + expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]); + const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const); + const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()]).toEqual(before); + const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!; + const archive = objects.get(archiveKey)!; + const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1; + objects.set(archiveKey, corrupted); + const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]); + objects.set(archiveKey, archive); + const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!; + const manifest = objects.get(manifestKey)!; objects.delete(manifestKey); + const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]); + objects.set(manifestKey, manifest); + const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!); + expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]); + expect([...objects.entries()].sort()).toEqual([...before].sort()); + expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))) + .toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]); + } finally { + await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key)); + vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects")))); + for (const [key, value] of Object.entries(previousCredentials)) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }, 180_000); + it.each([ { storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" }, { storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3", diff --git a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts index 1e4d56436e7..d9cf9880357 100644 --- a/server/src/__tests__/merged-pr-confirmation-sweep.test.ts +++ b/server/src/__tests__/merged-pr-confirmation-sweep.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { inArray } from "drizzle-orm"; @@ -131,7 +132,8 @@ describe("merged pull-request confirmation extraction", () => { }); }); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_SWEEP_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { @@ -139,8 +141,8 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -153,6 +155,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); @@ -276,7 +279,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => { repo: "paperclip", number: 39, }])).resolves.toEqual({ - checked: 6, + checked: 3, candidates: 3, accepted: 2, woken: 2, diff --git a/server/src/__tests__/merged-pr-confirmation-work.test.ts b/server/src/__tests__/merged-pr-confirmation-work.test.ts new file mode 100644 index 00000000000..95565fd96e7 --- /dev/null +++ b/server/src/__tests__/merged-pr-confirmation-work.test.ts @@ -0,0 +1,366 @@ +import { createServer } from "node:http"; +import { createPullRequestMergeStateResolver } from "../services/github-pull-request-merge.js"; +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { eq, sql } from "drizzle-orm"; +import { activityLog, agents, chatActions, chatConversations, chatEndpoints, chatPublications, companies, createDb, issues, issueThreadInteractions, toolApplications, toolConnections, type Db } from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { issueService } from "../services/issues.js"; +import { registerMergeConfirmationWork, MERGE_CONFIRMATION_RECONCILE_MS } from "../services/merged-pr-confirmation-work.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("merge confirmation work with PostgreSQL and simulated GitHub", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let companyId: string, issueId: string, agentId: string; + let coordinator: ReturnType | undefined; + let draining = false, standby = false; + const errors: unknown[] = []; + const provider = vi.fn(async (_companyId: string, _reference: { number: number }): Promise<"open" | "merged" | "unknown"> => "merged"); + const wakeup = vi.fn(async () => ({ id: "simulated-durable-wake" })); + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("merged-pr-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + errors.length = 0; draining = false; standby = false; + provider.mockReset().mockResolvedValue("merged"); wakeup.mockClear(); + companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Merge worker", issuePrefix: `M${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", adapterType: "codex_local" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Merge task", status: "todo", assigneeAgentId: agentId }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(chatActions); await db.delete(chatPublications); await db.delete(chatConversations); await db.delete(chatEndpoints); + await db.delete(toolConnections); await db.delete(toolApplications); + await db.delete(activityLog); await db.delete(issueThreadInteractions); await db.delete(issues); + await db.delete(agents); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + const service = (database: Db = db) => issueThreadInteractionService(database, { resolvePullRequestState: provider, wakeup }); + const input = (number = 39) => ({ kind: "request_confirmation" as const, title: `Merge paperclipai/paperclip#${number}?`, + resolverPolicy: "human_only" as const, continuationPolicy: "wake_assignee_on_accept" as const, + payload: { version: 1 as const, prompt: "Merge the PR?" } }); + const create = (database: Db = db, number = 39) => service(database).create({ id: issueId, companyId }, input(number), { agentId }); + const stored = (id: string) => db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, id)).then(rows => rows[0]); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining && !standby, + canReconcile: () => !standby, onError: error => errors.push(error) }); + const worker = registerMergeConfirmationWork(coordinator, service(), () => !draining && !standby); + await worker.ready; + return worker; + } + async function waitFor(check: () => Promise | void) { + for (let attempt = 0; attempt < 100; attempt++) { + await vi.advanceTimersByTimeAsync(1000); + await delay(10); + try { await check(); return; } catch (error) { if (attempt === 99) throw error; } + } + } + const settle = () => waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + async function quiet() { + await waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); }); + const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute"); + const calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(3 * MERGE_CONFIRMATION_RECONCILE_MS); + expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(provider).toHaveBeenCalledTimes(calls); + reads.mockRestore(); transactions.mockRestore(); executes.mockRestore(); + } + + it("settles empty and configured idle, then ordinary creation accepts, audits, wakes once and becomes quiet with retained history", async () => { + await start(); await quiet(); expect(provider).not.toHaveBeenCalled(); + const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); + await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${card.id}:accepted`, allowRunCoalescing: false })); + expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({ + action: "issue.thread_interaction_accepted", actorId: "system:pr-merged", details: expect.objectContaining({ interactionId: card.id }), + })])); + expect(errors).toEqual([]); + }); + + it("reconciles an open candidate at its deadline, never early, then stops after merge", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + const dueAt = coordinator!.nextWakeAt()!; + expect(dueAt).toBeGreaterThan(Date.now()); + await vi.advanceTimersByTimeAsync(dueAt - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("manual cancellation removes the candidate deadline without provider work", async () => { + provider.mockResolvedValue("open"); await start(); const card = await create(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await service().withdrawInteraction({ id: issueId, companyId }, card.id, {}, { agentId }); + await quiet(); expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).not.toHaveBeenCalled(); + expect((await stored(card.id)).status).toBe("cancelled"); + }); + + it("does not accept before caller-owned outer commit and forgets a rolled-back create", async () => { + await start(); await quiet(); + let release!: () => void, created!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const inserted = new Promise(resolve => { created = resolve; }); + let id = ""; + const transaction = db.transaction(async tx => { + id = (await create(tx as unknown as Db)).id; created(); await barrier; + }); + await inserted; + await vi.advanceTimersByTimeAsync(10_000); expect(provider).not.toHaveBeenCalled(); + expect(await stored(id)).toBeUndefined(); release(); await transaction; + await waitFor(async () => expect((await stored(id)).status).toBe("accepted")); await quiet(); + const prior = provider.mock.calls.length; + await expect(db.transaction(async tx => { await create(tx as unknown as Db, 41); throw new Error("rollback"); })).rejects.toThrow("rollback"); + await quiet(); expect(provider).toHaveBeenCalledTimes(prior); expect(errors).toEqual([]); + }); + + it("retains provider failures across drain, resumes without overlap, and clears retry after success", async () => { + provider.mockRejectedValue(new Error("GitHub unavailable")); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); await settle(); + expect((await stored(card.id)).status).toBe("pending"); + draining = true; + const reads = vi.spyOn(db, "select"), calls = provider.mock.calls.length; + await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(calls); reads.mockRestore(); + draining = false; provider.mockResolvedValue("merged"); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + }); + + it("keeps first work committed during drain and startup during standby until resume", async () => { + standby = true; await start(); const card = await create(); + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + standby = false; draining = true; + await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled(); + draining = false; + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + }); + + it("reconstructs pending cards on restart and ignores terminal or hidden tasks", async () => { + provider.mockResolvedValue("open"); const card = await create(); await start(); + await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle(); + await coordinator!.stop(); coordinator = undefined; provider.mockResolvedValue("merged"); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + const hidden = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { hiddenAt: new Date() }); + return card; + }); + await quiet(); expect((await stored(hidden.id)).status).toBe("pending"); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(hidden.id)).status).toBe("accepted")); await quiet(); + }); + + it("preserves a real chat binding's durable continuation outbox on automatic acceptance", async () => { + const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID(); + await db.insert(toolApplications).values({ id: applicationId, companyId, applicationKey: `chat:slack:${endpointId}`, name: "Slack", type: "chat", status: "active" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: endpointId, + connectionPurpose: "channel", transport: "chat_sdk", status: "active", enabled: true }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "slack", publicId: endpointId, + assignedAgentId: agentId, status: "active" }); + const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId, issueId, + externalConversationId: "merge-channel", externalThreadId: "merge-thread", externalLabel: "Merge thread", state: "active" }).returning(); + await start(); await quiet(); const card = await create(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(await db.select().from(chatActions).where(eq(chatActions.kind, "interaction_wakeup"))).toEqual([ + expect.objectContaining({ companyId, endpointId, conversationId: conversation.id, providerActionId: `interaction_wakeup:${card.id}`, + payload: expect.objectContaining({ interactionId: card.id, issueId, agentId, interactionStatus: "accepted" }) }), + ]); + expect(await db.select().from(chatPublications)).toEqual(expect.arrayContaining([ + expect.objectContaining({ idempotencyKey: `interaction:${card.id}:${endpointId}`, state: "cancelled" }), + ])); + expect(errors).toEqual([]); + }); + + it("does not lose a newer card created during provider observation or wake the superseded card", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const old = await create(); + await vi.advanceTimersByTimeAsync(1000); await observing; + const current = await create(db, 41); + expect((await stored(old.id)).status).toBe("expired"); + release(); + await waitFor(async () => expect((await stored(current.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); + expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${current.id}:accepted` })); + expect(errors).toEqual([]); + }); + + it("keeps unknown provider results as retryable debt and does not act before retry", async () => { + provider.mockResolvedValue("unknown"); await start(); const card = await create(); + await waitFor(() => expect(errors).toHaveLength(1)); + const next = coordinator!.nextWakeAt()!; + await vi.advanceTimersByTimeAsync(next - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1); + provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(2); + }); + + it("honors terminal issues, pure merge URL targets and continuationPolicy none", async () => { + await start(); await quiet(); + const target = await service().create({ id: issueId, companyId }, { ...input(), title: "Merge the linked PR?", continuationPolicy: "none", + payload: { version: 1, prompt: "Merge the PR?", target: { type: "custom", key: "github-pr-39", href: "https://github.com/paperclipai/paperclip/pull/39" } } }, { agentId }); + await waitFor(async () => expect((await stored(target.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + provider.mockClear(); + const terminal = await db.transaction(async tx => { + const card = await create(tx as unknown as Db, 41); + await issueService(tx as unknown as Db).update(issueId, { status: "cancelled" }); + return card; + }); + await quiet(); expect((await stored(terminal.id)).status).toBe("expired"); + expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + }); + + it("rechecks hidden state under the acceptance lock when visibility changes during a provider read", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + await issueService(db).update(issueId, { hiddenAt: new Date() }); release(); await quiet(); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + await issueService(db).update(issueId, { hiddenAt: null }); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("unrelated pending confirmations do not retain worker SQL or provider work", async () => { + await start(); await quiet(); + await service().create({ id: issueId, companyId }, { ...input(), title: "Approve the plan?", payload: { version: 1, prompt: "Approve the plan?" } }, { agentId }); + await quiet(); expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled(); + expect(errors).toEqual([]); + }); + + it("awaits an in-flight observation at shutdown and reconstructs its unaccepted card on restart", async () => { + let release!: () => void, observed!: () => void; + const barrier = new Promise(resolve => { release = resolve; }); + const observing = new Promise(resolve => { observed = resolve; }); + provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; }); + await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing; + let stopped = false; + const stopping = coordinator!.stop().then(() => { stopped = true; }); + await delay(10); expect(stopped).toBe(false); release(); await stopping; coordinator = undefined; + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + const reads = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(2 * MERGE_CONFIRMATION_RECONCILE_MS); expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); + await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it.each(["headers", "body", "not-found probe", "public visibility probe"] as const)("cancels an actual unresponsive HTTP %s read promptly at stop", async phase => { + vi.useRealTimers(); + const card = await create(); + let observed!: () => void, closed!: () => void; + const requested = new Promise(resolve => { observed = resolve; }); + const disconnected = new Promise(resolve => { closed = resolve; }); + const signals: AbortSignal[] = []; + const server = createServer((request, response) => { + if (phase === "not-found probe" && request.url?.includes("/pulls/")) { response.writeHead(404).end(); return; } + response.on("close", closed); + if (phase === "body") { response.writeHead(200, { "content-type": "application/json" }); response.write('{"merged":'); } + observed(); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const resolveState = createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 10_000, + credentialProvider: () => phase === "public visibility probe" ? { token: "fixture", source: "server_env", secretName: null } : null, + fetch: (url, init) => { signals.push(init!.signal!); return fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init); }, + }); + const real = issueThreadInteractionService(db, { resolvePullRequestState: resolveState, wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startupAt = performance.now(); + const worker = registerMergeConfirmationWork(coordinator, real, () => true); + expect(performance.now() - startupAt).toBeLessThan(100); + await Promise.race([requested, delay(2000).then(() => { throw new Error("HTTP observation did not start"); })]); + const stoppingAt = performance.now(); + await coordinator.stop(); coordinator = undefined; + expect(performance.now() - stoppingAt).toBeLessThan(1000); + await worker.ready; + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("HTTP request remained open after stop"); })]); + expect(signals.at(-1)?.aborted).toBe(true); + expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("times out a real unresponsive HTTP read and recovers its durable candidate on restart", async () => { + vi.useRealTimers(); const card = await create(); + let serveMerge = false, closed!: () => void; + const disconnected = new Promise(resolve => { closed = resolve; }); + const server = createServer((_request, response) => { + if (serveMerge) { response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ merged: true, state: "closed" })); return; } + response.on("close", closed); + }); + await new Promise(resolve => { server.listen(0, "127.0.0.1", resolve); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected test HTTP address"); + const real = issueThreadInteractionService(db, { resolvePullRequestState: createPullRequestMergeStateResolver(db, { + requestTimeoutMs: 100, tokenProvider: () => null, + fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init), + }), wakeup }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + try { + const startedAt = performance.now(); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect(performance.now() - startedAt).toBeLessThan(2000); + await Promise.race([disconnected, delay(1000).then(() => { throw new Error("Timed out HTTP request remained open"); })]); + expect(errors).toHaveLength(1); expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); + expect(coordinator.nextWakeAt()).not.toBeNull(); + await coordinator.stop(); coordinator = undefined; serveMerge = true; + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) }); + await registerMergeConfirmationWork(coordinator, real, () => true).ready; + expect((await stored(card.id)).status).toBe("accepted"); expect(wakeup).toHaveBeenCalledTimes(1); + } finally { + await coordinator?.stop(); coordinator = undefined; + server.closeAllConnections(); await new Promise(resolve => { server.close(() => resolve()); }); + } + }); + + it("pages beyond a full pending-card page without duplicating the merge parser in SQL", async () => { + const card = await create(); + const id = BigInt(`0x${card.id.replaceAll("-", "")}`); + await db.insert(issueThreadInteractions).values(Array.from({ length: 100 }, (_, index) => { + const hex = (id - BigInt(index + 1)).toString(16).padStart(32, "0"); + return { id: `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`, + companyId, issueId, kind: "request_confirmation", title: "Approve the plan?", payload: { version: 1 as const, prompt: "Approve the plan?" } }; + })); + await start(); expect(provider).not.toHaveBeenCalled(); + await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet(); + expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]); + }); + + it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => { + provider.mockResolvedValue("open"); const own = await create(); + const foreignCompany = randomUUID(), foreignIssue = randomUUID(); + await db.insert(companies).values({ id: foreignCompany, name: "Other company", issuePrefix: "OTH" }); + await db.insert(issues).values({ id: foreignIssue, companyId: foreignCompany, title: "Other", status: "todo" }); + const foreign = await service().create({ id: foreignIssue, companyId: foreignCompany }, input(), { userId: "board" }); + const local = service(); + const hint = [{ companyId, owner: "PAPERCLIPAI", repo: "paperclip", number: 39 }]; + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(1); + expect((await stored(own.id)).status).toBe("accepted"); expect((await stored(foreign.id)).status).toBe("pending"); + expect(provider).not.toHaveBeenCalled(); + expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(0); expect(wakeup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index 2437d9711b5..9d16ec519ff 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -1,3 +1,12 @@ +import { notifyNativeEnvironmentSettlement } from "../services/native-environment-settlement-notifications.js"; +import { notifyRetainedNativeSessionCleanup, onRetainedNativeSessionCleanupRelease, releaseRetainedNativeSessionCleanup } from "../services/retained-native-session-cleanup-notifications.js"; +import { environmentService } from "../services/environments.js"; +import { agentService } from "../services/agents.js"; +import { publishNativeHarnessBackup } from "../services/native-runtime/native-harness-backup-publication.js"; +import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js"; +import { noteNativeHarnessBackupChange } from "../services/native-harness-backup-expiration-notifications.js"; +import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js"; +import { issueService } from "../services/issues.js"; import { applyPendingMigrations } from "@paperclipai/db"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { registerNativeRunnerEvidenceRetentionWork } from "../services/native-runtime/native-runner-evidence-retention-work.js"; @@ -8,7 +17,7 @@ import os from "node:os"; import path from "node:path"; import { promises as fs } from "node:fs"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { agents, agentTaskSessions, @@ -55,7 +64,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => { let coordinator: ReturnType | undefined; let worker: ReturnType | undefined; let admitted = true; + let backupWorker: ReturnType | undefined; + const backupErrors: unknown[] = []; afterEach(async () => { + await backupWorker?.stop(); backupWorker = undefined; backupErrors.length = 0; worker?.unsubscribe(); await coordinator?.stop(); coordinator = undefined; worker = undefined; admitted = true; vi.useRealTimers(); vi.restoreAllMocks(); await db.delete(agentTaskSessions); @@ -75,6 +87,7 @@ describeEmbeddedPostgres("native harness backup expiration", () => { }); async function seedSession(input: { + companyId?: string; agentId?: string; issueStatus?: string; closedDaysAgo?: number; runStatus?: string; @@ -84,9 +97,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => { cleanupActive?: boolean; } = {}) { if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-")); - const companyId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` }); - const [agent] = await db.insert(agents).values({ companyId, name: "Coder" }).returning(); + const companyId = input.companyId ?? randomUUID(); + if (!input.companyId) await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` }); + const [agent] = input.agentId ? [{ id: input.agentId }] + : await db.insert(agents).values({ companyId, name: "Coder" }).returning(); const issueStatus = input.issueStatus ?? "done"; const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS); const [issue] = await db.insert(issues).values({ @@ -558,6 +572,444 @@ describeEmbeddedPostgres("native harness backup expiration", () => { reads.mockRestore(); writes.mockRestore(); directories.mockRestore(); } + async function startBackups(options: Parameters[1] = {}) { + if (!stateBase) await emptyStateBase(); + if (!vi.isFakeTimers()) { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + vi.setSystemTime(now); + } + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => true, + onError: error => backupErrors.push(error) }); + backupWorker = registerNativeHarnessBackupExpirationWork(coordinator, db, + { stateBase, isSessionBusy: () => false, ...options }, () => admitted); + await backupWorker.ready; + } + + it.each([false, true])("backup deadline worker leaves empty and retained terminal history quiet (%s)", async history => { + await emptyStateBase(); + if (history) { + const session = await seedSession(); + await fs.rm(path.join(session.root, "failover-backups"), { recursive: true }); + await db.insert(heartbeatRuns).values(Array.from({ length: 500 }, () => ({ + companyId: session.companyId, agentId: session.agentId, status: "succeeded", issueId: session.issueId, + nativeIssueId: session.issueId, nativeSessionId: randomUUID(), finishedAt: now, + }))); + for (let index = 0; index < 250; index++) await fs.mkdir(path.join(stateBase, + createHash("sha256").update(String(index)).digest("hex"))); + } + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + await startBackups({ maxEntries: 20 }); await settled(); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); + reads.mockRestore(); writes.mockRestore(); await quiet(); + expect(backupErrors).toEqual([]); + }); + + it("ordinary task close arms exact backup retention, deletes physical bytes and resume pointer, then stays quiet", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); await quiet(); + const closed = await issueService(db).update(session.issueId, { status: "done", companyGuard: session.companyId }); + const deadline = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(deadline); await quiet(60_000); + await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1); + expect(await exists(session.current)).toBe(true); + expect(await taskSessionCount(session.agentId)).toBe(1); + await vi.advanceTimersByTimeAsync(1); await settled(); + expect(await exists(session.current)).toBe(false); + expect(await taskSessionCount(session.agentId)).toBe(0); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toHaveLength(1); + expect(backupErrors).toEqual([]); await quiet(); + }); + + it("ordinary backup publication after quiet schedules its actual file age and leaves a durable cleanup result", async () => { + const session = await seedSession(); + const backupRoot = path.dirname(session.current); + const pendingRoot = path.join(backupRoot, "pending-publication"); + await fs.rename(session.current, pendingRoot); + await startBackups(); await settled(); await quiet(); + const publishedAt = Date.now(); + await fs.utimes(pendingRoot, new Date(publishedAt), new Date(publishedAt)); + await publishNativeHarnessBackup({ db, companyId: session.companyId, scopeDigest: session.scope, backupRoot, pendingRoot }); + const stat = await fs.stat(session.current); + const due = stat.mtime.getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); expect(await taskSessionCount(session.agentId)).toBe(0); + expect(await fs.readdir(backupRoot)).toEqual([]); await quiet(); + }); + + it("a committed earlier close timestamp replaces the later deadline without early cleanup", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + const earlier = new Date(now.getTime() - 6 * DAY_MS); + await issueService(db).update(session.issueId, { completedAt: earlier }); + const due = earlier.getTime() + 7 * DAY_MS + 1; + await settled(due); await quiet(60_000); + await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("holds nested ordinary close activation until the outer transaction commits", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); + let release!: () => void, written!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const staged = new Promise(resolve => { written = resolve; }); + const transaction = db.transaction(async outer => { + await outer.transaction(nested => issueService(db).update(session.issueId, { status: "done" }, nested)); + written(); await gate; + }); + await staged; + await vi.advanceTimersByTimeAsync(60_000); + expect(await exists(session.current)).toBe(true); + expect(await taskSessionCount(session.agentId)).toBe(1); + release(); await transaction; + const [closed] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("preserves failed cleanup retry across drain, then finishes its physical tombstone on resume", async () => { + const session = await seedSession(); const remove = fs.rm; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (String(args[0]).includes(".expired-")) throw new Error("controlled drain retry failure"); + return remove(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); admitted = false; fault.mockRestore(); + await quiet(60_000); + expect((await fs.readdir(path.dirname(session.current))).some(name => name.startsWith(".expired-"))).toBe(true); + admitted = true; await settled(); + expect(await fs.readdir(path.dirname(session.current))).toEqual([]); await quiet(); + }); + + it("reopen cancels a saved deadline and a later ordinary close restores retention", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + await issueService(db).update(session.issueId, { status: "todo" }); + await settled(); await quiet(8 * DAY_MS); expect(await exists(session.current)).toBe(true); + const closed = await issueService(db).update(session.issueId, { status: "done" }); + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("reconstructs a retained backup deadline on restart without changing business timestamps", async () => { + const session = await seedSession({ closedDaysAgo: 2 }); + const [before] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups(); await settled(due); + await backupWorker!.stop(); await coordinator!.stop(); + await startBackups(); await settled(due); + const [after] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + expect(after!.updatedAt).toEqual(before!.updatedAt); expect(after!.completedAt).toEqual(before!.completedAt); + await quiet(); + }); + + it("reports dry-run backups once and rechecks only after source invalidation", async () => { + const session = await seedSession(); const report = vi.spyOn(logger, "info"); + await startBackups({ dryRun: true }); await settled(); + expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(1); + expect(await taskSessionCount(session.agentId)).toBe(1); expect(await exists(session.current)).toBe(true); + await quiet(); + noteNativeHarnessBackupChange(db, { companyId: session.companyId, scopeDigest: session.scope }); + await settled(); + expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(2); + await quiet(); + }); + + it("company-only notifications inspect only the indexed company's blocked backup sources", async () => { + const own = await seedSession({ issueStatus: "todo" }); + const other = await seedSession({ issueStatus: "todo" }); + await startBackups({ maxEntries: 1 }); await settled(); await quiet(); + const sql: Array<{ query: string; parameters: unknown[] }> = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query, parameters) => { sql.push({ query, parameters }); }; + const directories = vi.spyOn(fs, "readdir"); + try { + const closed = await issueService(db).update(own.issueId, { status: "done" }); + await settled(new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1); + const unrelated = [other.companyId, other.agentId, other.runId, other.sessionId, other.issueId]; + expect(sql.filter(query => query.parameters.some(parameter => + unrelated.some(id => String(parameter).includes(id))))).toEqual([]); + expect(directories.mock.calls.some(call => call[0] === stateBase + || String(call[0]).startsWith(other.root))).toBe(false); + expect(await exists(other.current)).toBe(true); + } finally { db.$client.options.debug = previous; directories.mockRestore(); } + await quiet(); + }); + + it("coalesces repeated task writes to one retained root in a large company with the same historical agent", async () => { + const own = await seedSession({ issueStatus: "todo" }); + const unrelated = []; + for (let index = 0; index < 80; index++) unrelated.push(await seedSession({ companyId: own.companyId, + agentId: own.agentId, issueStatus: index % 2 ? "todo" : "done", closedDaysAgo: 2, + runStatus: index % 3 ? "succeeded" : "running" })); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups({ isSessionBusy: digest => unrelated.some((entry, index) => index % 4 === 0 && entry.scope === digest) }); + await settled(due); await quiet(60_000); + const reads = vi.spyOn(fs, "readFile"); + const queries: Array<{ query: string; parameters: unknown[] }> = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query, parameters) => { queries.push({ query, parameters }); }; + try { + await db.transaction(async tx => { + await issueService(db).update(own.issueId, { status: "done" }, tx); + for (let index = 0; index < 12; index++) await issueService(db).update(own.issueId, + { completedAt: new Date(now.getTime() - DAY_MS) }, tx); + }); + const writes = queries.length; + await settled(due); + expect(reads.mock.calls.map(call => String(call[0]))).toEqual([path.join(own.root, "control-plane", "control-plane-state.json")]); + expect(queries.length - writes).toBeLessThanOrEqual(20); + const otherIds = unrelated.flatMap(entry => [entry.issueId, entry.runId, entry.sessionId]); + expect(queries.filter(query => query.parameters.some(parameter => otherIds.some(id => String(parameter).includes(id))))).toEqual([]); + } finally { reads.mockRestore(); db.$client.options.debug = previous; } + await issueService(db).update(own.issueId, { status: "todo" }); await settled(due); + await quiet(); expect(await exists(own.current)).toBe(true); + }); + + it("indexes protected session relations so a new run, historical task close and exact process release reactivate only its backup", async () => { + const own = await seedSession({ closedDaysAgo: 2 }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + const [linked] = await db.insert(issues).values({ companyId: own.companyId, title: "New linked task", status: "todo" }).returning(); + const due = now.getTime() + 5 * DAY_MS + 1; + await startBackups(); await settled(due); await quiet(60_000); + const reads = vi.spyOn(fs, "readFile"); + const runId = randomUUID(); + try { + await db.transaction(async tx => { + await notifyNativeEnvironmentSettlement(tx, { companyId: own.companyId, runId, issueId: linked!.id }); + await tx.insert(heartbeatRuns).values({ id: runId, companyId: own.companyId, agentId: own.agentId, + nativeSessionId: own.sessionId, nativeIssueId: linked!.id, issueId: other.issueId, status: "running" }); + }); + await settled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + expect(reads).toHaveBeenCalled(); reads.mockClear(); + await issueService(db).update(linked!.id, { status: "done" }); await settled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); reads.mockClear(); + await db.transaction(async tx => { + await notifyRetainedNativeSessionCleanup(tx, { companyId: own.companyId, runId, agentId: own.agentId }); + await tx.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + }); + const [closed] = await db.select().from(issues).where(eq(issues.id, linked!.id)); + const later = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(later); reads.mockClear(); + const cleanupScope = { companyId: own.companyId, agentId: own.agentId }; + const observed = vi.fn(); const stop = onRetainedNativeSessionCleanupRelease(db, observed); + releaseRetainedNativeSessionCleanup(db, cleanupScope, { companyId: own.companyId, runId }); + expect(observed).toHaveBeenCalledWith(cleanupScope, { companyId: own.companyId, runId }); stop(); + await settled(later); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + expect(reads).toHaveBeenCalled(); + await issueService(db).update(linked!.id, { status: "todo" }); await settled(); + } finally { reads.mockRestore(); } + expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("ordinary lease release without a run uses its source identity instead of other retained roots", async () => { + const own = await seedSession({ leaseStatus: "retained" }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + await startBackups(); await settled(); await quiet(60_000); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId)); + const reads = vi.spyOn(fs, "readFile"); + try { + await environmentService(db).releaseLease(lease!.id, "expired"); await settled(); + expect(reads).toHaveBeenCalled(); + expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + } finally { reads.mockRestore(); } + expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("retains the future deadline when a lease metadata edit removes its last backup reference", async () => { + const own = await seedSession({ leaseStatus: "retained", closedDaysAgo: 2 }); + const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" }); + await startBackups(); await settled(); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId)); + const reads = vi.spyOn(fs, "readFile"); const due = now.getTime() + 5 * DAY_MS + 1; + try { + await environmentService(db).updateLeaseMetadata(lease!.id, {}); await settled(due); + expect(reads).toHaveBeenCalled(); expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); + } finally { reads.mockRestore(); } + await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet(); + }); + + it("allows an ordinary task reopen while backup byte counting is held outside the retirement lock", async () => { + const own = await seedSession(); const lstat = fs.lstat; + const entered = Promise.withResolvers(), release = Promise.withResolvers(); + const held = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + if (String(args[0]) === path.join(own.current, "codex-home", "history.jsonl")) { + entered.resolve(); await release.promise; + } + return lstat(...args); + }); + const startup = startBackups(); await entered.promise; + try { + await db.transaction(async tx => { + await tx.execute(sql`set local lock_timeout = '500ms'`); + await issueService(db).update(own.issueId, { status: "todo" }, tx); + }); + } finally { release.resolve(); await startup; held.mockRestore(); } + await settled(); expect(await exists(own.current)).toBe(true); expect(backupErrors).toEqual([]); await quiet(); + }); + + it.each(["EIO", "EACCES"])("retains a failed identity read as real retry debt (%s)", async code => { + const own = await seedSession(); const read = fs.readFile; + const fault = vi.spyOn(fs, "readFile").mockImplementation(async (...args) => { + if (String(args[0]) === path.join(own.root, "control-plane", "control-plane-state.json")) { + throw Object.assign(new Error("controlled identity I/O failure"), { code }); + } + return read(...args); + }); + await startBackups(); expect(backupErrors).toHaveLength(1); + const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThanOrEqual(29_000); + expect(await exists(own.current)).toBe(true); fault.mockRestore(); await quiet(5000); + await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled(); + expect(await exists(own.current)).toBe(false); expect(await taskSessionCount(own.agentId)).toBe(0); await quiet(); + }); + + it.each(["EIO", "EACCES"])("retains expired-only cleanup after a directory read failure (%s)", async code => { + const session = await seedSession(); + const backupRoot = path.dirname(session.current); + const expired = path.join(backupRoot, `.expired-${randomUUID()}`); + await fs.rename(session.current, expired); + const read = fs.readdir; + const fault = vi.spyOn(fs, "readdir").mockImplementation(async (...args) => { + if (String(args[0]) === backupRoot) throw Object.assign(new Error("controlled cleanup listing failure"), { code }); + return read(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); + const retry = coordinator!.nextWakeAt(); + expect(retry).not.toBeNull(); + expect(retry! - Date.now()).toBeGreaterThanOrEqual(29_000); + expect(await exists(session.current)).toBe(false); + expect(await exists(expired)).toBe(true); + fault.mockRestore(); + await quiet(5000); + await vi.advanceTimersByTimeAsync(retry! - Date.now()); await settled(); + expect(await fs.readdir(backupRoot)).toEqual([]); + await quiet(); + }); + + it("reconstructs future retention without issue row locks", async () => { + await seedSession({ closedDaysAgo: 2 }); const queries: string[] = []; + const previous = db.$client.options.debug; + db.$client.options.debug = (_connection, query) => { queries.push(query); }; + try { + await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1); + expect(queries.some(query => /for\s+share/i.test(query))).toBe(false); + } finally { db.$client.options.debug = previous; } + await quiet(); + }); + + it("rejects destructive backup scopes outside the controller digest namespace", async () => { + await emptyStateBase(); const remove = vi.spyOn(fs, "rm"); + await expect(service().expireBackup("../outside")).rejects.toThrow("Invalid native harness backup scope"); + expect(remove).not.toHaveBeenCalled(); remove.mockRestore(); + }); + + it.each([false, true])("finishes cleanup journals after ordinary agent deletion (restart %s)", async restart => { + const session = await seedSession(); const remove = fs.rm; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (String(args[0]).includes(".expired-")) throw new Error("controlled orphan cleanup failure"); + return remove(...args); + }); + await startBackups(); + expect(backupErrors).toHaveLength(1); + const retryAt = coordinator!.nextWakeAt()!; + const backupRoot = path.dirname(session.current); + expect((await fs.readdir(backupRoot)).some(name => name.startsWith(".expired-"))).toBe(true); + if (restart) { await backupWorker!.stop(); await coordinator!.stop(); } + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId)); + await agentService(db).remove(session.agentId); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toEqual([]); + fault.mockRestore(); + if (restart) await startBackups(); else await vi.advanceTimersByTimeAsync(retryAt - Date.now()); + await settled(); + expect(await fs.readdir(backupRoot)).toEqual([]); expect(await taskSessionCount(session.agentId)).toBe(0); + await quiet(); + }); + + it("restores an orphan retiring journal without expiring the recovered backup or unrelated names", async () => { + const session = await seedSession(); const backupRoot = path.dirname(session.current); + const retiring = path.join(backupRoot, `.retiring-${randomUUID()}`); + await fs.rename(session.current, retiring); + await fs.mkdir(path.join(backupRoot, ".expired-unrecognized")); + await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId)); + await agentService(db).remove(session.agentId); + await fs.rm(path.join(session.root, "control-plane"), { recursive: true }); + await startBackups(); await settled(); + expect(await exists(session.current)).toBe(true); expect(await exists(retiring)).toBe(false); + expect(await exists(path.join(session.current, "manifest.json"))).toBe(true); + expect(await exists(path.join(backupRoot, ".expired-unrecognized"))).toBe(true); + await quiet(); + }); + + it("retains live lease references and only removes the owning company's backup after lease release", async () => { + const first = await seedSession({ leaseStatus: "retained" }); + const other = await seedSession({ leaseStatus: "retained" }); + await startBackups(); await settled(); await quiet(); + await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "expired" }) + .where(eq(environmentLeases.companyId, first.companyId)), { companyId: first.companyId }); + await settled(); expect(await exists(first.current)).toBe(false); expect(await exists(other.current)).toBe(true); + expect(await taskSessionCount(other.agentId)).toBe(1); await quiet(); + }); + + it("retries a partial physical deletion from its tombstone without recreating a resume pointer", async () => { + const session = await seedSession(); const remove = fs.rm; let failed = false; + const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => { + if (!failed && String(args[0]).includes(".expired-")) { failed = true; throw new Error("controlled filesystem delete failure"); } + return remove(...args); + }); + await startBackups(); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(backupErrors).toHaveLength(1); }); + const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThan(20_000); + expect(await taskSessionCount(session.agentId)).toBe(0); expect(await exists(session.current)).toBe(false); + fault.mockRestore(); await quiet(5000); + await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled(); + expect(await fs.readdir(path.join(session.root, "failover-backups"))).toEqual([]); await quiet(); + }); + + it("retains a first ordinary close during drain and dispatches it after resume", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); admitted = false; + const closed = await issueService(db).update(session.issueId, { status: "done" }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }); + await quiet(60_000); admitted = true; + const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1; + await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled(); + expect(await exists(session.current)).toBe(false); await quiet(); + }); + + it("honors outer rollback for nested ordinary close and activates only on committed work", async () => { + const session = await seedSession({ issueStatus: "todo" }); + await startBackups(); await settled(); + await expect(db.transaction(async outer => { + await outer.transaction(async nested => { + await issueService(db).update(session.issueId, { status: "done" }, nested); + }); + expect(await exists(session.current)).toBe(true); + throw new Error("rollback outer close"); + })).rejects.toThrow("rollback outer close"); + await settled(); await quiet(8 * DAY_MS); + expect(await exists(session.current)).toBe(true); + const [saved] = await db.select().from(issues).where(eq(issues.id, session.issueId)); + expect(saved!.status).toBe("todo"); + }); + + it("pages multiple company backups, preserves a busy one and wakes from process release", async () => { + const sessions = [await seedSession(), await seedSession(), await seedSession()]; let busy = true; + await startBackups({ maxEntries: 1, maxRemovals: 1, isSessionBusy: digest => busy && digest === sessions[1]!.scope }); + await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); + expect(await exists(sessions[0]!.current)).toBe(false); expect(await exists(sessions[2]!.current)).toBe(false); + expect(backupErrors.length).toBeGreaterThan(0); + }); + expect(await exists(sessions[1]!.current)).toBe(true); busy = false; + noteNativeHarnessBackupChange(db, { companyId: sessions[1]!.companyId }); + await settled(); expect(await exists(sessions[1]!.current)).toBe(false); await quiet(); + }); + it("leaves empty evidence and retained backups quiet without invoking backup SQL", async () => { const session = await seedSession(); const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); diff --git a/server/src/__tests__/routine-execution-status-work.test.ts b/server/src/__tests__/routine-execution-status-work.test.ts new file mode 100644 index 00000000000..0a6ff9339cb --- /dev/null +++ b/server/src/__tests__/routine-execution-status-work.test.ts @@ -0,0 +1,314 @@ +import { issueTreeControlService } from "../services/issue-tree-control.js"; +import { writeExecutionControl } from "../services/execution-control-notifications.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, companies, createDb, heartbeatRuns, issues, pluginManagedResources, plugins, routineRuns, type Db } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { notifyRoutineExecutionStatus } from "../services/routine-execution-status-notifications.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "../services/routine-run-status.js"; +import { routineService } from "../services/routines.js"; +import { issueService } from "../services/issues.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; +import { createHeartbeatRunState } from "../services/heartbeat/run-state.js"; +import { createHeartbeatRunPreparation } from "../services/heartbeat/run-preparation.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_WORK_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("routine execution status work with PostgreSQL", () => { + let db: Db, temporary: Awaited> | undefined; + let coordinator: ReturnType | undefined; + let enabled = true, draining = false, standby = false; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("routine-status-work-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); + }, 60_000); + beforeEach(() => { + enabled = true; draining = false; standby = false; errors.length = 0; + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); + }); + afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); }); + const canRun = () => enabled && !draining && !standby; + async function start(service = routineExecutionStatusService(db)) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, canReconcile: () => !standby, + onError: error => errors.push(error) }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, canRun); + await worker.ready; + return worker; + } + async function settled() { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); } + async function quiet() { + await settled(); + const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction"); + try { + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + } finally { select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); } + } + async function fixture() { + const companyId = randomUUID(), agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Routine status work", issuePrefix: `R${companyId.slice(0, 6)}`, + defaultResponsibleUserId: randomUUID(), requireBoardApprovalForNewAgents: false }); + await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", status: "active", lifecycleState: "ready", adapterType: "codex_local" }); + const service = routineService(db, { heartbeat: { wakeup: async () => null }, runtimeEnv: {} }); + const routine = await service.create(companyId, { title: "Status projection", assigneeAgentId: agentId, + priority: "medium", status: "active", concurrencyPolicy: "always_enqueue", catchUpPolicy: "skip_missed" }, {}); + return { companyId, agentId, routine, service }; + } + async function dispatch(f: Awaited>) { + return f.service.runRoutine(f.routine.id, { source: "manual", idempotencyKey: randomUUID() }); + } + async function saved(id: string) { return (await db.select().from(routineRuns).where(eq(routineRuns.id, id)))[0]; } + async function projected(id: string, status: string) { + await vi.waitFor(async () => expect((await saved(id)).status).toBe(status)); + await settled(); + } + + it("settles empty startup, configured idle, healthy active and retained terminal history without recurring SQL", async () => { + await start(); await quiet(); + const f = await fixture(); await quiet(); + const run = await dispatch(f); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await projected(run.id, "completed"); + const before = await saved(run.id); await quiet(); + expect(await saved(run.id)).toEqual(before); expect(errors).toEqual([]); + }); + it.each(["done", "blocked", "cancelled"])("reactivates from the first ordinary %s transition and preserves input", async status => { + await start(); await quiet(); + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status }); + await projected(run.id, status === "done" ? "completed" : "failed"); + expect((await saved(run.id)).completedAt).toBeInstanceOf(Date); + await quiet(); expect(errors).toEqual([]); + if (status !== "done") { + await issueService(db).update(run.linkedIssueId!, { status: "in_progress" }); + await projected(run.id, "issue_created"); + expect(await saved(run.id)).toMatchObject({ completedAt: null, failureReason: null }); + await quiet(); + } + }); + it("reconstructs legacy terminal debt and reopened transient failures but leaves skipped and coalesced history unchanged", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, first.linkedIssueId!)); + await db.update(routineRuns).set({ status: "failed", failureReason: "Execution issue moved to blocked", completedAt: new Date() }).where(eq(routineRuns.id, second.id)); + const terminal = await db.insert(routineRuns).values(["skipped", "coalesced"].map(status => ({ companyId: f.companyId, + routineId: f.routine.id, source: "manual", linkedIssueId: first.linkedIssueId, status, completedAt: new Date() }))).returning(); + await start(); await projected(first.id, "completed"); await projected(second.id, "issue_created"); + for (const row of terminal) expect(await saved(row.id)).toEqual(row); + await quiet(); expect(errors).toEqual([]); + }); + it("settles the declared custom origin of a real plugin-managed dispatch and preserves other companies", async () => { + const f = await fixture(), other = await fixture(); + const pluginKey = `routine-status-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "status", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }); + const run = await dispatch(f), otherRun = await dispatch(other); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + await start(); await quiet(); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); await projected(run.id, "completed"); + const before = await saved(run.id); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); }); + await settled(); await quiet(); + expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]); + }); + it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => { + const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`; + const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0", + manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test", + author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning(); + const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey, + resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning(); + const run = await dispatch(f); + expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId); + if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id)); + else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => { + const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort(); + const issueIds = ids.map(() => randomUUID()); + await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution", + status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id }))); + await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id, + source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" }))); + const service = routineExecutionStatusService(db), load = service.load; + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => { + const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page; + }); + const starting = start(service); await enteredGate; + await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting; + await vi.advanceTimersByTimeAsync(3000); + await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true)); + expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true); + expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true); + await quiet(); expect(errors).toEqual([]); + }); + it("scopes shared execution writes to their company and settles paused-company projection debt", async () => { + const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other); + const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load"); + await start(service); await quiet(); load.mockClear(); + await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!)); + await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus, + tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId }); + await projected(run.id, "completed"); + expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true); + expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet(); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); }); + await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("projects subtree cancel and restore without waking agents, then becomes quiet", async () => { + const f = await fixture(), root = await dispatch(f), child = await dispatch(f); + await issueService(db).update(child.linkedIssueId!, { parentId: root.linkedIssueId }); + const tree = issueTreeControlService(db), actor = { actorType: "system" as const, actorId: "routine-status-test" }; + await start(); await quiet(); + const cancel = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "cancel", actor }); + expect((await tree.cancelIssueStatusesForHold(f.companyId, root.linkedIssueId!, cancel.hold.id)).updatedIssueIds.sort()) + .toEqual([root.linkedIssueId, child.linkedIssueId].sort()); + await projected(root.id, "failed"); await projected(child.id, "failed"); await quiet(); + const restore = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "restore", actor }); + const restored = await tree.restoreIssueStatusesForHold(f.companyId, root.linkedIssueId!, restore.hold.id, { actor }); + expect(restored.updatedIssues).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: root.linkedIssueId, status: "todo" }), + expect.objectContaining({ id: child.linkedIssueId, status: "todo" }), + ])); + await projected(root.id, "issue_created"); await projected(child.id, "issue_created"); + expect(await saved(root.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await saved(child.id)).toMatchObject({ completedAt: null, failureReason: null }); + expect(await db.select().from(heartbeatRuns)).toEqual([]); + await quiet(); expect(errors).toEqual([]); + }); + it("serializes a direct status hook with later task changes before projecting the routine", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const holding = db.transaction(async tx => { + await tx.select().from(routineRuns).where(eq(routineRuns.id, run.id)).for("update"); + entered(); await releaseGate; + }); + let direct: Promise | undefined, changing: Promise | undefined; + let directPid: number | undefined; + try { + await enteredGate; direct = f.service.syncRunStatusForIssue(run.linkedIssueId!); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select pid from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and (query like 'update "routine_runs"%' or query like 'select %"routine_runs"%')`); + expect(rows).toHaveLength(1); directPid = Number(rows[0]!.pid); + }, { timeout: 5000 }); + await expect(db.transaction(tx => tx.select({ id: issues.id }).from(issues) + .where(eq(issues.id, run.linkedIssueId!)).for("update", { noWait: true }))) + .rejects.toMatchObject({ cause: expect.objectContaining({ code: "55P03" }) }); + changing = issueService(db).update(run.linkedIssueId!, { status: "blocked" }); + await vi.waitFor(async () => { + const rows = await db.execute(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and wait_event_type = 'Lock' + and ${directPid}::int = any(pg_blocking_pids(pid))`); + expect(Number(rows[0]?.count)).toBeGreaterThan(0); + }, { timeout: 5000 }); + release(); await holding; await direct; await changing; + await projected(run.id, "failed"); await quiet(); expect(errors).toEqual([]); + } finally { release(); await holding; await direct; await changing; } + }); + it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const written = new Promise(resolve => { entered = resolve; }); + const writing = db.transaction(async tx => { + await issueService(db).update(run.linkedIssueId!, { status: "done" }, tx); + entered(); await gate; + }); + try { await written; expect((await saved(run.id)).status).toBe("issue_created"); } + finally { release(); await writing; } + await projected(run.id, "completed"); await quiet(); + const other = await dispatch(f); + await expect(db.transaction(async tx => { + await issueService(db).update(other.linkedIssueId!, { status: "cancelled" }, tx); + throw new Error("outer rollback"); + })).rejects.toThrow("outer rollback"); + await settled(); expect((await saved(other.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); + it("keeps the first write during drain and reconstructs on resume without passive SQL", async () => { + const f = await fixture(), run = await dispatch(f); await start(); await quiet(); + draining = true; + await issueService(db).update(run.linkedIssueId!, { status: "cancelled" }); + await settled(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "failed"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["standby", "disabled"])("does no owner SQL while %s and reconstructs when enabled", async mode => { + const f = await fixture(), run = await dispatch(f); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + if (mode === "standby") standby = true; else enabled = false; + await start(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created"); + standby = false; enabled = true; await vi.advanceTimersByTimeAsync(30_000); + await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]); + }); + it("retries failed exact owners, retains the retry across drain and reconstructs after restart", async () => { + const f = await fixture(), run = await dispatch(f), service = routineExecutionStatusService(db); + const settle = vi.spyOn(service, "settle").mockRejectedValueOnce(new Error("projection unavailable")); + await start(service); + await issueService(db).update(run.linkedIssueId!, { status: "done" }); + await vi.waitFor(() => expect(errors).toHaveLength(1)); + expect((await saved(run.id)).status).toBe("issue_created"); + await vi.advanceTimersByTimeAsync(29_000); expect(settle).toHaveBeenCalledTimes(1); + draining = true; await vi.advanceTimersByTimeAsync(60_000); await quiet(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "completed"); await quiet(); + await coordinator!.stop(); coordinator = undefined; + const other = await dispatch(f); await issueService(db).update(other.linkedIssueId!, { status: "blocked" }); + await start(); await projected(other.id, "failed"); await quiet(); + }); + it("preserves notifications during a sweep, coalesces duplicate hints and never overlaps", async () => { + const f = await fixture(), first = await dispatch(f), second = await dispatch(f), service = routineExecutionStatusService(db); + let entered!: () => void, release!: () => void; + const enteredGate = new Promise(resolve => { entered = resolve; }); + const releaseGate = new Promise(resolve => { release = resolve; }); + const load = service.load; + vi.spyOn(service, "load").mockImplementationOnce(async issueId => { const rows = await load(issueId); entered(); await releaseGate; return rows; }); + const starting = start(service); await enteredGate; + await issueService(db).update(second.linkedIssueId!, { status: "done" }); + await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); }); + release(); await starting; + await projected(second.id, "completed"); expect((await saved(first.id)).status).toBe("issue_created"); + await quiet(); expect(errors).toEqual([]); + }); + it.each(["succeeded", "failed", "cancelled", "interrupted", "timed_out"])("uses real %s run lifecycle changes for activeIssue without an open-issue retry", async status => { + const f = await fixture(), run = await dispatch(f); await start(); + const [execution] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", contextSnapshot: { issueId: run.linkedIssueId } }).returning(); + await db.update(issues).set({ executionRunId: execution.id }).where(eq(issues.id, run.linkedIssueId!)); + expect((await f.service.getDetail(f.routine.id))?.activeIssue?.id).toBe(run.linkedIssueId); + const deps = { + ...createHeartbeatRunState(db), ...createHeartbeatRunPreparation(db), + issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() }, + recovery: { escalateStrandedAssignedIssue: vi.fn() }, companySkills: { completeTestRunForIssue: vi.fn() }, + budgets: { getInvocationBlock: vi.fn(async () => null) }, treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) }, + enqueueWakeup: vi.fn(), getCurrentUserRedactionOptions: vi.fn(async () => ({})), + getAgentInvokability: vi.fn(async () => ({ invokable: true })), emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {}, + } satisfies HeartbeatLifecycleDependencies; + await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(execution.id, status, { finishedAt: new Date() }); + expect((await f.service.getDetail(f.routine.id))?.activeIssue).toBeNull(); + expect((await saved(run.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/routine-execution-status-worker.test.ts b/server/src/__tests__/routine-execution-status-worker.test.ts new file mode 100644 index 00000000000..b62245ce3e2 --- /dev/null +++ b/server/src/__tests__/routine-execution-status-worker.test.ts @@ -0,0 +1,107 @@ +import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js"; + +let coordinator: ReturnType; +let canRun = true; +const errors: unknown[] = []; +const entry = { id: "run", companyId: "company", routineId: "routine", issueId: "issue" }; +beforeEach(() => { + vi.useFakeTimers(); canRun = true; errors.length = 0; + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, canReconcile: () => true, + onError: error => errors.push(error) }); +}); +afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); }); + +it("loads actual debt once and leaves no recurring owner work", async () => { + const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]); +}); + +it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => { + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + service.load.mockResolvedValueOnce([entry]); + worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" })); + await vi.advanceTimersByTimeAsync(1000); + expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2); + expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("retries only the failed owner at backoff and disarms after success", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("locked")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + expect(errors).toHaveLength(1); + const retryAt = coordinator.nextWakeAt(); expect(retryAt).toBe(Date.now() + 30_000); + await vi.advanceTimersByTimeAsync(29_999); expect(service.settle).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); expect(service.settle).toHaveBeenCalledTimes(2); + expect(service.load).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledTimes(2); +}); + +it("retains startup and first work during drain without passive owner calls", async () => { + canRun = false; + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + worker.invalidate(JSON.stringify({ issueId: "issue" })); + await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("keeps a failed owner across drain and retries after resume", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("write failed")) }; + await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready; + canRun = false; await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(service.load).toHaveBeenCalledOnce(); + canRun = true; await vi.advanceTimersByTimeAsync(30_000); + expect(service.settle).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull(); +}); + +it("retains a notification that arrives while the startup reconstruction is in flight", async () => { + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) }; + service.load.mockImplementationOnce(async () => { await gate; return []; }); + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); + await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce()); + worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready; + await vi.advanceTimersByTimeAsync(1000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); + +it("cancels retries and subscriptions at shutdown", async () => { + const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000); + expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); +}); + + +it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => { + const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` })); + const late = { ...entry, id: "run-late", issueId: "late-issue" }; + const service = { + load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late] + : afterId ? [{ ...entry, id: "run-050" }] : firstPage), + settle: vi.fn(async () => true), + }; + const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready; + expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + worker.invalidate(JSON.stringify({ issueId: "late-issue" })); + await vi.advanceTimersByTimeAsync(3000); + expect(service.load).toHaveBeenCalledWith(undefined, "run-049"); + expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined); + expect(service.settle).toHaveBeenCalledTimes(52); + expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late)); + expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]); +}); diff --git a/server/src/__tests__/routine-status-reconciliation.test.ts b/server/src/__tests__/routine-status-reconciliation.test.ts index eba0718ce25..343c6fb38b6 100644 --- a/server/src/__tests__/routine-status-reconciliation.test.ts +++ b/server/src/__tests__/routine-status-reconciliation.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -5,11 +6,12 @@ import { companies, createDb, issues, routineRuns, routines } from "@paperclipai import { routineService } from "../services/routines.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping routine status database tests: ${support.reason}`); (support.supported ? describe : describe.skip)("routine status reconciliation", () => { let db: ReturnType, database: Awaited>; - beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(database.connectionString); }, 60_000); + beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, 60_000); afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); }); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); async function seed(status = "blocked") { @@ -36,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null }); } }); - it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { + it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => { const f = await seed(); if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id)); - if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id)); if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); } if (mode === "wrong_routine_company") { const other = await seed("done"); @@ -53,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${ expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id)); if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull(); }); + it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => { + const f = await seed("done"); + await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed"); + }); + it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => { + const f = await seed("in_progress"); + await db.update(routineRuns).set({ status: "failed", failureReason: null, + triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status", + status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id)); + expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 }); + expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed"); + }); it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => { const f = await seed(); const transaction = db.transaction.bind(db); vi.spyOn(db, "transaction").mockImplementationOnce(async callback => { diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 29ff63b985b..9124ae88fe0 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,3 +1,6 @@ +import { routineExecutionStatusService } from "../services/routine-run-status.js"; +import { attentionRetentionService } from "../services/attention-retention.js"; +import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js"; import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js"; import { skillVersionRetentionService } from "../services/skill-version-retention.js"; import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js"; @@ -141,6 +144,9 @@ const { })), reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })), reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })), + registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), + registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), + registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })), registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })), @@ -187,6 +193,8 @@ const { const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock); const issueThreadInteractionServiceMock = { sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })), + mergedPullRequestConfirmationPage: vi.fn(async () => ({ rows: [], complete: true })), + mergedPullRequestConfirmationCandidates: vi.fn(async () => []), sweepMergedPullRequestConfirmations: vi.fn(async () => ({ checked: 0, candidates: 0, @@ -214,7 +222,8 @@ const { }; const executionWorkspaceServiceFactoryMock = vi.fn(() => executionWorkspaceServiceMock); const externalObjectsServiceMock = { - refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })), + registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) => + coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })), }; const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock); const routineServiceMock = { @@ -384,6 +393,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({ setupLiveEventsWebSocketServer: vi.fn(), })); +vi.mock("../services/attention-retention.js", async importOriginal => ({ + ...await importOriginal(), + attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })), +})); + vi.mock("../services/join-request-retention.js", async importOriginal => ({ ...await importOriginal(), joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), @@ -399,6 +413,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({ approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })), })); +vi.mock("../services/routine-run-status.js", async importOriginal => ({ + ...await importOriginal(), + routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })), +})); + vi.mock("../services/index.js", () => ({ backfillLegacyToolOAuthTokens: vi.fn(async () => ({ scannedConnections: 0, @@ -537,6 +556,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({ validateNativeQuestionResponseInput: vi.fn(), })); +vi.mock("../services/status-card-evaluation.js", () => ({ + statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })), + registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({ + ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}), + })), +})); + vi.mock("../services/secret-proposals.js", () => ({ createSecretProposalsService: vi.fn(() => ({ expiryDeadlines: vi.fn(async () => []), @@ -576,6 +602,8 @@ vi.mock("../services/native-runtime/native-harness-backup-expiration.js", () => tick: vi.fn(async () => ({ checked: 0, candidates: 0, evidenceCandidates: 0, failed: 0, truncated: false })), noteEvidence: vi.fn(), nextEvidenceAt: vi.fn(() => null), tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn(), + backupScopes: vi.fn(async () => ({ names: [], complete: true })), + reconcileBackup: vi.fn(async () => null), expireBackup: vi.fn(), })), })); @@ -751,9 +779,12 @@ describe("startServer feedback export wiring", () => { loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toHaveLength(enabled ? 1 : 0); - expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "evidence"] : ["backups"]); + expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "backups", "evidence"] : ["backups"]); const backups = vi.mocked(nativeHarnessBackupExpirationService).mock.results[0]!.value; expect(backups.registerCheckpointExpiration).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toHaveLength(enabled ? 1 : 0); + expect(backups.tick).not.toHaveBeenCalled(); + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toEqual([]); }); it("tracks held evidence reconstruction after HTTP readiness", async () => { @@ -761,6 +792,9 @@ describe("startServer feedback export wiring", () => { const pending = Promise.withResolvers<{ truncated: boolean }>(); vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(), noteEvidence: vi.fn(), nextEvidenceAt: () => null, tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never); + vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ + backupScopes: vi.fn(async () => ({ names: [], complete: true })), reconcileBackup: vi.fn(), expireBackup: vi.fn(), + } as never); vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(() => pending.promise), noteEvidence: vi.fn(), nextEvidenceAt: () => null, tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never); try { @@ -800,6 +834,28 @@ describe("startServer feedback export wiring", () => { expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]); }); + it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 })); + let tick: (() => void) | undefined; + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { + if (interval === 1000) tick = callback; + return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0); + if (enabled) { + const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value; + expect(service.run).toHaveBeenCalledOnce(); + expect(tick).toBeDefined(); + for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); } + expect(service.run).toHaveBeenCalledOnce(); + } + expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]); + } finally { timer.mockRestore(); } + }); + it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -933,6 +989,21 @@ describe("startServer feedback export wiring", () => { if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true); }); + it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + await startServer(); await settleStartupHousekeeping(); + expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0); + expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled(); + if (enabled) { + const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0]; + const app = await createAppMock.mock.results.at(-1)!.value; + expect(coordinator).toBe(app.locals.deliveryWork); + expect(admitted()).toBe(true); + } + }); + it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); @@ -984,7 +1055,7 @@ describe("startServer feedback export wiring", () => { } }); - it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => { + it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 })); let tick: (() => void) | undefined; const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => { @@ -996,12 +1067,17 @@ describe("startServer feedback export wiring", () => { const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery); if (!enabled) { expect(registration).toBeUndefined(); + expect(attentionRetentionService).not.toHaveBeenCalled(); expect(decisionRetentionService).not.toHaveBeenCalled(); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); expect(decisionRetentionService).not.toHaveBeenCalled(); return; } expect(registration).toBeDefined(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true); + expect(attentionRetentionService).toHaveBeenCalledOnce(); + const source = vi.mocked(attentionRetentionService).mock.results[0]!.value; + expect(source.load).toHaveBeenCalledOnce(); const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value; expect(retention.deliverNotifications).toHaveBeenCalledOnce(); expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce(); @@ -1009,9 +1085,10 @@ describe("startServer feedback export wiring", () => { const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) => vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length; const before = companyReads(); - expect(before).toBeGreaterThan(0); expect(tick).toBeDefined(); + expect(before).toBe(0); expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping(); - expect(companyReads()).toBeGreaterThan(before); + expect(companyReads()).toBe(before); + expect(source.load).toHaveBeenCalledOnce(); expect(retention.deliverNotifications).toHaveBeenCalledOnce(); await registration![1].run(new AbortController().signal); expect(retention.deliverNotifications).toHaveBeenCalledTimes(2); @@ -1357,13 +1434,14 @@ describe("startServer feedback export wiring", () => { await startServer(); await settleStartupHousekeeping(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); - expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1); + expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled(); expect(intervalCallback).not.toBeNull(); intervalCallback?.(); await new Promise((resolve) => setImmediate(resolve)); - expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2); - expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); + expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled(); + expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); + expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled(); expect(retiredDetector).not.toHaveBeenCalled(); } finally { delete (runtime as Partial).reconcileProductivityReviews; @@ -1402,6 +1480,27 @@ describe("startServer feedback export wiring", () => { } }); + it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])( + "registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => { + if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token"); + else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined); + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + const intervalCallbacks: Array<() => void> = []; + const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => { + intervalCallbacks.push(callback); return 1 as unknown as ReturnType; + }) as typeof setInterval); + try { + await startServer(); await settleStartupHousekeeping(); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0); + const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value; + if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined); + for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback(); + await settleStartupHousekeeping(); + if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce(); + } finally { intervals.mockRestore(); vi.unstubAllEnvs(); } + }, + ); + it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true, @@ -1435,8 +1534,12 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); - expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); + expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1); + expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled(); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true); expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1); expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1); const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules); @@ -1458,6 +1561,15 @@ describe("startServer feedback export wiring", () => { } }); + it.each([true, false])("does not wait for external-object provider work before readiness (scheduler=%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + externalObjectsServiceMock.registerRefreshWork.mockReturnValueOnce({ + ready: new Promise(() => {}), wake: vi.fn(), stop: vi.fn(async () => {}), + }); + await startServer(); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce(); + }); + it("keeps external object refresh active when heartbeat scheduling is disabled", async () => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: false, @@ -1485,7 +1597,9 @@ describe("startServer feedback export wiring", () => { await Promise.resolve(); await Promise.resolve(); - expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1); + expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function)); + expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true); expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled(); expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1); expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled(); @@ -1523,6 +1637,20 @@ describe("startServer feedback export wiring", () => { } }); + it("becomes ready without waiting for the merge-confirmation observation", async () => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); + let release!: () => void; + const blocked = new Promise<{ rows: never[]; complete: boolean }>(resolve => { release = () => resolve({ rows: [], complete: true }); }); + issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage.mockReturnValueOnce(blocked); + const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType); + let started = false; + const starting = startServer().then(server => { started = true; return server; }); + try { + await vi.waitFor(() => expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalled()); + await settleStartupHousekeeping(); expect(started).toBe(true); expect(getStartupRecoveryState().phase).toBe("ready"); + } finally { release(); await starting; timer.mockRestore(); } + }); + it.each(["retained cleanup", "workspace finalization", "environment settlement", "native run projection", "native result commit", "native recovery notices", "external conversation settlement"] as const)("becomes ready while %s is held but joins it at shutdown", async owner => { loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true })); let releaseCleanup!: () => void; diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts new file mode 100644 index 00000000000..d012a239efd --- /dev/null +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -0,0 +1,656 @@ +import { budgetService } from "../services/budgets.js"; +import { documentService } from "../services/documents.js"; +import * as companySearch from "../services/company-search.js"; +import { setImmediate as immediate } from "node:timers/promises"; +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection, + heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db"; +import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueService } from "../services/issues.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("status-card evaluation work", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let coordinator: ReturnType | undefined; + let draining = false; + let companyId: string, agentId: string; + const errors: unknown[] = []; + const evaluations = new Set>(); + let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); + const actor = { agentId: null, userId: "board-user" }; + const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({ + mode, debounceSeconds: 60, intervalMinutes: 15, + }); + // Controlled provider boundary: assignment persists a queued wake, and tests + // explicitly run the authorized summarizer writeback. No model/provider runs. + const wakeup = vi.fn(async (id: string, options: { payload?: Record | null; reason?: string | null }) => { + const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id)); + return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment", + reason: options.reason, payload: options.payload, status: "queued" }).returning(); + }); + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + }, 30_000); + beforeEach(async () => { + companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset(); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] }); + await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; + vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents); + await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards); + await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog); + await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns); + await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function start(service = statusCardEvaluationService(db, { wakeup })) { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, + onError: error => errors.push(error) }); + await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => { + const work = service.run(signal); + evaluations.add(work); + void work.then(() => evaluations.delete(work), () => evaluations.delete(work)); + return work; + } }, () => !draining).ready; + } + async function settled() { + // Do not leap across PostgreSQL connect timers while real I/O is pending. + // Await actual domain work, then require several settled event-loop turns + // so a transient zero hold cannot stand in for transaction reconciliation. + for (let turn = 0; turn < 3; turn++) { + await vi.waitFor(async () => { + await Promise.allSettled([...evaluations]); + await vi.advanceTimersByTimeAsync(500); + await Promise.allSettled([...evaluations]); + expect(idleWorkSnapshot().active).toBe(0); + }, { timeout: 5000 }); + await immediate(); + } + } + async function quiet() { + await settled(); + expect(coordinator!.nextWakeAt()).toBeNull(); + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute"); + const calls = wakeup.mock.calls.length; + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled(); + expect(wakeup).toHaveBeenCalledTimes(calls); + reads.mockRestore(); writes.mockRestore(); executes.mockRestore(); + } + async function writeback(cardId: string, generationIssueId: string, compile = false) { + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning(); + await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId)); + const writer = { agentId, runId: run!.id }; + const cards = statusCardService(db); + if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery], + title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer); + await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer); + await issueService(db).update(generationIssueId, { status: "done" }); + } + async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) { + const cards = statusCardService(db); + const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor); + const compile = await cards.requestCompile(created.id, actor); + await writeback(created.id, compile.generatingIssue.id, true); + return cards.getById(created.id).then(row => row!); + } + + it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => { + await start(); await quiet(); + await card("manual"); await card("reactive"); await quiet(); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(errors).toEqual([]); + }); + + it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => { + await start(); const created = await card(); await quiet(); + const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + await vi.waitFor(async () => { + expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0); + }, { timeout: 5000 }); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt).toBeInstanceOf(Date); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + const generating = await statusCardService(db).getById(created.id); + expect(generating?.generatingIssueId).toBeTruthy(); + const durable = await db.select().from(agentWakeupRequests); + expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId); + expect(changed.companyId).toBe(companyId); + await writeback(created.id, generating!.generatingIssueId!); await quiet(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId))) + .toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]); + expect(errors).toEqual([]); + }); + + it("keeps card deletion scoped to its company and a missing deletion quiet", async () => { + const removed = await card(); + await card(); + const ownCompany = companyId, ownAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card(); + companyId = ownCompany; agentId = ownAgent; + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner), actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + await start(); await quiet(); searchedCompanies.length = 0; + const cards = statusCardService(db); + expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany }); + await settled(); + // Deleting a card only reconstructs deadlines; it is not an issue change + // that warrants re-running any unchanged reactive card's saved search. + expect(searchedCompanies).toEqual([]); + expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 }); + expect(wakeup).not.toHaveBeenCalled(); + searchedCompanies.length = 0; + expect(await cards.remove(removed.id)).toBeNull(); + await quiet(); + expect(searchedCompanies).toEqual([]); + expect(errors).toEqual([]); + }); + + it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => { + const reactive = await card(); + const originalCompany = companyId, originalAgent = agentId; + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const other = await card("interval"); + const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" }); + const otherReactive = await card(); + companyId = originalCompany; agentId = originalAgent; + const service = statusCardEvaluationService(db, { wakeup }); + const originalRun = service.run; + let running: Promise | undefined; + const run = vi.spyOn(service, "run").mockImplementation(signal => { + const result = originalRun(signal); running = result; return result; + }); + await start(service); await settled(); + await vi.advanceTimersByTimeAsync(1000); + // Give each committed producer and any actual domain work time to settle. + // This prevents incidental overlap from hiding per-write reconstruction. + const flush = async () => { + for (let i = 0; i < 10; i++) { + await running; await immediate(); + } + }; + await flush(); + const select = vi.spyOn(db, "select"); + const factory = companySearch.companySearchService; + const searchedCompanies: string[] = []; + vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => { + const search = factory(owner); + const actual = search.search; + search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); }; + return search; + }); + run.mockClear(); + const burstStarted = Date.now(); + for (let i = 0; i < 50; i++) { + const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" }); + await flush(); + await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" }); + await flush(); + } + await flush(); + // One eager reconciliation after idle and one bounded batched continuation; + // the producers themselves are not counted as background SQL. + const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length; + const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length; + const elapsedMs = Date.now() - burstStarted; + const maximum = Math.ceil(elapsedMs / 1000) + 2; + console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length }); + expect(run.mock.calls.length).toBeLessThanOrEqual(maximum); + expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeLessThanOrEqual(maximum); + expect(searchedCompanies.length).toBeGreaterThan(0); + expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany])); + expect(wakeup).not.toHaveBeenCalled(); + // Re-register with an existing durable deadline inside the coalescing + // window. A company hint must not make that real deadline wait one second. + await coordinator!.stop(); coordinator = undefined; + const dueAt = Date.now() + 500; + await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id)); + await start(service); await flush(); + await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" }); + await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id, + "More changes while another company is due", { userId: "board-user" }); + await flush(); + await vi.advanceTimersByTimeAsync(500); await flush(); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + expect(Date.now()).toBeLessThan(dueAt + 500); + expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent); + const generated = (await statusCardService(db).getById(other.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0); + expect(searchedCompanies).toContain(other.companyId); + // Complete all ordinary results, then prove there is no recurring batch + // timer once the requested work has settled. + companyId = other.companyId; agentId = other.agentId!; + await writeback(other.id, generated.generatingIssueId!); + await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor); + companyId = originalCompany; agentId = originalAgent; + await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush(); + await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy()); + const pending = (await statusCardService(db).getById(reactive.id))!; + expect(pending.generatingIssueId).toBeTruthy(); + await writeback(reactive.id, pending.generatingIssueId!); + const otherPending = (await statusCardService(db).getById(otherReactive.id))!; + expect(otherPending.generatingIssueId).toBeTruthy(); + companyId = other.companyId; agentId = other.agentId!; + await writeback(otherReactive.id, otherPending.generatingIssueId!); + companyId = originalCompany; agentId = originalAgent; + select.mockRestore(); + await quiet(); expect(errors).toEqual([]); + }, 30_000); + + it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }; + const cards = []; + for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy)); + await start(); await quiet(); + const RealFormatter = Intl.DateTimeFormat; + const actualParts = RealFormatter.prototype.formatToParts; + let constructions = 0, conversions = 0; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++; + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) { + if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++; + return actualParts.call(this, date); + }); + const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" }); + await settled(); + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions).toBeLessThanOrEqual(1000); + const cold = { constructions, conversions }; + constructions = 0; conversions = 0; + for (let i = 0; i < 10; i++) { + await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" }); + await settled(); + } + expect(constructions).toBe(0); + expect(conversions).toBeGreaterThanOrEqual(200); + expect(conversions).toBeLessThanOrEqual(800); + expect(wakeup).not.toHaveBeenCalled(); + const opening = new Date("2026-11-03T09:00:00Z"); + for (const created of cards) expect((await statusCardService(db).getById(created.id))) + .toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening }); + process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20, + sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n"); + constructor.mockRestore(); parts.mockRestore(); + await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled(); + expect(wakeup).toHaveBeenCalledTimes(20); + for (const created of cards) { + const generated = (await statusCardService(db).getById(created.id))!; + expect(generated.generatingIssueId).toBeTruthy(); + await writeback(created.id, generated.generatingIssueId!); + } + await quiet(); expect(errors).toEqual([]); + }, 60_000); + + it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => { + await start(); let created = await card("interval"); await settled(); + const old = coordinator!.nextWakeAt(); + created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + await Promise.allSettled([...evaluations]); + expect(coordinator!.nextWakeAt()).toBeLessThan(old!); + }); + await statusCardService(db).update(created, { archived: true }, actor); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]); + }); + + it("restores overdue debt after the feature is disabled across its saved deadline", async () => { + await start(); const created = await card(); await quiet(); + await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + const pending = (await statusCardService(db).getById(created.id))!; + await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + await instanceSettingsService(db).updateExperimental({ enableStatusCards: true }); + await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce()); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet(); + }); + + it("does not react to rolled-back or nested uncommitted card configuration", async () => { + await start(); const created = await card("manual"); await quiet(); + await expect(db.transaction(async tx => { + await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled(); + throw new Error("rollback"); + })).rejects.toThrow("rollback"); + await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + await db.transaction(async tx => { + await tx.transaction(async nested => { + await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor); + }); + expect(coordinator!.nextWakeAt()).not.toBeNull(); + expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull(); + }); + await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); + + it("serializes competing connections when one source event preempts a future evaluation", async () => { + const created = await card(); + await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const first = statusCardEvaluationService(db, { wakeup }); + const second = statusCardEvaluationService(db, { wakeup }); + first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`])); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const pending = await statusCardService(db).getById(created.id); + expect(pending?.pendingChangeCount).toBeGreaterThan(0); + expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000); + expect(wakeup).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(60_000); + await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]); + const updated = await statusCardService(db).getById(created.id); + expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce(); + expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId))) + .toHaveLength(1); + }); + + it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => { + vi.useRealTimers(); + const previousDb = db; + db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + try { + const created = await card("reactive", { ...policy(), debounceSeconds: 1 }); + const countLocks = async () => Number((await db.execute<{ count: string }>(sql` + select count(*) from pg_stat_activity where datname = current_database() + and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count); + const baseline = await countLocks(); + await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + })); + await acquired.promise; + try { + await service.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + expect(await service.next()).toBeGreaterThan(Date.now()); + } finally { release.resolve(); await holder; } + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + await vi.waitFor(async () => { + await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }, { timeout: 5000, interval: 100 }); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + await service.run(new AbortController().signal); + expect(await service.next()).toBeNull(); + expect(idleWorkSnapshot().active).toBe(0); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + // Failure exits also release the physical connection and advisory lock. + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + await service.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry"); + await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); + await withDedicatedDbConnection(db, connection => connection.transaction(async tx => { + expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true); + })); + } finally { await db.$client.end({ timeout: 1 }); db = previousDb; } + }, 15_000); + + it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => { + await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" }); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" }); + const created = await card(); + expect(created.nextEvalReason).toBe("membership"); + const oldExpiry = created.nextEvalAt!.getTime(); + // No coordinator exists to receive this committed source hint. A fresh + // service has no in-memory hint and must reconstruct from durable truth. + await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" }); + if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id)); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); + expect(pending.nextEvalReason).toBe("policy"); + expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); + expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership"); + }); + + it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => { + const created = await card(); + const retry = new Date(Date.now() + 240_000); + await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id)); + await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" }); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(retry.getTime()); + vi.setSystemTime(retry); await restarted.run(new AbortController().signal); + expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0); + }); + + it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => { + vi.setSystemTime(new Date("2026-11-02T18:00:00Z")); + const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}), + ...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) }; + const created = await card("reactive", refreshPolicy); + if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive", + status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 }); + await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" }); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now()); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt); + vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal); + expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("unions startup reconstruction with a scoped source change in another company", async () => { + const firstCompany = companyId; + const first = await card(); + await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" }); + companyId = randomUUID(); agentId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} }); + const second = await card(); + await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" }); + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + await service.run(new AbortController().signal); + const cards = statusCardService(db); + expect((await cards.getById(first.id))?.companyId).toBe(firstCompany); + expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0); + expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0); + await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledTimes(2); + expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId)))) + .toEqual(new Set([firstCompany, companyId])); + }); + + it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => { + await start(); await card(); await quiet(); + await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" }); + await quiet(); expect(wakeup).not.toHaveBeenCalled(); + compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" }); + const searchCard = await card(); await quiet(); + const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" }); + await quiet(); + await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet(); + await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0)); + }); + + it("retains relative query membership expiry across restart and emits the eventual removal", async () => { + const now = new Date(); + const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" }); + await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" }); + const created = await card(); + const expires = now.getTime() + 3_600_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const restarted = statusCardEvaluationService(db, { wakeup }); + await restarted.run(new AbortController().signal); + expect(await restarted.next()).toBe(expires); + vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(expires); await restarted.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000); + vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => { + const now = new Date(); + const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, { + title: `${priority} blocker`, priority, status: "blocked", + }))); + await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id)); + for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id)); + compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 }); + const created = await card(); + expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]); + const expires = now.getTime() + 60_001; + expect(created.nextEvalAt?.getTime()).toBe(expires); + const service = statusCardEvaluationService(db, { wakeup }); + vi.setSystemTime(expires); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled(); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!)); + expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id])); + }); + + it("retains a scoped retry while another evaluator owns the card lock", async () => { + const created = await card(); + await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" }); + const acquired = Promise.withResolvers(), release = Promise.withResolvers(); + const holder = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`); + acquired.resolve(); await release.promise; + }); + await acquired.promise; + const service = statusCardEvaluationService(db, { wakeup }); + service.changed(new Set([`changes:${companyId}`])); + try { + await service.run(new AbortController().signal); + expect(await service.next()).toBeGreaterThan(Date.now()); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + expect(wakeup).not.toHaveBeenCalled(); + } finally { release.resolve(); await holder; } + await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal); + const pending = (await statusCardService(db).getById(created.id))!; + expect(pending.pendingChangeCount).toBeGreaterThan(0); + vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal); + expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => { + const created = await card(); + await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); + const service = statusCardEvaluationService(db, { wakeup }); + await service.run(new AbortController().signal); + const failed = (await statusCardService(db).getById(created.id))!; + const retryAt = failed.nextEvalAt!.getTime(); + expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled(); + await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt); + draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled(); + // Repair the controlled malformed stored-query fault; no tested result is precreated. + await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id)); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet(); + expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull(); + }); + + it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => { + await start(); const created = await card(); await quiet(); + wakeup.mockRejectedValueOnce(new Error("assignment unavailable")); + await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" }); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect((await statusCardService(db).getById(created.id))!.state).toBe("error"); + }); + const failed = (await statusCardService(db).getById(created.id))!; + expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null }); + expect(errors.length).toBeGreaterThan(0); + const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id)); + const ledger = history.find(row => row.status === "failed")!; + expect(ledger.finishedAt).toBeInstanceOf(Date); + expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled"); + const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count); + // An explicit ordinary refresh is still the user's supported retry action. + const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor }); + expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet(); + expect((await statusCardService(db).getById(created.id))!.state).toBe("active"); + }); + + it("budget-owned company resume reactivates reactive work committed while paused", async () => { + await start(); const created = await card(); await quiet(); + const budgets = budgetService(db); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user"); + const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test", + costCents: 200, occurredAt: new Date() }).returning(); + await budgets.evaluateCostEvent(cost!); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused"); + await quiet(); + await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet(); + expect(wakeup).not.toHaveBeenCalled(); + await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user"); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active"); + await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0)); + await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce(); + }); + + it("preserves the first write during drain until resume", async () => { + await start(); const created = await card("manual"); await quiet(); + draining = true; + await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor); + await settled(); expect(wakeup).not.toHaveBeenCalled(); + draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled(); + expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]); + }); +}); diff --git a/server/src/__tests__/status-card-notifications.test.ts b/server/src/__tests__/status-card-notifications.test.ts new file mode 100644 index 00000000000..0566ebdd394 --- /dev/null +++ b/server/src/__tests__/status-card-notifications.test.ts @@ -0,0 +1,31 @@ +import { expect, it, vi } from "vitest"; +import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js"; + +it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => { + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise) => + fn({ execute: async () => [{ xid: "1" }] }) }); + const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, + DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]; + const settled: Array<{ queue: string; hints?: string[] }> = []; + const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => { + if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined }); + })); + try { + const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined; + await owner.transaction(async tx => { + await notifyExecutionRunWork(tx, scope); + expect(settled).toEqual([]); + }); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints) + .toEqual(scoped ? ["changes:changed-company"] : undefined); + settled.length = 0; + const write = vi.fn(async () => "written"); + expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written"); + expect(write).toHaveBeenCalledOnce(); + expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery, + ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])])); + } finally { for (const stop of unsubscribe) stop(); } +}); diff --git a/server/src/__tests__/status-card-update-engine.test.ts b/server/src/__tests__/status-card-update-engine.test.ts index 09a660e8741..9fdf18584e9 100644 --- a/server/src/__tests__/status-card-update-engine.test.ts +++ b/server/src/__tests__/status-card-update-engine.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { statusCardRefreshPolicySchema } from "@paperclipai/shared"; import { chooseStatusCardUpdateKind, @@ -8,10 +8,12 @@ import { filterStatusCardChanges, isWithinStatusCardActiveHours, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, statusCardChangesHash, } from "../services/status-card-update-engine.js"; describe("status card update engine", () => { + afterEach(() => vi.restoreAllMocks()); const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 }); it("retains non-terminal and terminal status transitions plus membership changes", () => { @@ -78,6 +80,149 @@ describe("status card update engine", () => { expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours"); }); + it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T16:59:30Z"); + expect(nextStatusCardEvaluationAt(policy, now)).toBeNull(); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + const open = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"), + completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 })) + .toEqual(new Date("2026-07-23T14:30:00.001Z")); + expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 })) + .toEqual(new Date("2026-07-24T09:00:00Z")); + }); + + it("retains empty-window pending changes without a recurring eligibility scan", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + }); + + it("bounds empty-window formatter work before any timezone conversion", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } }); + const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () { + throw new Error("Empty active hours must not construct a formatter"); + }); + const now = new Date("2026-07-23T14:00:00Z"); + expect(isWithinStatusCardActiveHours(policy, now)).toBe(false); + expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull(); + expect(formatter).not.toHaveBeenCalled(); + }); + + it("bounds formatter construction and reuses the next opening across cards and source writes", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } }); + const RealFormatter = Intl.DateTimeFormat; + let constructions = 0; + vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters"); + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts"); + const now = new Date("2026-07-23T22:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + const started = performance.now(); + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z")); + const coldMs = performance.now() - started; + const coldConversions = conversions.mock.calls.length; + expect(coldConversions).toBeLessThanOrEqual(2881); + conversions.mockClear(); + const warmStarted = performance.now(); + for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) { + expect(nextStatusCardPolicyEligibility({ ...input, + policy: { ...policy, activeHours: { ...policy.activeHours! } }, + now: new Date(now.getTime() + write * 1000), + })).toEqual(new Date("2026-07-24T11:30:00Z")); + } + expect(constructions).toBeLessThanOrEqual(1); + expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000); + process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions, + warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n"); + }); + + it.each([ + ["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"], + ["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"], + ["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"], + ["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"], + ["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"], + ["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"], + ["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"], + ])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } }); + const now = new Date(time); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected)); + // A later closed instant reuses the opening, while the exact opening itself + // remains eligible without rounding it into the next minute or day. + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected)); + const opening = new Date(expected); + expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening); + expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1)); + }); + + it("invalidates an upcoming opening for clock rollback and changed policy values", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } }); + const calculate = (time: string, activeHours = policy.activeHours!) => { + const now = new Date(time); + return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now, + lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z")); + expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z")); + expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z")); + }); + + it("checks the precise instant before a cached minute opening for historical second offsets", () => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } }); + const now = new Date("1970-01-01T08:00:00Z"); + const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }; + expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z")); + const inside = new Date("1970-01-01T09:44:45Z"); + expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true); + expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside); + }); + + it("bounds formatter and upcoming-opening cache retention", () => { + const now = new Date("2026-07-25T00:00:00Z"); + const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: "09:00", end: "17:00", timezone } }); + const zones = Intl.supportedValuesOf("timeZone").slice(0, 34); + const first = timezonePolicy(zones[0]!); + isWithinStatusCardActiveHours(first, now); + for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now); + const RealFormatter = Intl.DateTimeFormat; + const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters) { + return new RealFormatter(...args); + } as unknown as typeof Intl.DateTimeFormat); + isWithinStatusCardActiveHours(first, now); + expect(constructor).toHaveBeenCalledTimes(1); + constructor.mockRestore(); + const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`; + const calculate = (minute: number) => { + const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, + activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } }); + return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 }); + }; + calculate(1); + for (let minute = 2; minute <= 258; minute++) calculate(minute); + const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts"); + expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z")); + expect(conversions.mock.calls.length).toBeGreaterThan(1); + }); + it("selects full rebuilds for bounded drift rules and incremental otherwise", () => { const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false }; expect(chooseStatusCardUpdateKind(base)).toBe("incremental"); diff --git a/server/src/__tests__/status-cards.test.ts b/server/src/__tests__/status-cards.test.ts index e1436666316..05e1bba48e5 100644 --- a/server/src/__tests__/status-cards.test.ts +++ b/server/src/__tests__/status-cards.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; @@ -37,7 +38,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; type Db = ReturnType; @@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); @@ -189,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => { .send({ title: "Launch health", titlePinned: true }); expect(patched.status).toBe(200); expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true }); + expect(patched.body).not.toHaveProperty("nextEvalReason"); const scheduled = await request(app) .patch(`/api/status-cards/${created.body.id}`) @@ -1149,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => { }); expect(summaryWrite.status).toBe(200); expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null }); + expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason"); expect(summaryWrite.body.document.latestBody).toContain("**Decide:**"); expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 }); diff --git a/server/src/__tests__/work-products.test.ts b/server/src/__tests__/work-products.test.ts index 476e1156184..cef395409fd 100644 --- a/server/src/__tests__/work-products.test.ts +++ b/server/src/__tests__/work-products.test.ts @@ -1,3 +1,5 @@ +import { PgDialect } from "drizzle-orm/pg-core"; +import type { SQL } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { enrichWorkProductMetadataWithDiff, @@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial> = {}) }; } +function retentionReads(insideTransaction: () => boolean) { + return vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(insideTransaction()).toBe(true); + expect(statement.params).toContain("company-1"); + expect(statement.params).toContain("issue-1"); + if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }]; + throw new Error(`Unexpected work-product fixture query: ${statement.sql}`); + }); +} + describe("workProductService", () => { it("extracts runner totals and enriches work-product metadata", () => { const summary = workProductDiffSummaryFromEventPayload({ @@ -158,11 +172,16 @@ describe("workProductService", () => { const insertValues = vi.fn(() => ({ returning: insertReturning })); const txInsert = vi.fn(() => ({ values: insertValues })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), update: txUpdate, insert: txInsert, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.createForIssue("issue-1", "company-1", { @@ -194,11 +213,16 @@ describe("workProductService", () => { const updateSet = vi.fn(() => ({ where: updateWhere })); const txUpdate = vi.fn(() => ({ set: updateSet })); + let transactionOpen = false; const tx = { + execute: retentionReads(() => transactionOpen), select: txSelect, update: txUpdate, }; - const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => await callback(tx)); + const transaction = vi.fn(async (callback: (input: typeof tx) => Promise) => { + transactionOpen = true; + try { return await callback(tx); } finally { transactionOpen = false; } + }); const svc = workProductService({ transaction } as any); const result = await svc.update("work-product-1", { diff --git a/server/src/index.ts b/server/src/index.ts index b2db762a808..fb0585967a1 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,4 +1,9 @@ import { registerGitHubConnectionEventWork, registerGitHubConnectionContinuityWork } from "./services/github-connection-work.js"; +import { registerNativeHarnessBackupExpirationWork } from "./services/native-runtime/native-harness-backup-expiration-work.js"; +import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js"; +import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js"; +import { routineExecutionStatusService } from "./services/routine-run-status.js"; +import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js"; import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; @@ -33,8 +38,6 @@ import { skillVersionRetentionService, } from "./services/skill-version-retention.js"; import { - NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS, - NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS, nativeHarnessBackupExpirationService, } from "./services/native-runtime/native-harness-backup-expiration.js"; import { registerNativeRunnerEvidenceRetentionWork } from "./services/native-runtime/native-runner-evidence-retention-work.js"; @@ -110,7 +113,6 @@ import { createApp } from "./app.js"; import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js"; import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js"; import { logger } from "./middleware/logger.js"; -import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js"; import { setStartupRecoveryPhase } from "./startup-recovery-state.js"; import { StartupRefusalError, @@ -128,7 +130,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi import { feedbackService, applyManagedEnvironments, - attentionService, backfillPrincipalAccessCompatibility, backfillLegacyToolOAuthTokens, bootstrapExecutionPolicyFromEnv, @@ -146,13 +147,12 @@ import { reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup, routineService, - statusCardService, toolAccessService, workspaceOperationService, } from "./services/index.js"; import { questionResponseDeliveryService } from "./services/question-response-delivery.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; -import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js"; +import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js"; import { createSecretProposalsService } from "./services/secret-proposals.js"; import { environmentRuntimeService } from "./services/environment-runtime.js"; import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js"; @@ -974,9 +974,6 @@ async function startServerWithDatabaseTeardown( } }; const pluginWorkerManager = createPluginWorkerManager(); - const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter( - (details, message) => logger.warn(details, message), - ); const heartbeat = config.heartbeatSchedulerEnabled ? heartbeatService(db as any, { pluginWorkerManager }) : null; @@ -1296,47 +1293,10 @@ async function startServerWithDatabaseTeardown( evidenceRetentionDays: config.nativeRunnerEvidenceRetentionDays, shouldContinue: () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), }); - let nativeHarnessBackupExpirationInFlight = false; - let nativeHarnessBackupExpirationContinuation: ReturnType | undefined; - const sweepNativeHarnessBackups = () => { - if (!config.nativeHarnessBackupExpirationEnabled || nativeHarnessBackupExpirationInFlight || isIdleTaskDrainActive() || heartbeatSchedulerStopped || isWarmStandby()) return; - nativeHarnessBackupExpirationInFlight = true; - trackHeartbeatSchedulerWork(nativeHarnessBackupExpiration.tick() - .then((result) => { - if (result.candidates > 0 || result.evidenceCandidates > 0 || result.failed > 0) { - logger.info({ ...result, dryRun: config.nativeHarnessBackupExpirationDryRun }, "native harness backup expiration"); - } - if (result.truncated && !heartbeatSchedulerStopped) { - nativeHarnessBackupExpirationContinuation = setTimeout(sweepNativeHarnessBackups, 1000); - nativeHarnessBackupExpirationContinuation.unref?.(); - } - }) - .catch((err) => { - logger.error({ err }, "native harness backup expiration failed"); - }) - .finally(() => { nativeHarnessBackupExpirationInFlight = false; })); - }; - const nativeHarnessBackupExpirationInterval = setInterval(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS); - nativeHarnessBackupExpirationInterval.unref?.(); - const nativeHarnessBackupExpirationStartup = setTimeout(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS); - nativeHarnessBackupExpirationStartup.unref?.(); const externalObjects = externalObjectService(db as any, { pluginWorkerManager, enabled: async () => (await instanceSettingsService(db).getExperimental()).enableExternalObjects === true, }); - const scheduleExternalObjectRefreshSweep = (now = new Date()) => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(externalObjects - .refreshDueObjectsForActiveCompanies(50, now) - .then((result) => { - if (result.checked > 0 || result.refreshed > 0) { - logger.info({ ...result }, "external-object scheduler tick refreshed due objects"); - } - }) - .catch((err) => { - logger.error({ err }, "external-object scheduler tick failed"); - })); - }; // The retry backstop for orphan sandboxes. An acquire that rejects a // foreign-company insert tears the provisioned sandbox down. If that teardown @@ -1363,8 +1323,16 @@ async function startServerWithDatabaseTeardown( resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction), }); const deliveryWork = app.locals.deliveryWork as ReturnType; + externalObjects.registerRefreshWork(deliveryWork, + () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); if (config.nativeHarnessBackupExpirationEnabled) { nativeHarnessBackupExpiration.registerCheckpointExpiration(deliveryWork); + const backupExpirationWork = registerNativeHarnessBackupExpirationWork(deliveryWork, db as any, { + dryRun: config.nativeHarnessBackupExpirationDryRun, + closedForDays: config.nativeHarnessBackupExpirationDays, + }, () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); + trackHeartbeatSchedulerWork(backupExpirationWork.ready); + server.on("close", backupExpirationWork.stop); } const taskDrainExpiryWork = registerTaskDrainExpiryWork(deliveryWork, () => { if (heartbeatSchedulerStopped || isWarmStandby()) return; @@ -1543,16 +1511,7 @@ async function startServerWithDatabaseTeardown( heartbeat.drainActiveRunExecutions(); prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown; const routines = routineService(db as any, { pluginWorkerManager }); - let nextRoutineStatusReconcile = 0; - const scheduleRoutineStatusReconcile = () => { - if (!isCloudManagedInstance() || isWarmStandby() || isIdleTaskDrainActive() || heartbeatSchedulerStopped || - Date.now() < nextRoutineStatusReconcile) return; - nextRoutineStatusReconcile = Date.now() + 5 * 60_000; - trackHeartbeatSchedulerWork(routines.reconcileExecutionRunStatuses().then(result => { - if (result.settled > 0) logger.info(result, "reconciled routine execution issue status"); - }).catch(() => logger.warn("routine execution status reconciliation deferred"))); - }; - const statusCards = statusCardService(db as any); + const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat); const issues = issueService(db as any); const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, { wakeup: heartbeat.wakeup, @@ -1583,19 +1542,8 @@ async function startServerWithDatabaseTeardown( : config.workspaceCompactionDelayHours * 60 * 60 * 1000, terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS, }); - const scheduleMergedPullRequestConfirmationSweep = () => { - if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork(mergedPullRequestConfirmations - .sweepMergedPullRequestConfirmations() - .then((result) => { - if (result.accepted > 0) { - logger.info(result, "accepted merge confirmations for merged pull requests"); - } - }) - .catch((err) => { - logger.error({ err }, "merged pull-request confirmation sweep failed"); - })); - }; + registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations, + () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()); const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, { storage: storageService, retentionDays: config.gitRestorePointRetentionDays, @@ -1756,8 +1704,6 @@ async function startServerWithDatabaseTeardown( } } - const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries()); - await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns()); const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions()); if ( recoveredGoalActions.enqueued > 0 || @@ -1775,48 +1721,10 @@ async function startServerWithDatabaseTeardown( "startup session-goal recovery resumed durable agent goals", ); } - const reconciled = await heartbeat.reconcileStrandedAssignedIssues(); - if ( - promotion.promoted > 0 || - reconciled.assignmentDispatched > 0 || - reconciled.dispatchRequeued > 0 || - reconciled.continuationRequeued > 0 || - reconciled.successfulRunHandoffEscalated > 0 || - reconciled.successfulRunHandoffRetried > 0 || - reconciled.escalated > 0 - ) { - logger.warn( - { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, - "startup heartbeat recovery changed assigned issue state", - ); - } - - const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes(); - if (dependencyWakesReconciled.healed > 0) { - logger.warn( - { ...dependencyWakesReconciled }, - "startup dependency-wake reconciliation restored task execution paths", - ); - } - - const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs(); - reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup"); - if (taskWatchdogsReconciled.triggered > 0) { - logger.warn( - { ...taskWatchdogsReconciled }, - "startup task-watchdog reconciliation triggered watchdog work", - ); - } - const scanned = await heartbeat.scanSilentActiveRuns(); if (scanned.created > 0 || scanned.escalated > 0) { logger.warn({ ...scanned }, "startup active-run output watchdog created review work"); } - - const swept = await heartbeat.sweepStaleIssueLocks(); - if (swept.cleared > 0) { - logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks"); - } })().catch((err) => { logger.error({ err }, "startup heartbeat recovery failed"); throw err; @@ -1825,6 +1733,12 @@ async function startServerWithDatabaseTeardown( await startupHeartbeatRecovery; } + if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; + if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; + if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready; if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; @@ -1904,23 +1818,6 @@ async function startServerWithDatabaseTeardown( if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork, () => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready; - const runRetentionSweep = async () => { - const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active")); - let archived = 0; - for (const company of activeCompanies) { - // Cursor pagination rebuilds the whole feed for every page; one - // unscoped all-items build keeps this sweep at a single feed build - // per company per tick. - const page = await attentionService(db as any).list(company.id, { - includeDismissed: true, - excludeSources: ["approval", "join_request"], - all: true, - allowUnscopedAll: true, - }); - archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items }); - } - return { archived }; - }; // Startup housekeeping: provider and connection sweeps that every timer tick // repeats and that neither serving requests nor run recovery depends on. // Readiness does not wait for it. It is tracked as scheduler work, so idle @@ -1963,7 +1860,10 @@ async function startServerWithDatabaseTeardown( () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any), () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); - await step("retention_sweep", runRetentionSweep); + await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation, + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); + await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready); }; const createHeartbeatSchedulerTick = (scheduled: ReturnType) => () => { @@ -1972,10 +1872,6 @@ async function startServerWithDatabaseTeardown( // captures the authoritative set of running heartbeat rows. trackHeartbeatSchedulerWork((async () => { if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleRoutineStatusReconcile(); - trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => { - logger.error({ err }, "decision retention sweep failed"); - })); const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses(); if (sweptRuntimeStatuses > 0) { logger.info( @@ -1985,90 +1881,24 @@ async function startServerWithDatabaseTeardown( } if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleExternalObjectRefreshSweep(new Date()); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - scheduleMergedPullRequestConfirmationSweep(); if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; - trackHeartbeatSchedulerWork((async () => { - const experimental = await instanceSettingsService(db).getExperimental(); - if (experimental.enableStatusCards !== true) return; - const result = await statusCards.tickDueStatusCards(new Date()); - await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => { - try { - await queueIssueAssignmentWakeup({ - heartbeat, - issue: generatingIssue, - reason: "status_card_update_assigned", - mutation: "status_card.scheduler_update_requested", - contextSource: "status_card_scheduler", - requestedByActorType: "system", - taskKey: `status-card:${cardId}`, - rethrowOnError: true, - }); - } catch (err) { - await issues.update(generatingIssue.id, { status: "cancelled" }); - throw err; - } - })); - if (result.evaluated > 0 || result.enqueued.length > 0) { - logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete"); - } - })().catch((err) => { - logger.error({ err }, "status-card scheduler tick failed"); - })); + if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return; if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) { // Periodically reap orphaned runs (5-min staleness threshold) and make sure // persisted queued work is still being driven forward. runPeriodicRecovery(() => heartbeat - .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) - .then(() => heartbeat.promoteDueScheduledRetries()) - .then(async (promotion) => { - await heartbeat.resumeQueuedRuns(); - const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true }); - if ( - promotion.promoted > 0 || - reconciled.assignmentDispatched > 0 || - reconciled.dispatchRequeued > 0 || - reconciled.continuationRequeued > 0 || - reconciled.successfulRunHandoffEscalated > 0 || - reconciled.successfulRunHandoffRetried > 0 || - reconciled.escalated > 0 - ) { - logger.warn( - { promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled }, - "periodic heartbeat recovery changed assigned issue state", - ); - } - }) - .then(async () => { - const reconciled = await heartbeat.reconcileResolvedDependencyWakes(); - if (reconciled.healed > 0) { - logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths"); - } - }) - .then(async () => { - const reconciled = await heartbeat.reconcileTaskWatchdogs(); - reportIncompleteWatchdogScans(reconciled, "periodic"); - if (reconciled.triggered > 0) { - logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work"); - } - }) + .reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true }) .then(async () => { const scanned = await heartbeat.scanSilentActiveRuns(); if (scanned.created > 0 || scanned.escalated > 0) { logger.warn({ ...scanned }, "periodic active-run output watchdog created review work"); } }) - .then(async () => { - const swept = await heartbeat.sweepStaleIssueLocks(); - if (swept.cleared > 0) { - logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks"); - } - }) .catch((err) => { logger.error({ err }, "periodic heartbeat recovery failed"); })); @@ -2078,6 +1908,11 @@ async function startServerWithDatabaseTeardown( })); }; const startHeartbeatScheduler = () => { + if (isCloudManagedInstance()) { + const status = registerRoutineExecutionStatusWork(deliveryWork, routineExecutionStatusService(db as any), + () => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped); + trackHeartbeatSchedulerWork(status.ready); + } const scheduled = registerScheduledWork(deliveryWork, { retryMs: config.heartbeatSchedulerIntervalMs, tickAgentTimers: async signal => { @@ -2109,7 +1944,6 @@ async function startServerWithDatabaseTeardown( // as the enabled path. await startEnvironmentCleanup(); startHeartbeatSchedulerInterval(() => { - scheduleExternalObjectRefreshSweep(new Date()); }); } @@ -2214,9 +2048,6 @@ async function startServerWithDatabaseTeardown( heartbeatSchedulerStopped = true; skillVersionRetentionWorker?.unsubscribe(); await taskDrainExpiryWork.stop(); - clearInterval(nativeHarnessBackupExpirationInterval); - clearTimeout(nativeHarnessBackupExpirationStartup); - clearTimeout(nativeHarnessBackupExpirationContinuation); nativeRunnerEvidenceRetentionWorker?.unsubscribe(); await skillSourceSync.stop(); unsubscribeChatCompletions(); diff --git a/server/src/modules/active-run-watchdog/adapters/postgres.ts b/server/src/modules/active-run-watchdog/adapters/postgres.ts index 9531c449260..002415c03a6 100644 --- a/server/src/modules/active-run-watchdog/adapters/postgres.ts +++ b/server/src/modules/active-run-watchdog/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; @@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch )); } + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ diff --git a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts index 246cb9f707a..404e76d6b30 100644 --- a/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts +++ b/server/src/modules/agent-lifecycle/__tests__/lifecycle.test.ts @@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu let companyId: string; const workers: ReturnType[] = []; beforeAll(async () => { - if (external) { await applyPendingMigrations(external); db = createDb(external); } + if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); } else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); } }, 90_000); afterAll(async () => { diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index 1c59758dd91..96892987b01 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -80,22 +80,20 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { throw conflict("Only agents in error status can have their error cleared"); } - const updated = await db - .update(agents) - .set({ - status: "idle", - pauseReason: null, - pausedAt: null, - errorReason: null, - updatedAt: new Date(), - }) - .where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))) - .returning() - .then((rows) => rows[0] ?? null); + const updated = await effects.transaction(db, existing.companyId, async tx => { + const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState }) + .from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update"); + if (current?.status !== "error" || current.lifecycleState !== "ready") { + throw conflict("Only agents in error status can have their error cleared"); + } + await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle"); + const [saved] = await tx.update(agents).set({ + status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(), + }).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning(); + if (!saved) throw conflict("Only agents in error status can have their error cleared"); + return saved; + }); - if (!updated) { - throw conflict("Only agents in error status can have their error cleared"); - } return getById(updated.id); }, diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index cf9946bafe9..5acbfcf6884 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js"; import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js"; import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js"; @@ -511,6 +512,15 @@ export function createPostgresRunDispatchAdapter( return decideScheduledRetryGate(factsResult.facts, input.now); } + async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise { + const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.status, "scheduled_retry"), + sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`, + input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined)) + .orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1); + return row?.at?.getTime() ?? null; + } + async function listDueRetries(input: ListDueRetriesInput): Promise { const rows = await db .select() @@ -518,6 +528,7 @@ export function createPostgresRunDispatchAdapter( .where( and( eq(heartbeatRuns.status, "scheduled_retry"), + sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`, lte(heartbeatRuns.scheduledRetryAt, input.now), input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined, ), @@ -986,6 +997,7 @@ export function createPostgresRunDispatchAdapter( recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : []; })); if (commentIds.some(id => !recordedCommentIds.has(id))) return null; + await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id }); const [deferred] = await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null, payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context, @@ -1414,6 +1426,7 @@ export function createPostgresRunDispatchAdapter( return { evaluateScheduledRetryGate, + nextScheduledRetryAt, listDueRetries, cancelStaleQueuedRun, dispatchResolvedInteractionIfCurrent, diff --git a/server/src/modules/run-dispatch/application/ports.ts b/server/src/modules/run-dispatch/application/ports.ts index 83e41fd3da6..be6faf39ca7 100644 --- a/server/src/modules/run-dispatch/application/ports.ts +++ b/server/src/modules/run-dispatch/application/ports.ts @@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = { /** Read-only scheduled-retry operations exposed to application use cases. */ export interface ScheduledRetryReader { + nextScheduledRetryAt(input: { cutoff: Date | null }): Promise; evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise; listDueRetries(input: ListDueRetriesInput): Promise; } diff --git a/server/src/modules/run-dispatch/application/use-cases.test.ts b/server/src/modules/run-dispatch/application/use-cases.test.ts index 58e5ca6f8df..1cce6637ba0 100644 --- a/server/src/modules/run-dispatch/application/use-cases.test.ts +++ b/server/src/modules/run-dispatch/application/use-cases.test.ts @@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu const evaluateCalls: unknown[] = []; return { evaluateCalls, + async nextScheduledRetryAt() { return null; }, async evaluateScheduledRetryGate(input) { evaluateCalls.push(input); return { allowed: true }; diff --git a/server/src/modules/run-dispatch/index.ts b/server/src/modules/run-dispatch/index.ts index 5293ca0ae64..d44d559618a 100644 --- a/server/src/modules/run-dispatch/index.ts +++ b/server/src/modules/run-dispatch/index.ts @@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) { }); return { + nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input), evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }), promoteScheduledRetry, promoteDueScheduledRetries: createPromoteDueScheduledRetries({ diff --git a/server/src/modules/wake-queue/adapters/postgres.test.ts b/server/src/modules/wake-queue/adapters/postgres.test.ts index 068a21f2420..09fbb1749ac 100644 --- a/server/src/modules/wake-queue/adapters/postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/postgres.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js"; import { instanceSettingsService } from "../../../services/instance-settings.js"; import { randomBytes, randomUUID } from "node:crypto"; @@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js"; // a foreign-company row is invisible to a read, and a deferred-status // compare-and-set that affects no row leaves no other trace. The decision // branching itself is proven against plain facts in `domain/policy.test.ts`. -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-"); + db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 }); }, 20_000); afterEach(async () => { @@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index b888ee51fb3..2309741de26 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,6 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; +import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js"; +import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -610,6 +613,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru }, async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) { + await notifyStatusCardIssueChange(tx, companyId); const executionState = parseIssueExecutionState(issue.executionState); const wakeupRequest = await tx .insert(agentWakeupRequests) @@ -637,6 +641,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -715,6 +720,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx, { companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ @@ -1007,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { )).for("update"); if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company"); const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId); + const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload)); + const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id)); + const newInteraction = readNonEmptyString(input.mergedPayload.interactionId); + if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId)) + await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id }); const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId); const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) && priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and( @@ -1078,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { async insertNewDeferredWake(scope, input) { const tx = requireAdmissionTx(scope, input.companyId); + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.insert(agentWakeupRequests).values({ ...input.durableReceipt, companyId: input.companyId, diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 184c799f82d..a10419f0c41 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js"; import { notifyExternalAgentSettlement } from "../../external-agents/index.js"; import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js"; @@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = { function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); const updated = await tx .update(issueComments) .set({ body, updatedAt }) @@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu }, async touchIssueUpdatedAt({ issueId, updatedAt }) { + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); }, diff --git a/server/src/routes/execution-workspaces.ts b/server/src/routes/execution-workspaces.ts index e87fbbeea6b..7be8e505a94 100644 --- a/server/src/routes/execution-workspaces.ts +++ b/server/src/routes/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js"; import { removePluginWorkspace } from "../services/plugin-workspaces.js"; import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js"; import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js"; @@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true }); if (existing.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, existing.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, @@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P eq(issues.executionWorkspaceId, existing.id), ), ); + }); } try { diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index cf86556ce5e..efdf363975d 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js"; import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js"; import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js"; @@ -16674,6 +16675,7 @@ export function issueRoutes( } // The click is durable fresh user intent, including when the message // predates a failed run's stop. Keep its content and original attribution. + await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id }); await tx.update(agentWakeupRequests).set({ payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: { actorId: actor.actorId, requestedAt: new Date().toISOString(), diff --git a/server/src/routes/status-cards.ts b/server/src/routes/status-cards.ts index da4f91680bb..d21616b7cf2 100644 --- a/server/src/routes/status-cards.ts +++ b/server/src/routes/status-cards.ts @@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService, import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js"; import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js"; +// Deadline provenance belongs to scheduling, not the public card contract. +function publicCard(card: T) { + const { nextEvalReason: _reason, ...result } = card; + return result; +} + export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) { const router = Router(); const access = accessService(db); @@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake assertCompanyAccess(req, companyId); await assertStatusCardsEnabled(); const query = listStatusCardsQuerySchema.parse(req.query); - res.json(await service.list(companyId, query.archived, { + res.json((await service.list(companyId, query.archived, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + })).map(publicCard)); }); router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => { @@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake try { const compile = await enqueueCompile(req, card.id); await logMutation(req, companyId, "status_card.created", card.id, { state: card.state }); - res.status(201).json(compile.card); + res.status(201).json(publicCard(compile.card)); } catch (error) { await service.remove(card.id); throw error; @@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake await assertStatusCardsEnabled(); const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found"); if (!card) return; - res.json(await service.hydrate(card, { + res.json(publicCard(await service.hydrate(card, { issueReadCondition: await issueReadSqlCondition(db, req.actor), - })); + }))); }); router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => { @@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake fields: Object.keys(req.body), archived: Boolean(updated.archivedAt), }); - res.json(compile?.card ?? restore?.card ?? updated); + res.json(publicCard(compile?.card ?? restore?.card ?? updated)); }); router.delete("/status-cards/:id", async (req, res) => { @@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generatingIssueId: result.generatingIssue.id, alreadyGenerating: result.alreadyGenerating, }); - res.status(result.alreadyGenerating ? 200 : 202).json(result); + res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) }); }); router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => { @@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake alreadyGenerating: result.alreadyGenerating, enqueued: result.enqueued, }); - res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result); + res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) }); }); router.get("/status-cards/:id/dry-run", async (req, res) => { @@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake generationIssueId: req.body.generationIssueId, changeSummary: req.body.changeSummary, }); - res.json(updated); + res.json(publicCard(updated)); }); router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => { @@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake documentId: result.document.id, changeSummary: req.body.changeSummary, }); - res.json(result); + res.json({ ...result, card: publicCard(result.card) }); }); return router; diff --git a/server/src/services/access.ts b/server/src/services/access.ts index aa0f9dcc828..bcc1a719b71 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -743,8 +745,10 @@ export function accessService(db: Db) { sql`${issues.status} not in ('done', 'cancelled')`, ); await notifyNativeStatusReassessment(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); const resetInProgress = await tx .update(issues) .set({ diff --git a/server/src/services/accounting-integrity.ts b/server/src/services/accounting-integrity.ts index e3f5ef1f649..02b6043ab2d 100644 --- a/server/src/services/accounting-integrity.ts +++ b/server/src/services/accounting-integrity.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db"; import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared"; @@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { if (!reason.trim()) throw conflict("A repair reason is required"); const before = await inspect(tx, companyId); if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing"); + if (before.findings.some(f => f.repairable && f.kind === "agent_projection")) + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); for (const finding of before.findings.filter(f => f.repairable)) { if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId)); if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agent-configuration.ts b/server/src/services/agent-configuration.ts index 4139dc6dfe0..745dd9ff1ab 100644 --- a/server/src/services/agent-configuration.ts +++ b/server/src/services/agent-configuration.ts @@ -1,3 +1,6 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js"; import { budgetServiceInTransaction } from "./budgets.js"; export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId: existing.companyId }); const id = existing.id; const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db); if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) { @@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura existing.adapterConfig, ); } + await notifyRunDispatch(db, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId }); if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) { await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); } diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index af44a0355d8..ef8a5e554f7 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; @@ -269,6 +271,7 @@ export async function prepareConversationTurn( context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId; } } + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await tx @@ -389,6 +392,7 @@ export async function settleConversationTurn( issue.conversationState === conversationState ) return true; + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id }); @@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId), )).for("update"); if (!isConversation(issue) || comment.conversationSessionGeneration != null) return; + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const released = await tx.update(issueTreeHolds).set({ status: "released", releasedAt: new Date(), updatedAt: new Date(), releasedByActorType: "user", releasedByUserId: comment.authorUserId, diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index 3650c40f1a6..8c3defcbd86 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", { code: "agent_prompt_injection_accounting_pending", }); + await notifyAttentionRetentionWork(tx, { companyId }); await notifyExternalConversationSettlement(tx, { companyId }); await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id)); await tx .update(issues) diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index a150937a212..0c238907bbc 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js"; import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared"; import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils"; @@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js"; import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { + await notifyAttentionRetentionWork(db, { companyId }); const { runner, ...data } = input; const selection = options?.runnerResolved ? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index 0a6dcaf0fc9..b7d42462ba0 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -1,3 +1,6 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { isAgentStatusInvokable } from "@paperclipai/shared"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li }, notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle), async beforeStatusChange(tx, companyId, agentId, before, after) { - if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyAttentionRetentionWork(tx, { companyId }); + if (before !== after) { + await notifyRunDispatch(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + } if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) { await notifyNativeResponsePresentation(tx, { companyId, agentId }); await notifyNativeReviewResponse(tx, { companyId, agentId }); diff --git a/server/src/services/agent-project-access.ts b/server/src/services/agent-project-access.ts index 4c6ee81659a..feb183924c1 100644 --- a/server/src/services/agent-project-access.ts +++ b/server/src/services/agent-project-access.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { and, eq, inArray } from "drizzle-orm"; import { agents, projects, type Db } from "@paperclipai/db"; import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared"; @@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c const added = requested.filter(id => !previous.includes(id)); const next = [...previous, ...added]; if (added.length) { + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: { ...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next }, } }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId))); diff --git a/server/src/services/agents.ts b/server/src/services/agents.ts index c9dd824f495..8a5f7551ee2 100644 --- a/server/src/services/agents.ts +++ b/server/src/services/agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { type BudgetServiceHooks } from "./budgets.js"; import { and, desc, eq, inArray } from "drizzle-orm"; @@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { }); } - const updated = await db + const updated = await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); + return tx .update(agents) .set({ permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }), @@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) { .where(eq(agents.id, id)) .returning() .then((rows) => rows[0] ?? null); + }); return updated ? getById(updated.id) : null; }, diff --git a/server/src/services/attention-retention.ts b/server/src/services/attention-retention.ts new file mode 100644 index 00000000000..a0bca4281f5 --- /dev/null +++ b/server/src/services/attention-retention.ts @@ -0,0 +1,320 @@ +import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm"; +import { agents, decisionRetention, decisions, heartbeatRuns, + issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db"; +import { evaluateAgentInvokability } from "./agent-invokability.js"; +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; +import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { attentionIssueCondition } from "./issue-visibility.js"; +import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js"; +import { issueService } from "./issues.js"; +import { parseIssueExecutionState } from "./issue-execution-policy.js"; +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { AttentionRetentionScope } from "./retention-work-notifications.js"; + +const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000; +const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review", + "failed_run", "budget_alert", "agent_error_alert"] as const; +type Kind = typeof kinds[number]; +type Scope = AttentionRetentionScope; +type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null }; +type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" }; +type Cursor = { sourceKind: string; sourceId: string }; + +type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number; + blockedActivityAt: string | null; blockedDeadlineAt: number | null }; +type PageEvidence = { + decisions: Map; + recoveries: Map; + runs: Map; + agents: Set; + visibleIssues: Map; + newerRuns: Set; + deferredRoots: Set; + blockers: Awaited["listBlockerAttention"]>>; +}; + +const sourceTables: Record = { + decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`, + recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`, + failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`, +}; + +const sources: Record = { + decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id + from decisions d where d.status = 'open'`, + issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text + from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`, + recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text + from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`, + blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text + from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`, + review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text + from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`, + failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId') + from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists ( + select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id + and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`, + budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b + where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0 + and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`, + agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`, +}; + +export function attentionRetentionService(db: Db) { + const cursors = new Map(); + async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise { + const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []); + const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind) + && (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind))); + if (!selected.length) return []; + const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)}) + select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId", + s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt" + from source_candidates s inner join companies c on c.id = s.company_id + left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id + where c.status = 'active' + ${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``} + and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid) + and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null}) + and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null)) + ${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``} + order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[]; + return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) })); + } + + async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) { + const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), + attentionIssueCondition(conversations))); + return row; + } + + async function interactionEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "issue_thread_interaction") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids), + eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId))); + if (!rows.length) continue; + const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true))); + const visibleIds = new Set(visible.map(row => row.id)); + const roster = await database.select().from(agents).where(eq(agents.companyId, companyId)); + const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId + || !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable; + const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId); + const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and( + eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds), + eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"), + isNull(issueThreadInteractions.addresseeUserId))) : []; + const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id)); + for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row) + && (row.kind !== "request_confirmation" || newest.has(row.id))); + } + return result; + } + + async function reviewEligibility(database: Db, candidates: Candidate[]) { + const result = new Map(); + const byCompany = new Map(); + for (const candidate of candidates) { + if (candidate.sourceKind !== "review") continue; + const ids = byCompany.get(candidate.companyId) ?? []; + ids.push(candidate.sourceId); + byCompany.set(candidate.companyId, ids); + } + for (const [companyId, ids] of byCompany) { + const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId), + inArray(issues.id, ids), attentionIssueCondition(false))); + const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals) + .innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and( + eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId), + inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending"))); + const approved = new Set(pendingApprovals.map(row => row.issueId)); + const stalled = await issueService(database).listReviewAttention(companyId, rows); + for (const row of rows) { + const state = parseIssueExecutionState(row.executionState); + result.set(row.id, requiresReviewAttention({ + hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user", + hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id), + stalled: stalled.get(row.id)?.state === "stalled", + })); + } + } + return result; + } + + async function pageEvidence(database: Db, candidates: Candidate[]) { + const byCompany = new Map(); + for (const candidate of candidates) { + if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue; + const rows = byCompany.get(candidate.companyId) ?? []; + rows.push(candidate); + byCompany.set(candidate.companyId, rows); + } + const result = new Map(); + for (const [companyId, rows] of byCompany) { + const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId); + const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention"); + const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId }) + .from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : []; + const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and( + eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : []; + const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : []; + const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and( + eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]), + sql`${agents.status} <> 'terminated'`)) : []; + const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues), + activityAt: sql`${issues.updatedAt}::text`, + deadlineAt: sql`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + blockedActivityAt: sql`${issues.blockedTransitionAt}::text`, + blockedDeadlineAt: sql`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number), + }).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]), + attentionIssueCondition(false))) : []; + const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId), + ...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id)); + const visibleRows = await readIssues(issueIds); + const visible = new Map(visibleRows.map(row => [row.id, row])); + const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id)); + const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots) + : { attention: new Map(), deferredRootIds: new Set() }; + const blockers: PageEvidence["blockers"] = blockerBatch.attention; + const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention) + .map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id)); + for (const row of await readIssues(terminalIds)) visible.set(row.id, row); + result.set(companyId, { + decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])), + runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible, + newerRuns: await listFailedRunsWithNewerRun(database, companyId, + runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds, + }); + } + return result; + } + + async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map, interactionAttention?: Map, evidence?: PageEvidence): Promise { + if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) { + return { ...candidate, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() }; + } + let sourceId = candidate.sourceId; + let activityAt = candidate.activityAt; + let dueAt = candidate.deadlineAt; + if (candidate.sourceKind === "decision") { + const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions) + .where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0]; + if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId) + : await visibleIssue(database, candidate.companyId, row.originIssueId))) return null; + } else if (candidate.sourceKind === "issue_thread_interaction") { + const attention = interactionAttention ?? await interactionEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; + } else if (candidate.sourceKind === "recovery_action") { + const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0]; + if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId) + : await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null; + } else if (candidate.sourceKind === "review") { + const attention = reviewAttention ?? await reviewEligibility(database, [candidate]); + if (!attention.get(sourceId)) return null; + } else if (candidate.sourceKind === "blocker_attention") { + const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); + if (!row) return null; + if (variant === "owner") { + if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null; + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at, + ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; + activityAt = activity.at; dueAt = Number(activity.due); + } else { + const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id); + if (!terminalBlockerRequiresAttention(attention)) return null; + sourceId = attention.terminalBlockerIssueId ?? row.id; + if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`); + const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId); + if (!terminal) return null; + const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at, + ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues + where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0]; + activityAt = activity.at; dueAt = Number(activity.due); + } + const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId), + eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId))); + if (retention?.keep || retention?.archivedAt) return null; + } else if (candidate.sourceKind === "failed_run") { + const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0]; + if (!run) return null; + const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId), + eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0]; + const issueId = readRunIssueId(run.contextSnapshot); + if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId) + : await visibleIssue(database, candidate.companyId, issueId))) return null; + if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null; + } + return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant, + id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt }; + } + + async function load(scope?: Scope, signal?: AbortSignal) { + const key = JSON.stringify(scope ?? {}); + const rows = await candidates(db, scope, cursors.get(key)); + const entries: Entry[] = []; + const reviews = await reviewEligibility(db, rows); + const interactions = await interactionEligibility(db, rows); + const evidence = await pageEvidence(db, rows); + for (const row of rows) { + if (signal?.aborted) return { rows: null, complete: true }; + for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) { + const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId)); + if (entry) entries.push(entry); + } + } + if (rows.length === 100) { + const last = rows[rows.length - 1]!; + cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId }); + } else cursors.delete(key); + return { rows: entries, complete: rows.length < 100 }; + } + + async function check(entry: Entry) { + return db.transaction(async transaction => { + const database = transaction as unknown as Db; + await database.execute(sql`select id from ${sourceTables[entry.sourceKind]} + where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`); + const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId); + if (!candidate) return null; + const current = await eligible(database, candidate, entry.variant, true); + if (!current || current.dueAt > Date.now()) return current; + const retention = decisionRetentionService(database); + const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt }; + const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!; + await retention.autoArchive({ companyId: current.companyId, items: [{ ...item, + keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] }); + return null; + }); + } + return { load, check }; +} + +export function registerAttentionRetentionWork(coordinator: ReturnType, + service: ReturnType, canRun: () => boolean) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.attentionRetention, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope, + matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId) + && (!scope?.sourceKind || scope.sourceKind === entry.sourceKind) + && (!scope?.sourceId || scope.sourceId === entry.sourceId) + && (!scope?.issueId || scope.issueId === entry.issueId) + && (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)), + load: service.load, check: service.check, + }, canRun); +} diff --git a/server/src/services/attention-source-eligibility.ts b/server/src/services/attention-source-eligibility.ts new file mode 100644 index 00000000000..f129c522130 --- /dev/null +++ b/server/src/services/attention-source-eligibility.ts @@ -0,0 +1,112 @@ +import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm"; +import { heartbeatRuns, type Db } from "@paperclipai/db"; + +export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; +export const ATTENTION_BUDGET_WARNING_PERCENT = 85; +export const ATTENTION_AGENT_ERROR_STATUS = "error"; + +export function collapsePendingConfirmationsToNewest(rows: T[]) { + const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); + const newestByGroup = new Map(); + for (const row of rows) { + if (row.kind !== "request_confirmation" || independent(row)) continue; + const groupKey = `${row.issueId}:${row.kind}`; + const newest = newestByGroup.get(groupKey); + if ( + !newest + || row.createdAt.getTime() > newest.createdAt.getTime() + || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) + ) { + newestByGroup.set(groupKey, row); + } + } + + return rows.filter((row) => ( + row.kind !== "request_confirmation" + || independent(row) + || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id + )); +} + +export function readRunIssueId(contextSnapshot: Record | null) { + const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; + return typeof issueId === "string" && issueId.length > 0 ? issueId : null; +} + +type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; + +const FAILED_RUN_PROBE_BATCH_SIZE = 1000; + +export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + const withNewerRun = new Set(); + const keyed = failedRuns.filter(run => run.issueId !== null); + for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); + for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); + } + const unkeyed = failedRuns.filter(run => run.issueId === null); + if (unkeyed.length === 0) return withNewerRun; + const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; + const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, + unkeyed[0]!.createdAt); + const latestByAgent = new Map(); + for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { + const rows = await db.select({ + agentId: heartbeatRuns.agentId, + createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), + }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, companyId), + inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), + gt(heartbeatRuns.createdAt, oldestCreatedAt), + sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, + )).groupBy(heartbeatRuns.agentId); + for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); + } + for (const run of unkeyed) { + if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); + } + return withNewerRun; +} + +async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { + if (failedRuns.length === 0) return []; + const failed = sql.join( + failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), + sql`, `, + ); + const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; + const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; + const newerRunExists = (match: SQL) => sql`exists ( + select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${companyId} + and ${heartbeatRuns.agentId} = failed.agent_id + and ${heartbeatRuns.createdAt} >= failed.newer_from + and ${match} + )`; + const rows = await db.execute(sql` + select failed.id + from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) + where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} + or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} + `) as unknown as Array<{ id: string }>; + return rows.map((row) => row.id); +} + + +export function requiresReviewAttention(input: { + hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean; +}) { + return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled; +} + +export function terminalBlockerRequiresAttention( + attention: T | null | undefined, +): attention is T & { state: "stalled" | "needs_attention" } { + return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? "")); +} diff --git a/server/src/services/attention.ts b/server/src/services/attention.ts index 8d5dedd3b03..cef6019f852 100644 --- a/server/src/services/attention.ts +++ b/server/src/services/attention.ts @@ -1,5 +1,6 @@ +import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js"; import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js"; -import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agents, @@ -58,7 +59,7 @@ import { BLOCKER_ATTENTION_MAX_NODES, issueService, } from "./issues.js"; -import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js"; +import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js"; import { parseIssueExecutionState } from "./issue-execution-policy.js"; import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js"; @@ -107,7 +108,6 @@ const SOURCE_RANK: Record = { const PENDING_INTERACTION_STATUSES = ["pending"] as const; const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const; -const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const; const DETAIL_EXCERPT_LENGTH = 160; const DETAIL_IMAGE_LIMIT = 3; const OPEN_DECISION_DEFAULT_LIMIT = 500; @@ -766,37 +766,6 @@ export function interactionResolverAudience( }; } -function collapsePendingConfirmationsToNewest(rows: T[]) { - // Quality gates bind independent outputs; a newer visual approval must not - // hide a still-pending copy approval (or affect its badge count). - const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview); - const newestByGroup = new Map(); - for (const row of rows) { - if (row.kind !== "request_confirmation" || independent(row)) continue; - const groupKey = `${row.issueId}:${row.kind}`; - const newest = newestByGroup.get(groupKey); - if ( - !newest - || row.createdAt.getTime() > newest.createdAt.getTime() - || (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id) - ) { - newestByGroup.set(groupKey, row); - } - } - - return rows.filter((row) => ( - row.kind !== "request_confirmation" - || independent(row) - || newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id - )); -} - function budgetObservedPercent(amountObserved: number, amountLimit: number) { return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0; } @@ -866,9 +835,7 @@ async function issueSummaryMap( .where(and( eq(issues.companyId, companyId), inArray(issues.id, ids), - includeConversations - ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId))) - : executionIssueCondition(), + attentionIssueCondition(includeConversations), readCondition, )); return new Map(rows.map((row) => [row.id, { @@ -1108,70 +1075,6 @@ function resolveBlockingIssue( return null; } -function readRunIssueId(contextSnapshot: Record | null) { - const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId; - return typeof issueId === "string" && issueId.length > 0 ? issueId : null; -} - -type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date }; - -const FAILED_RUN_PROBE_BATCH_SIZE = 1000; - -async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - const withNewerRun = new Set(); - const keyed = failedRuns.filter(run => run.issueId !== null); - for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE); - for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id); - } - const unkeyed = failedRuns.filter(run => run.issueId === null); - if (unkeyed.length === 0) return withNewerRun; - const agentIds = [...new Set(unkeyed.map(run => run.agentId))]; - const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest, - unkeyed[0]!.createdAt); - const latestByAgent = new Map(); - for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) { - const rows = await db.select({ - agentId: heartbeatRuns.agentId, - createdAt: sql`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt), - }).from(heartbeatRuns).where(and( - eq(heartbeatRuns.companyId, companyId), - inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)), - gt(heartbeatRuns.createdAt, oldestCreatedAt), - sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`, - )).groupBy(heartbeatRuns.agentId); - for (const row of rows) latestByAgent.set(row.agentId, row.createdAt); - } - for (const run of unkeyed) { - if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id); - } - return withNewerRun; -} - -async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) { - if (failedRuns.length === 0) return []; - const failed = sql.join( - failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`), - sql`, `, - ); - const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`; - const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`; - const newerRunExists = (match: SQL) => sql`exists ( - select 1 from ${heartbeatRuns} - where ${heartbeatRuns.companyId} = ${companyId} - and ${heartbeatRuns.agentId} = failed.agent_id - and ${heartbeatRuns.createdAt} >= failed.newer_from - and ${match} - )`; - const rows = await db.execute(sql` - select failed.id - from (values ${failed}) as failed (id, agent_id, issue_id, newer_from) - where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)} - or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)} - `) as unknown as Array<{ id: string }>; - return rows.map((row) => row.id); -} - export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) { const openDecisionLimit = Math.min( Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1), @@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions })); } const blockerAttention = issue.blockerAttention; - if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue; - if (blockerAttention.blockingTreeLive) continue; + if (!terminalBlockerRequiresAttention(blockerAttention)) continue; const issueSummary = blockedIssueSummaries.get(issue.id) ?? null; const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id; const terminalSummary = terminalBlockerSummaries.get(terminalIssueId) @@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null; const reviewAttention = reviewAttentionByIssueId.get(review.id); const stalled = reviewAttention?.state === "stalled"; - if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue; + if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId), + hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue; const issue = reviewIssueMap.get(review.id); if (!issue) continue; const dedupKey = `review:${review.id}`; @@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions const budgetOverview = await budgetService(db).overview(companyId); for (const incident of budgetOverview.activeIncidents) { const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit); - if (incident.thresholdType !== "hard" && observedPercent < 85) continue; + if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue; const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`; add(createItem({ companyId, @@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions updatedAt: agents.updatedAt, }) .from(agents) - .where(and(eq(agents.companyId, companyId), eq(agents.status, "error"))) + .where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS))) .orderBy(desc(agents.updatedAt), desc(agents.id)); for (const agent of erroredAgents) { diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 6777124ee90..cf76f7592e2 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -1,5 +1,9 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; -import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js"; import { addCents, compareCents, normalizeCents } from "@paperclipai/shared"; import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -163,6 +167,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s }; } +async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise { + let scope: ScopeRecord; + try { + scope = await resolveScopeRecord(db, scopeType, scopeId); + } catch (error) { + if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error; + return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project", + paused: false, pauseReason: null, budgetHeld: false }; + } + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); + return scope; +} + // Keep a large dashboard from filling the database pool with queued reads. async function mapBudgetReads(rows: T[], read: (row: T) => Promise): Promise { const result: R[] = []; @@ -287,6 +304,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -300,7 +319,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.paused) return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -346,6 +369,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (policy.scopeType === "project") { + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await db .update(projects) .set({ @@ -360,7 +385,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync); const scope = await resolveScopeRecord(db, "company", policy.scopeId); if (scope.pauseReason !== "budget") return; + await notifyRunDispatch(db, { companyId: policy.companyId }); + await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId }); await notifyCompanyRetentionWork(db, policy.companyId); + await notifyExternalObjectRefresh(db, policy.companyId); + await notifyStatusCardIssueChange(db, policy.companyId); await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors); @@ -395,7 +424,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } async function buildPolicySummary(policy: PolicyRow): Promise { - const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId); const observed = await computeObservedSpend(db, policy); const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed; const { windowStart: start, windowEnd: end } = observed; @@ -466,6 +495,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica windowEnd: end, }); + await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" }); if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId); const approval = thresholdType === "hard" ? await db @@ -504,7 +534,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica return incident ? { incident, created: true } : null; } - async function resolveOpenSoftIncidents(policyId: string) { + async function resolveOpenSoftIncidents(policyId: string, companyId: string) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await db .update(budgetIncidents) .set({ @@ -523,10 +554,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica async function resolveOpenIncidentsForPolicy( policyId: string, + companyId: string, approvalStatus: "approved" | "rejected" | null, decidedByUserId: string | null, thresholdType?: BudgetThresholdType, ) { + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"), thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined); const openRows = await db @@ -560,7 +593,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status])); return mapBudgetReads(rows, async (row) => { - const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId); + const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId); return { id: row.id, companyId: row.companyId, @@ -615,9 +648,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy; // A closed UTC window, inactive policy, or budget raise releases only // budget-owned pauses. Manual pauses remain operator decisions. - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard"); if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { - await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft"); + await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft"); } } // Agent status has its own admission gate and error code. Preserve a @@ -674,7 +707,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } if (observedBlocks(policy, observed)) { - await resolveOpenSoftIncidents(policy.id); + await resolveOpenSoftIncidents(policy.id, policy.companyId); const hardIncident = await createIncidentIfNeeded(policy, "hard", observed); await pauseAndCancelScopeForBudget(policy); if (hardIncident?.created) { @@ -844,11 +877,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica const observed = await computeObservedSpend(db, row); if (!observedBlocks(row, observed)) { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard"); if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) { await createIncidentIfNeeded(row, "soft", observed); } else { - await resolveOpenIncidentsForPolicy(row.id, null, null, "soft"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft"); } } else { const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100); @@ -856,14 +889,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica await createIncidentIfNeeded(row, "soft", observed); } if (observedBlocks(row, observed)) { - await resolveOpenSoftIncidents(row.id); + await resolveOpenSoftIncidents(row.id, row.companyId); await createIncidentIfNeeded(row, "hard", observed); await pauseAndCancelScopeForBudget(row); } } } else { await resumeScopeFromBudget(row); - await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service"); + await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service"); } await recordActivity({ @@ -925,6 +958,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica if (incident.companyId !== companyId) throw notFound("Budget incident not found"); const policy = await getPolicyRow(incident.policyId); + if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) { + throw notFound("Budget scope not found"); + } + const scope = input.action === "keep_paused" + ? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId) + : await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId); + if (scope.companyId !== companyId) throw notFound("Budget scope not found"); if (input.action === "raise_budget_and_resume") { const nextAmount = Math.max(0, Math.floor(input.amount ?? 0)); const observed = await computeObservedSpend(db, policy); @@ -937,6 +977,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica } const now = new Date(); + await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" }); await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId); await db .update(budgetPolicies) diff --git a/server/src/services/chat-channels/failed-run-retry.ts b/server/src/services/chat-channels/failed-run-retry.ts index c324a6f3c34..f8b8c3ebeab 100644 --- a/server/src/services/chat-channels/failed-run-retry.ts +++ b/server/src/services/chat-channels/failed-run-retry.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js"; import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js"; import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; import { createFailedRetryNotices } from "./failed-retry-notices.js"; @@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti // Source authorization above checked every external receipt and its owner. // Reset only this certified unstarted batch under the task's admission lock. const skippedIds = source.taskRetryInput?.skippedActionIds ?? []; + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null, payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId), @@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1; const settle = async (status: string, result: Record) => { const [current] = await db.transaction(async (tx) => { + if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, { + companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId, + }); await notifyChatActionWork(tx, "failed_run_retry"); if (status === "processed") await notifyNativeResponsePresentation(tx, { companyId: claimed.companyId, issueId: String(claimed.payload.issueId), diff --git a/server/src/services/chat-channels/task-retry-input.ts b/server/src/services/chat-channels/task-retry-input.ts index 71743f42ad4..8257a8ec079 100644 --- a/server/src/services/chat-channels/task-retry-input.ts +++ b/server/src/services/chat-channels/task-retry-input.ts @@ -1,3 +1,4 @@ +import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; import { z } from "zod"; import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db"; @@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId )).for("update"); if (receipt?.runId) return; await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id }); + await notifyDeferredExecutionWork(tx, { companyId: input.companyId }); await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null, payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(), }).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"), diff --git a/server/src/services/chat-channels/telegram/attachment-failure.ts b/server/src/services/chat-channels/telegram/attachment-failure.ts index 58d5d4a2677..a7e9e4367e3 100644 --- a/server/src/services/chat-channels/telegram/attachment-failure.ts +++ b/server/src/services/chat-channels/telegram/attachment-failure.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "../../status-card-notifications.js"; +import { notifyIssueRecovery } from "../../issue-recovery-notifications.js"; import { createProviderEffects, type ProviderEffectTarget } from "../provider-effects.js"; import type { EndpointRow, RuntimeContext, InboundRuntimeContext } from "../runtime-types.js"; @@ -78,10 +80,12 @@ export function createTelegramAttachmentFailure(db: Db, options: TelegramAttachm })(), ); const effect = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId: input.endpoint.companyId }); // Replace the optimistic attachment placeholder with an explicit safe // Paperclip-visible failure. The message link stays authoritative for // exact provider redelivery, while no agent wakeup is enqueued for an // empty turn that contains no usable bytes. + await notifyStatusCardIssueChange(tx, input.endpoint.companyId); await tx .update(issueComments) .set({ body: visibleFailure, updatedAt: new Date() }) diff --git a/server/src/services/chat-confirmation-notifications.ts b/server/src/services/chat-confirmation-notifications.ts index e55d8c0d7ac..02cdfbee8c8 100644 --- a/server/src/services/chat-confirmation-notifications.ts +++ b/server/src/services/chat-confirmation-notifications.ts @@ -1,3 +1,4 @@ +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type ChatConfirmationScope = { companyId: string; issueId?: string }; @@ -10,6 +11,7 @@ export async function notifyChatConfirmationWork(tx: object, scope: ChatConfirma /** Interaction resolution can release a callback parked while its human input * was pending. Register on the owning transaction before its resolution write. */ export async function notifyChatConfirmationResolution(tx: object, interaction: ChatConfirmationScope & { kind: string }) { + if (interaction.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, interaction); if (["request_confirmation", "request_checkbox_confirmation", "suggest_tasks", "connection_intent"].includes(interaction.kind)) { await notifyChatConfirmationWork(tx, interaction); } diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts index 73535f5434a..524936395b5 100644 --- a/server/src/services/chat-run-milestone-notifications.ts +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; @@ -11,6 +12,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type ChatRunMilestoneScope = { companyId: string; issueId?: string; runId?: string }; export async function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { + if (!scope.issueId && !scope.runId) await notifyRunDispatch(tx, scope); if (scope.issueId || scope.runId) { await notifyIssueReviewState(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.issueId ? undefined : scope.runId }); diff --git a/server/src/services/companies.ts b/server/src/services/companies.ts index d26f41c761f..6b513ea3763 100644 --- a/server/src/services/companies.ts +++ b/server/src/services/companies.ts @@ -1,6 +1,10 @@ import { notifyGitHubCompanyChange } from "./github-connection-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { notifyCompanyRetentionWork } from "./retention-work-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { isRunnerApiResponseAsset } from "./asset-content-access.js"; import { deleteCompany } from "./company-deletion.js"; @@ -334,10 +338,14 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (companyPatch.status !== undefined && companyPatch.status !== existing.status) { await notifyGitHubCompanyChange(tx, id); + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); @@ -491,10 +499,14 @@ export function companyService(db: Db, _budgetHooks: BudgetServiceHooks = {}) { if (!wasAlreadyArchived) { await notifyGitHubCompanyChange(tx, id); + await notifyRunDispatch(tx, { companyId: id }); + await notifyDeferredExecutionEligibility(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyExternalObjectRefresh(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, id); await notifyCompanyRetentionWork(tx, id); await notifyChatInboundWakeNotices(tx, { companyId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle, id); diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 7ab88aadfa9..37894b17113 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -1,3 +1,6 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyCancelledWorkspacePreservation } from "./cancelled-workspace-preservation-notifications.js"; import { notifyInstructionCollection } from "./instruction-collection-notifications.js"; @@ -71,12 +74,15 @@ export async function deleteCompany(db: Db, id: string) { const [existing] = await tx.select({ id: companies.id }).from(companies) .where(eq(companies.id, id)).for("no key update"); if (!existing) return null; + await notifyExternalObjectRefresh(tx, id); + await notifyIssueRecovery(tx, { companyId: id }); await notifyChatCompanyConfiguration(tx, id); await chatCompanyDeletion.deleteCompanyData(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillPackages); await notifyDeliveryWork(tx, DELIVERY_QUEUES.routineSchedules); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardEvaluation(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.secretProposalExpiry, id); await notifyToolReviewChange(tx, id); await agentObservabilityService(tx as unknown as Db).removeForCompanyDeletion(id); diff --git a/server/src/services/company-portability.ts b/server/src/services/company-portability.ts index dc7dca311d6..0b41482b0fa 100644 --- a/server/src/services/company-portability.ts +++ b/server/src/services/company-portability.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { agentAppearanceSchema, agentHarnessType, paperclipRunnerProfileForHarness, agentRunner, type AgentRunnerChoice } from "@paperclipai/shared"; import { resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; @@ -6465,7 +6467,10 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { } if (relationRows.length > 0) { const relationCompanyId = targetCompany.id; - await db + await db.transaction(async tx => { + await notifyAttentionRetentionWork(tx, { companyId: relationCompanyId }); + await notifyIssueRecovery(tx, { companyId: relationCompanyId }); + await tx .insert(issueRelations) .values(relationRows.map((row) => ({ companyId: relationCompanyId, @@ -6476,6 +6481,7 @@ export function companyPortabilityService(db: Db, storage?: StorageService) { createdByUserId: actorUserId ?? null, }))) .onConflictDoNothing(); + }); } } diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index 220967e362d..e6595beebd8 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -207,14 +207,20 @@ function priorityRank(priority: string | null | undefined) { return index >= 0 ? index : ISSUE_PRIORITIES.length; } -function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { +/** Duration used by saved search membership and its deadline consumers. */ +export function companySearchUpdatedWithinMs(value: string | undefined): number | null { if (!value) return null; const match = /^(\d+)(h|d|w|m)$/.exec(value); if (!match) return null; const amount = Number.parseInt(match[1]!, 10); const unit = match[2]; const hours = unit === "h" ? amount : unit === "d" ? amount * 24 : unit === "w" ? amount * 24 * 7 : amount * 24 * 30; - return new Date(now.getTime() - hours * 60 * 60 * 1000); + return hours * 60 * 60 * 1000; +} + +function updatedWithinStart(value: string | undefined, now = new Date()): Date | null { + const duration = companySearchUpdatedWithinMs(value); + return duration === null ? null : new Date(now.getTime() - duration); } function issueOnlyFiltersActive(query: CompanySearchQuery) { @@ -542,7 +548,7 @@ export function companySearchService(db: Db) { search: async ( companyId: string, query: CompanySearchQuery, - options?: { issueReadCondition?: SQL; projectReadCondition?: SQL }, + options?: { issueReadCondition?: SQL; projectReadCondition?: SQL; onNextMembershipChange?: (at: Date | null) => void }, ): Promise => { const taskSearch = parseTaskSearch(query.q); const normalizedQuery = taskSearch.normalizedQuery; @@ -553,6 +559,7 @@ export function companySearchService(db: Db) { const limit = query.limit; const offset = query.offset; if (!hasSearchText && !issueOnlyFiltersActive(query)) { + options?.onNextMembershipChange?.(null); return { query: query.q, normalizedQuery, @@ -721,6 +728,13 @@ export function companySearchService(db: Db) { } } + const membershipWindow = companySearchUpdatedWithinMs(query.updatedWithin); + if (options?.onNextMembershipChange && membershipWindow !== null && scopeIncludesIssues(scope)) { + // Includes rows outside the visible page: their expiry can move an + // offset/priority-ranked result into the watched set without a write. + branches.push(sql`SELECT 'membership-expiry' AS kind, min(m.updated_at)::text AS value, + 0::int AS count ${countTail} FROM matched m ${branchWhere([scopeCond, ...facetsAll])}`); + } const resultRows = await db.execute(sql` ${taskSearchCtes(companyId, taskSearch, scope !== "issues", and(...issueFilters), options?.issueReadCondition)} ${sql.join(branches, sql` UNION ALL `)} @@ -732,6 +746,7 @@ export function companySearchService(db: Db) { totals: { current: 0, unfiltered: 0, omit: {} }, }; const issueRowsRaw: IssueSearchRow[] = []; + let nextMembershipChange: Date | null = null; for (const row of resultRows) { if (row.kind === "result") { issueRowsRaw.push({ @@ -756,6 +771,9 @@ export function companySearchService(db: Db) { }); continue; } + if (row.kind === "membership-expiry" && row.value && membershipWindow !== null) { + nextMembershipChange = new Date(new Date(row.value).getTime() + membershipWindow + 1); + } const count = Number(row.count ?? 0); if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; @@ -776,6 +794,7 @@ export function companySearchService(db: Db) { aggregates.totals.omit[row.kind.slice("total:omit:".length) as CompanySearchIssueFilterKey] = count; } } + options?.onNextMembershipChange?.(nextMembershipChange); return { rows: await enrichIssueSnippets(issueRowsRaw), aggregates }; } diff --git a/server/src/services/company-skills.ts b/server/src/services/company-skills.ts index 8ba95a27e99..cd75e0ca00e 100644 --- a/server/src/services/company-skills.ts +++ b/server/src/services/company-skills.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { skillExecutionContent } from "./skill-execution-content.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { managedSkillContent, type SkillContentPublication } from "./managed-skill-content.js"; @@ -7340,6 +7341,7 @@ export function companySkillService(db: Db, options: { actor?: AuthorizationActo for (const row of rows) { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx .update(issues) .set({ hiddenAt: now, updatedAt: now }) diff --git a/server/src/services/decision-expiry.ts b/server/src/services/decision-expiry.ts index ea5236a6ac0..796200128a7 100644 --- a/server/src/services/decision-expiry.ts +++ b/server/src/services/decision-expiry.ts @@ -23,7 +23,7 @@ export function decisionExpiryService(db: Db, deliverContinuation: (id: string) agentId: row.originAgentId, runId: row.originRunId, action: "decision.expired", entityType: "decision", entityId: row.id, details: { expiredReason: reason } }, publications); return row; - }); + }, { companyId: decision.companyId, sourceId: decision.id }); if (!updated) return 0; for (const publication of publications) publishActivity(publication); await deliverContinuation(updated.id); diff --git a/server/src/services/decision-retention.ts b/server/src/services/decision-retention.ts index fb22df2a583..b5ca6c9f1fb 100644 --- a/server/src/services/decision-retention.ts +++ b/server/src/services/decision-retention.ts @@ -1,3 +1,4 @@ +import { isProspectiveBlockedTransition } from "./routable-blocked.js"; import { notifyRetentionWork } from "./retention-work-notifications.js"; import { decisionArchiveDeliveryService, type ArchiveNotificationBatch } from "./decision-archive-delivery.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -15,7 +16,6 @@ import { issueRecoveryActions, issueThreadInteractions, issues, - joinRequests, } from "@paperclipai/db"; import type { AttentionArchiveManifestEntry, @@ -145,7 +145,7 @@ export function decisionRetentionService( const existing = new Set(rows.map(row => sourceKey(row.sourceKind, row.sourceId))); const missing = unique.filter(item => !existing.has(itemSourceKey(item))); if (missing.length) { - for (const kind of new Set(missing.map(item => item.sourceKind))) await notifyRetentionWork(tx, kind); + for (const item of missing) await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); await tx.insert(decisionRetention).values(missing.map(item => ({ companyId, sourceKind: item.sourceKind, sourceId: item.subject.id, sourceActivityAt: new Date(item.activityAt), }))).onConflictDoNothing({ target: [decisionRetention.companyId, decisionRetention.sourceKind, decisionRetention.sourceId] }); @@ -155,7 +155,7 @@ export function decisionRetentionService( for (const item of unique) { const key = itemSourceKey(item), row = byKey.get(key), activityAt = new Date(item.activityAt); if (!row || row.sourceActivityAt.getTime() === activityAt.getTime()) continue; - await notifyRetentionWork(tx, item.sourceKind); + await notifyRetentionWork(tx, item.sourceKind, { companyId, sourceId: item.subject.id }); const [updated] = await tx.update(decisionRetention).set({ sourceActivityAt: activityAt, version: sql`${decisionRetention.version} + 1`, updatedAt: new Date(), }).where(and(eq(decisionRetention.id, row.id), ne(decisionRetention.sourceActivityAt, activityAt))).returning(); @@ -169,9 +169,9 @@ export function decisionRetentionService( return sourceKey(item.sourceKind, item.subject.id); } - async function writeRetention(kind: AttentionSourceKind, write: (tx: Db) => Promise) { + async function writeRetention(companyId: string, kind: AttentionSourceKind, sourceId: string, write: (tx: Db) => Promise) { return db.transaction(async tx => { - await notifyRetentionWork(tx, kind); + await notifyRetentionWork(tx, kind, { companyId, sourceId }); return write(tx as unknown as Db); }); } @@ -185,13 +185,28 @@ export function decisionRetentionService( } async function materializeSourceState(database: Db, input: { companyId: string; sourceKind: AttentionSourceKind; sourceId: string }) { - const table = input.sourceKind === "approval" ? approvals : input.sourceKind === "join_request" ? joinRequests : null; - if (!table) return; - const [source] = await database.select({ updatedAt: table.updatedAt }).from(table) - .where(and(eq(table.companyId, input.companyId), eq(table.id, input.sourceId))).for("update"); + const tables: Record = { + approval: "approvals", join_request: "join_requests", decision: "decisions", + issue_thread_interaction: "issue_thread_interactions", recovery_action: "issue_recovery_actions", + failed_run: "heartbeat_runs", productivity_review: "issues", blocker_attention: "issues", review: "issues", + budget_alert: "budget_incidents", agent_error_alert: "agents", + }; + if (input.sourceKind === "blocker_attention") { + const [source] = await database.select().from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, input.sourceId))).for("update"); + if (!source) throw notFound("Attention source not found"); + const activityAt = source.unblockDescriptor?.owner === "board" && isProspectiveBlockedTransition(source) + ? source.blockedTransitionAt : source.updatedAt; + await decisionRetentionService(database).syncItems(input.companyId, [{ + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: activityAt.toISOString(), + }]); + return; + } + const activity = input.sourceKind === "failed_run" ? sql`coalesce(finished_at, updated_at, created_at)` : sql`updated_at`; + const [source] = await database.execute(sql`select ${activity} as "activityAt" from ${sql.identifier(tables[input.sourceKind])} + where company_id = ${input.companyId}::uuid and id = ${input.sourceId}::uuid for update`) as unknown as Array<{ activityAt: Date | string }>; if (!source) throw notFound("Attention source not found"); await decisionRetentionService(database).syncItems(input.companyId, [{ - sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: source.updatedAt.toISOString(), + sourceKind: input.sourceKind, subject: { id: input.sourceId }, activityAt: new Date(source.activityAt).toISOString(), }]); } @@ -206,7 +221,7 @@ export function decisionRetentionService( if (!(await canReadDecisionSource(db, input.authActor, input.companyId, input.sourceKind, input.sourceId))) { throw notFound("Attention source not found"); } - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set({ keep: input.keep, @@ -247,7 +262,7 @@ export function decisionRetentionService( throw notFound("Attention source not found"); } const now = new Date(); - const updated = await writeRetention(input.sourceKind, async tx => { + const updated = await writeRetention(input.companyId, input.sourceKind, input.sourceId, async tx => { await materializeSourceState(tx, input); return tx.update(decisionRetention).set(input.archived ? { archivedAt: now, @@ -324,7 +339,7 @@ export function decisionRetentionService( } const now = new Date(); for (const entry of input.manifest) { - await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind); + await notifyRetentionWork(txDb, entry.sourceKind as AttentionSourceKind, { companyId: input.companyId, sourceId: entry.sourceId }); const archived = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "accepted_proposal", @@ -368,7 +383,7 @@ export function decisionRetentionService( await db.transaction(async (tx) => { const txDb = tx as unknown as Db; for (const item of candidates) { - await notifyRetentionWork(txDb, item.sourceKind); + await notifyRetentionWork(txDb, item.sourceKind, { companyId: input.companyId, sourceId: item.subject.id }); const updated = await txDb.update(decisionRetention).set({ archivedAt: now, archivedReason: "idle_ttl", diff --git a/server/src/services/decision-work-notifications.ts b/server/src/services/decision-work-notifications.ts index 081edc8fc39..9e2ca840960 100644 --- a/server/src/services/decision-work-notifications.ts +++ b/server/src/services/decision-work-notifications.ts @@ -1,18 +1,23 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export function notifyDecisionCreated(transaction: object) { +type DecisionWorkScope = { companyId: string; sourceId?: string }; + +export async function notifyDecisionCreated(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); return notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export async function notifyDecisionChange(transaction: object) { +export async function notifyDecisionChange(transaction: object, scope: DecisionWorkScope) { + await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: "decision" }); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionContinuation); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionExpiry); } -export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise): Promise { +export function writeDecisionChange(db: Db, write: (transaction: Db) => Promise, scope: DecisionWorkScope): Promise { return db.transaction(async transaction => { - await notifyDecisionChange(transaction); + await notifyDecisionChange(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.decisionEffectRecovery); return write(transaction as unknown as Db); }); diff --git a/server/src/services/decisions.ts b/server/src/services/decisions.ts index 282327d433f..8a84743fcef 100644 --- a/server/src/services/decisions.ts +++ b/server/src/services/decisions.ts @@ -239,7 +239,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { ...(input.additionalTargetSnapshots ?? {}), }; const id = randomUUID(); - await notifyDecisionCreated(dbOrTx); + await notifyDecisionCreated(dbOrTx, { companyId: input.companyId, sourceId: id }); await notifyDecisionTargetWork(dbOrTx, `decision:${id}`); const [created] = await dbOrTx.insert(decisions).values({ id, companyId: input.companyId, bundleId: input.bundleId ?? null, originAgentId: input.agentId, originIssueId: provenance.issueId, originRunId: input.runId, ruleKey: input.ruleKey ?? null, @@ -581,7 +581,8 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return attempt; } - async function settleInvalidEffectOutcome(id: string) { + async function settleInvalidEffectOutcome(decision: typeof decisions.$inferSelect) { + const id = decision.id; const publications: ActivityPublication[] = []; await writeDecisionChange(db, async tx => { const [current] = await tx.select().from(decisions).where(eq(decisions.id, id)).for("update"); @@ -601,14 +602,14 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { action: "decision.recovery_failed", entityType: "decision", entityId: id, details: { reason, executionStatus, chosenOptionId: current.chosenOptionId, decidedByUserId: current.decidedByUserId }, }, publications); - }); + }, { companyId: decision.companyId, sourceId: id }); for (const publication of publications) publishActivity(publication); return outcome(id); } async function executeDecisionEffects(decision: typeof decisions.$inferSelect, userActor: AuthorizationActor) { const option = decision.options.find((item) => item.id === decision.chosenOptionId); - if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision.id); + if (!option || !decision.decidedByUserId) return settleInvalidEffectOutcome(decision); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, decision.originRunId)).then((rows) => rows[0] ?? null); const metadata = decision.metadata as Record; if (metadata.kind === "attention_archive_proposal") { @@ -618,7 +619,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { targetSnapshots: decision.targetSnapshots as Record, }), decision.signedSpec)) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "invalid_signature", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } if (option.id === "archive") { @@ -629,7 +630,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const manifestHash = manifest ? hashAttentionArchiveManifest(manifest) : null; if (!manifest || manifestHash !== metadata.manifestHash) { await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: "manifest_mismatch", ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } const originActor: AuthorizationActor = { @@ -651,7 +652,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { } catch (error) { const message = error instanceof Error ? error.message : "archive_proposal_failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: "failed", updatedAt: new Date(), metadata: { ...metadata, archiveProposalError: message, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }) - .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + .where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } } @@ -659,7 +660,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { executionStatus: "succeeded", updatedAt: new Date(), metadata: { ...metadata, ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) }, - }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } for (let index = 0; index < option.effects.length; index += 1) await executeEffect(decision, option.effects[index]!, index, userActor, decision.decidedByUserId, run?.responsibleUserId ?? null); @@ -667,7 +668,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { const successful = rows.filter((row) => row.status === "executed").length; const status = rows.every((row) => row.status === "executed") ? "succeeded" : successful ? "partial" : "failed"; await writeDecisionChange(db, async tx => tx.update(decisions).set({ executionStatus: status, updatedAt: new Date(), metadata: { ...decision.metadata, - ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running")))); + ...(decision.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, decision.id), eq(decisions.executionStatus, "running"))), { companyId: decision.companyId, sourceId: decision.id }); return outcome(decision.id); } @@ -706,7 +707,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { decideIdempotencyKey: input.idempotencyKey ?? null, ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}), ...(input.dismissed ? { dismissed: true, dismissReason: input.dismissReason ?? null } : {}) } }) - .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning()); + .where(and(eq(decisions.id, current.id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: current.id }); if (!claimed) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); const run = await db.select({ responsibleUserId: heartbeatRuns.responsibleUserId }).from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed.originRunId)).then((rows) => rows[0] ?? null); await logActivity(db, { companyId: claimed.companyId, actorType: "system", actorId: "decision-executor", agentId: claimed.originAgentId, runId: claimed.originRunId, @@ -724,7 +725,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { return (await get(id))!; } const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "cancelled", updatedAt: new Date(), metadata: { ...current.metadata, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: actor.actorType, actorId: actor.actorId, runId: actor.runId, action: "decision.cancelled", entityType: "decision", entityId: id }); await continuations.deliver(updated.id); @@ -738,7 +739,7 @@ export function decisionService(db: Db, options: DecisionServiceOptions) { if (empty) return decide({ id, optionId: empty.id, decidedByUserId: userId, userActor, dismissed: true, dismissReason: reason }); const [updated] = await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "succeeded", chosenOptionId: "dismissed", decidedByUserId: userId, decidedAt: new Date(), updatedAt: new Date(), metadata: { ...current.metadata, dismissed: true, dismissReason: reason ?? null, - ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning()); + ...(current.continuationPolicy === "wake_origin_agent" ? { continuationPending: true } : {}) } }).where(and(eq(decisions.id, id), eq(decisions.status, "open"))).returning(), { companyId: current.companyId, sourceId: id }); if (!updated) throw conflict("decision_already_resolved", { code: "decision_already_resolved" }); await logActivity(db, { companyId: updated.companyId, actorType: "system", actorId: "decision-executor", agentId: updated.originAgentId, runId: updated.originRunId, responsibleUserIdOverride: userId, action: "decision.dismissed", entityType: "decision", entityId: updated.id, diff --git a/server/src/services/deferred-execution-notifications.ts b/server/src/services/deferred-execution-notifications.ts new file mode 100644 index 00000000000..b82cea725a5 --- /dev/null +++ b/server/src/services/deferred-execution-notifications.ts @@ -0,0 +1,34 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type DeferredExecutionScope = { companyId: string; requestId?: string }; +export type DeferredReceiptRecoveryOptions = { + companyId?: string; + requestId?: string; + onError?: (error: unknown) => void; +}; + +function hint(scope: DeferredExecutionScope) { + return JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }); +} + +export async function notifyDeferredExecutionWork(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, hint(scope)); +} + +export async function notifyNativeStatusWakeWork(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, hint(scope)); +} + +export async function notifyTaskRetrySettlement(tx: object, scope: DeferredExecutionScope) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.taskRetryReceiptSettlement, hint(scope)); +} + +export async function notifyDeferredExecutionEligibility(tx: object, scope?: { companyId: string }) { + if (scope) { + await notifyDeferredExecutionWork(tx, scope); + await notifyNativeStatusWakeWork(tx, scope); + } else { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.deferredExecutionReceipts, "*"); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusWakeIntents, "*"); + } +} diff --git a/server/src/services/deferred-execution-wait.ts b/server/src/services/deferred-execution-wait.ts new file mode 100644 index 00000000000..c4857d5b692 --- /dev/null +++ b/server/src/services/deferred-execution-wait.ts @@ -0,0 +1,24 @@ +import { and, eq, sql } from "drizzle-orm"; +import { agentWakeupRequests, type Db } from "@paperclipai/db"; +import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; + +export async function writeDeferredExecutionWait(db: Db, scope: { companyId: string; requestId: string }, + condition: Record, options: { replace?: boolean } = {}) { + return db.transaction(async tx => { + const [current] = await tx.select({ payload: agentWakeupRequests.payload }).from(agentWakeupRequests) + .where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))).for("update"); + if (!current) return; + const previous = current.payload?.executionWait; + const previousReason = previous && typeof previous === "object" && !Array.isArray(previous) + ? (previous as Record).reason : undefined; + if (!previous || previousReason !== condition.reason) await notifyDeferredExecutionWork(tx, scope); + await tx.update(agentWakeupRequests).set({ + payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', + ${options.replace ? sql`'{}'::jsonb` : sql`coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb)`} + || ${JSON.stringify(condition)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(agentWakeupRequests.companyId, scope.companyId), eq(agentWakeupRequests.id, scope.requestId), + eq(agentWakeupRequests.status, "deferred_issue_execution"))); + }); +} diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 5e7648c8703..4a8fd69ff77 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,10 +2,23 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + deferredExecutionReceipts: "deferred-execution-receipts", + taskRetryReceiptSettlement: "task-retry-receipt-settlement", + nativeStatusWakeIntents: "native-status-wake-intents", + orphanBlockers: "orphan-blockers", + activeRecoveryActions: "active-recovery-actions", + strandedAssignments: "stranded-assignments", + resolvedDependencies: "resolved-dependencies", + taskWatchdogs: "task-watchdogs", + staleIssueLocks: "stale-issue-locks", + scheduledRetryDispatch: "scheduled-retry-dispatch", + queuedRunDispatch: "queued-run-dispatch", + statusCardEvaluation: "status-card-evaluation", runEvidenceMigration: "run-evidence-migration", providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", skillVersionRetention: "skill-version-retention", + externalObjectRefresh: "external-object-refresh", databaseBackups: "database-backups", importSpoolCleanup: "import-spool-cleanup", externalConversationSettlement: "external-conversation-settlement", @@ -35,6 +48,7 @@ export const DELIVERY_QUEUES = { executionDisposition: "execution-disposition", environmentCleanup: "environment-cleanup", approvalQueueSeeding: "approval-queue-seeding", + attentionRetention: "attention-retention", approvalRetention: "approval-retention", joinRequestRetention: "join-request-retention", decisionTargets: "decision-targets", @@ -97,6 +111,7 @@ export const DELIVERY_QUEUES = { browser: "browser-use", email: "email-channels", museMaintenance: "muse-maintenance", + mergedPullRequestConfirmations: "merged-pull-request-confirmations", mcpEvents: "mcp-events", dotEvents: "dot-events", grokDispatch: "grok-dispatch", @@ -116,10 +131,12 @@ export const DELIVERY_QUEUES = { pluginLifecycle: "plugin-resource-lifecycle", toolAction: "tool-action-receipts", terminalWorkspaces: "terminal-workspaces", + routineExecutionStatus: "routine-execution-status", routineSchedules: "routine-schedules", issueMonitors: "issue-monitors", agentTimers: "agent-timers", caseHandoffs: "case-handoffs", + nativeHarnessBackupExpiration: "native-harness-backup-expiration", nativeCheckpointExpiration: "native-checkpoint-expiration", } as const; export type DeliveryQueue = typeof DELIVERY_QUEUES[keyof typeof DELIVERY_QUEUES]; diff --git a/server/src/services/documents.ts b/server/src/services/documents.ts index 493c495e007..7c81d075354 100644 --- a/server/src/services/documents.ts +++ b/server/src/services/documents.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { randomUUID } from "node:crypto"; import { and, asc, desc, eq, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -238,6 +240,7 @@ export function documentService(db: Db) { for (let attempt = 0; attempt < maxAttempts; attempt += 1) { try { return await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); const now = new Date(); const existing = await tx .select({ @@ -275,6 +278,7 @@ export function documentService(db: Db) { .where(eq(issueDocuments.issueId, issue.id)); const fallbackKey = nextAvailableDocumentKey(key, issueDocumentKeys.map((row) => row.key)); + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const [document] = await tx .insert(documents) .values({ @@ -376,6 +380,7 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [revision] = await tx .insert(documentRevisions) @@ -438,6 +443,7 @@ export function documentService(db: Db) { throw conflict("Document does not exist yet", { key }); } + await notifyIssueRecovery(tx, { companyId: issue.companyId }); const [document] = await tx .insert(documents) .values({ @@ -592,6 +598,10 @@ export function documentService(db: Db) { }); } await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) { + await notifyStatusCardIssueChange(tx, companyId); + await notifyIssueRecovery(tx, { companyId }); + } await insertRowsInChunks(tx, documents, documentRows); await insertRowsInChunks(tx, documentRevisions, revisionRows); await insertRowsInChunks(tx, issueDocuments, linkRows); @@ -644,6 +654,7 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); const nextRevisionNumber = existing.latestRevisionNumber + 1; const [restoredRevision] = await tx @@ -662,6 +673,7 @@ export function documentService(db: Db) { }) .returning(); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx .update(documents) .set({ @@ -722,6 +734,7 @@ export function documentService(db: Db) { }; } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); await tx .update(documents) @@ -769,6 +782,7 @@ export function documentService(db: Db) { }; } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); const now = new Date(); await tx .update(documents) @@ -817,7 +831,9 @@ export function documentService(db: Db) { }); } + await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueDocuments).where(eq(issueDocuments.documentId, existing.id)); + await notifyStatusCardIssueChange(tx, existing.companyId); await tx.delete(documents).where(eq(documents.id, existing.id)); return { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 277a5a0254c..33f161457f9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -15,6 +15,7 @@ import { hasStopOnlyCleanup, prepareSandboxStopAndRetain, readStopOnlyCleanup, s import { JsonRpcCallError, readEnvironmentAcquisitionDiagnostic, readEnvironmentCreationCleanupError } from "@paperclipai/plugin-sdk"; import { preserveEnvironmentSyncOutErrorDiagnostic } from "./environment-sync-out-error.js"; import { remoteTerminationReceipt } from "./remote-execution-termination.js"; +import { inspectCloudSandboxCleanupAbsence } from "./sandbox-absence-proof.js"; import { hasNativeWorkspaceExportResume, releaseCompletedNativeWorkspaceExportRetention } from "./native-runtime/native-workspace-export-resume.js"; import { createHash, randomUUID } from "node:crypto"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; @@ -4021,7 +4022,12 @@ export function environmentRuntimeService( const lease = await environmentsSvc.getLeaseById(leaseId); if (!lease || lease.status !== "failed" || lease.leasePolicy !== "ephemeral" || !lease.providerLeaseId) return undefined; const environment = lease.environmentId ? await environmentsSvc.getById(lease.environmentId) : null; - return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }); + // Legacy Cloud leases can lack a credential version that permits a + // provider-local observation. Cloud's manager verifies the historical + // organization and signs only exact absence; no replacement key or + // cleanup label is treated as proof. Explicit provider answers win. + return await getDriver(getLeaseDriverKey(lease, environment))?.inspectLeaseResource?.({ environment, lease }) + ?? await inspectCloudSandboxCleanupAbsence(lease); }, async retryPendingSandboxTeardown(input: { diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index e46df62ed7d..a9985370ae8 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1705,7 +1705,7 @@ export function environmentService(db: Db) { creationOperationId: sql`${environmentLeases.metadata}->>'creationOperationId'`, }).from(environmentLeases).where(condition).for("update"); if (!previous) return null; - await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined }); + await notifyEnvironmentCleanup(tx, { companyId: previous.companyId, runId: previous.heartbeatRunId ?? undefined, leaseId: id }); if (previous.heartbeatRunId && (previous.status !== status || (options?.cleanupStatus !== undefined && previous.cleanupStatus !== options.cleanupStatus) || (status !== "retained" && !previous.releasedAt))) { @@ -1830,7 +1830,7 @@ export function environmentService(db: Db) { const [lease] = await tx.select({ companyId: environmentLeases.companyId, heartbeatRunId: environmentLeases.heartbeatRunId, metadata: environmentLeases.metadata }) .from(environmentLeases).where(eq(environmentLeases.id, id)).for("update"); if (!lease) return null; - await notifyEnvironmentCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined }); + await notifyEnvironmentCleanup(tx, { companyId: lease.companyId, runId: lease.heartbeatRunId ?? undefined, leaseId: id }); return tx.update(environmentLeases) .set({ metadata: preserveReusableLeaseHandoff(lease.metadata, withoutCleanupVerification(metadata)), diff --git a/server/src/services/execution-control-notifications.ts b/server/src/services/execution-control-notifications.ts index 22c8913bbcd..431ff1210e2 100644 --- a/server/src/services/execution-control-notifications.ts +++ b/server/src/services/execution-control-notifications.ts @@ -1,7 +1,21 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import type { Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork, type DeliveryQueue } from "./delivery-work-notifications.js"; -export async function notifyExecutionRunWork(transaction: object) { +export type ExecutionWorkScope = { companyId: string; agentId?: string; runId?: string; issueId?: string }; + +export async function notifyExecutionRunWork(transaction: object, scope?: ExecutionWorkScope) { + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); + if (scope) { + await notifyAttentionRetentionWork(transaction, scope); + await notifyStatusCardIssueChange(transaction, scope.companyId); + await notifyRunDispatch(transaction, scope); + await notifyDeferredExecutionEligibility(transaction, scope); + } for (const queue of [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement, DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors]) { await notifyDeliveryWork(transaction, queue); @@ -12,8 +26,14 @@ export function writeExecutionControl( db: Db | Parameters[0]>[0], queue: DeliveryQueue | readonly DeliveryQueue[], write: (tx: Db) => PromiseLike, + scope: ExecutionWorkScope | undefined, ): Promise { return db.transaction(async transaction => { + if (scope) await notifyRoutineExecutionStatus(transaction, scope.issueId, scope.companyId); + if (scope) await notifyAttentionRetentionWork(transaction, scope); + if (scope) await notifyStatusCardIssueChange(transaction, scope.companyId); + await notifyRunDispatch(transaction, scope); + if (scope) await notifyDeferredExecutionEligibility(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.issueMonitors); for (const topic of typeof queue === "string" ? [queue] : queue) await notifyDeliveryWork(transaction, topic); return write(transaction as unknown as Db); diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 2f2a435982d..621fd210cfc 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -196,7 +196,7 @@ export async function reconcileAbandonedExecutionControl( await notifyNativeEnvironmentSettlement(tx, { companyId: run.companyId, runId: run.id }); await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, run); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const [updatedRun] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/execution-recovery-resolution.ts b/server/src/services/execution-recovery-resolution.ts index f3f5a3e4310..a6f80f26ba0 100644 --- a/server/src/services/execution-recovery-resolution.ts +++ b/server/src/services/execution-recovery-resolution.ts @@ -1,3 +1,4 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { notifyChatResponseAvailability } from "./chat-response-availability.js"; @@ -546,7 +547,7 @@ export async function settleUnrecoverableExecutions( companyId: run.companyId, issueId: task.id, runId: run.id, }).limit(1).for("update"); if (!hold) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); const note = "Exact session recovery stopped. The recorded workspace recovery holds this execution without replaying its actions."; await tx.update(issueRecoveryActions).set({ status: "cancelled", outcome: "cancelled", resolvedAt: now, updatedAt: now, wakePolicy: null, monitorPolicy: null, @@ -596,6 +597,7 @@ export async function settleUnrecoverableExecutions( if (task.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: task.companyId, issueId: task.id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, task.id, task.companyId); const [projected] = await tx .update(issues) .set({ @@ -654,7 +656,7 @@ export async function settleUnrecoverableExecutions( replay: "not_authorized", }, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(heartbeatRuns) .set({ executionStatusDeliveryId: randomUUID() }) diff --git a/server/src/services/execution-workspaces.ts b/server/src/services/execution-workspaces.ts index 4b716916638..15ff32f292e 100644 --- a/server/src/services/execution-workspaces.ts +++ b/server/src/services/execution-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy } from "./project-workspace-provider.js"; import { inspectPluginWorkspace, removePluginWorkspace, restorePluginWorkspace, type WorkspaceWorkers } from "./plugin-workspaces.js"; import { acquireExecutionWorkspaceLifecycleLock, withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifecycle-lock.js"; @@ -1590,6 +1591,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic await validateSelection(input, tx); const intent: TaskWorkspaceIntent = { version: 1, request: { key: input.requestKey, expectedBindingRevision: input.expectedBindingRevision }, selection: input.selection, source: input.source ?? "explicit" }; const nextRevision = task.workspaceBindingRevision + 1; + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ workspaceBindingRevision: nextRevision, workspacePendingSelection: { version: 1, requestKey: input.requestKey, expectedBindingRevision: nextRevision, intent }, updatedAt: new Date() }).where(eq(issues.id, task.id)); @@ -1633,6 +1635,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic } } } + await notifyStatusCardIssueChange(tx, task.companyId); await tx.update(issues).set({ ...patch, assigneeAdapterOverrides, workspaceSelection: pending.intent, workspacePendingSelection: null, workspaceBindingRevision: task.workspaceBindingRevision + 1, updatedAt: new Date() }).where(eq(issues.id, task.id)); return true; @@ -1651,6 +1654,7 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic const pendingAfterFirstBinding = !task.executionWorkspaceId && pending?.expectedBindingRevision === task.workspaceBindingRevision ? { ...pending, expectedBindingRevision: nextRevision } : pending; + await notifyStatusCardIssueChange(tx, companyId); const [bound] = await tx.update(issues).set({ ...patch, executionWorkspaceId: workspaceId, workspaceBindingRevision: nextRevision, workspacePendingSelection: pendingAfterFirstBinding, updatedAt: new Date() }) .where(eq(issues.id, issueId)).returning(); @@ -2526,13 +2530,16 @@ export function executionWorkspaceService(db: Db, opts: ExecutionWorkspaceServic forceWorktreeRemoval: false, }); if (cleanup.cleaned && workspace.mode === "shared_workspace") { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, workspace.companyId); + await tx .update(issues) .set({ executionWorkspaceId: null, updatedAt: now() }) .where(and( eq(issues.companyId, workspace.companyId), eq(issues.executionWorkspaceId, workspace.id), )); + }); } const cleanupReason = [ISSUE_TERMINAL_WORKSPACE_CLEANUP_REASON, ...cleanup.warnings].join(" | "); if (!cleanup.cleaned || cleanup.warnings.length > 0) { diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 074ffc3bdf5..084a25e457f 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -475,7 +475,7 @@ export async function admitExplicitNativeContinuation(input: { }).where(and(eq(agentWakeupRequests.companyId, companyId), eq(agentWakeupRequests.id, input.queuedCommentInterruptId), eq(agentWakeupRequests.agentId, agentId), eq(agentWakeupRequests.status, "deferred_issue_execution"))); if (cancelledStartupIds.size) await db.transaction(async tx => { - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId, agentId }); for (const runId of cancelledStartupIds) { await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: "native_startup_cancelled", nextAttemptAt: null, diff --git a/server/src/services/external-conversation-settlement-notifications.ts b/server/src/services/external-conversation-settlement-notifications.ts index 46f4493ff2a..d5e4cdd506c 100644 --- a/server/src/services/external-conversation-settlement-notifications.ts +++ b/server/src/services/external-conversation-settlement-notifications.ts @@ -1,7 +1,9 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { ExternalConversationSettlementScope } from "./slack-conversation-lifecycle.js"; -export function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) { +export async function notifyExternalConversationSettlement(tx: object, scope?: ExternalConversationSettlementScope) { + await notifyIssueRecovery(tx, scope); return notifyDeliveryWork(tx, DELIVERY_QUEUES.externalConversationSettlement, scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"); } diff --git a/server/src/services/external-object-refresh-notifications.ts b/server/src/services/external-object-refresh-notifications.ts new file mode 100644 index 00000000000..271d172e463 --- /dev/null +++ b/server/src/services/external-object-refresh-notifications.ts @@ -0,0 +1,9 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type ExternalObjectRefreshIdentity = { providerKey: string; objectType: string; externalId: string; ignoreExternalIdCase?: boolean }; + +export async function notifyExternalObjectRefresh(tx: object, companyId?: string, object?: string | ExternalObjectRefreshIdentity) { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.externalObjectRefresh, + companyId ? JSON.stringify({ companyId, ...(typeof object === "string" ? { objectId: object } : object ? { identity: object } : {}) }) : "*"); +} + diff --git a/server/src/services/external-objects.ts b/server/src/services/external-objects.ts index 56a8fb91834..81e10a208f2 100644 --- a/server/src/services/external-objects.ts +++ b/server/src/services/external-objects.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; -import { and, asc, eq, inArray, isNull, lte, not, or, sql } from "drizzle-orm"; +import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, lte, not, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; -import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins } from "@paperclipai/db"; +import { companies, documents, externalObjectMentions, externalObjects, issueComments, issueDocuments, issues, plugins, pluginCompanySettings } from "@paperclipai/db"; import { formatExternalObjectMentionSourceLabel, type ExternalObjectCanonicalUrl, @@ -18,6 +18,10 @@ import { notFound } from "../errors.js"; import { logger } from "../middleware/logger.js"; import { logActivity, type LogActivityInput } from "./activity-log.js"; import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; +import { notifyExternalObjectRefresh, type ExternalObjectRefreshIdentity } from "./external-object-refresh-notifications.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; import { publishLiveEvent } from "./live-events.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; @@ -84,6 +88,7 @@ export interface ExternalObjectResolver { resolve(input: { companyId: string; object: ExternalObjectRecord; + signal?: AbortSignal; }): Promise; } @@ -95,6 +100,18 @@ const DEFAULT_RETRY_AFTER_SECONDS = 300; const DEFAULT_REFRESH_LEASE_SECONDS = 300; const REFRESH_LEASE_RENEW_INTERVAL_MS = 60_000; +type RefreshScope = { companyId: string; objectId?: string; identity?: ExternalObjectRefreshIdentity }; +type RefreshDeadline = { id: string; companyId: string; dueAt: number; providerKey: string; objectType: string; externalId: string }; + + +function refreshableObjectWhere() { + return or(eq(externalObjects.isTerminal, false), and( + eq(externalObjects.providerKey, "github"), + eq(externalObjects.statusKey, "not_found"), + not(sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`), + ))!; +} + function sourceWhere(input: ExternalObjectSourceContext) { const conditions = [ eq(externalObjectMentions.companyId, input.companyId), @@ -245,7 +262,7 @@ function objectSnapshot(object: ExternalObjectRecord): PluginExternalObjectRecor }; } -async function readyObjectReferencePlugins(db: Db) { +async function readyObjectReferencePlugins(db: Db, companyId?: string) { return db .select({ id: plugins.id, @@ -253,7 +270,12 @@ async function readyObjectReferencePlugins(db: Db) { manifestJson: plugins.manifestJson, }) .from(plugins) - .where(eq(plugins.status, "ready")) + .where(and(eq(plugins.status, "ready"), companyId ? not(exists(db.select({ id: pluginCompanySettings.id }) + .from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, companyId), + eq(pluginCompanySettings.enabled, false), + )))) : undefined)) .orderBy(asc(plugins.installOrder)) .then((rows) => rows.filter((row) => (row.manifestJson.objectReferences?.length ?? 0) > 0), @@ -267,7 +289,7 @@ function createPluginProviderDetector( return { key: "plugin-object-reference-providers", async detect(input) { - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, input.companyId); const detections: ExternalObjectDetection[] = []; for (const provider of providers) { @@ -324,31 +346,35 @@ async function resolveViaPluginProvider( db: Db, pluginWorkerManager: PluginWorkerManager | undefined, object: ExternalObjectRecord, -): Promise { +): Promise<{ pluginId: string; result: PluginExternalObjectResolveResult } | null> { if (!pluginWorkerManager) return null; - const providers = await readyObjectReferencePlugins(db); + const providers = await readyObjectReferencePlugins(db, object.companyId); for (const provider of providers) { const manifest = provider.manifestJson; if (!manifest.capabilities.includes("external.objects.read")) continue; if (!manifestProvidesObject(manifest, object)) continue; try { - return await pluginWorkerManager.call(provider.id, "resolveExternalObject", { + const result = await pluginWorkerManager.call(provider.id, "resolveExternalObject", { companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, object: objectSnapshot(object), }); + return { pluginId: provider.id, result }; } catch (err) { logger.warn( { err, pluginId: provider.id, pluginKey: provider.pluginKey, objectId: object.id }, "plugin external object resolver failed", ); return { - ok: false, - liveness: "unreachable", - errorCode: "plugin_resolver_failed", - errorMessage: err instanceof Error ? err.message : String(err), + pluginId: provider.id, + result: { + ok: false, + liveness: "unreachable", + errorCode: "plugin_resolver_failed", + errorMessage: err instanceof Error ? err.message : String(err), + }, }; } } @@ -374,10 +400,43 @@ export function externalObjectService( ...(opts.detectors ?? []), ...(githubProvider ? [githubProvider.detector] : []), ]); - const resolverRegistry = createExternalObjectResolverRegistry([ + const resolvers = [ ...(opts.resolvers ?? []), ...(githubProvider?.resolvers ?? []), - ]); + ]; + const resolverRegistry = createExternalObjectResolverRegistry(resolvers); + + function pluginRefreshWhere(database: Db, selectedPluginId?: string) { + return exists(database.select({ id: plugins.id }).from(plugins).where(and( + selectedPluginId ? eq(plugins.id, selectedPluginId) : undefined, + eq(plugins.status, "ready"), + sql`${plugins.manifestJson}->'capabilities' ? 'external.objects.read'`, + sql`exists (select 1 from jsonb_array_elements(coalesce(${plugins.manifestJson}->'objectReferences', '[]'::jsonb)) as reference + where reference->>'providerKey' = ${externalObjects.providerKey} + and reference->'objectTypes' ? ${externalObjects.objectType})`, + not(exists(database.select({ id: pluginCompanySettings.id }).from(pluginCompanySettings).where(and( + eq(pluginCompanySettings.pluginId, plugins.id), + eq(pluginCompanySettings.companyId, externalObjects.companyId), + eq(pluginCompanySettings.enabled, false), + )))), + ))); + } + + function automaticRefreshWhere(database: Db = db) { + const supported = resolvers.map(resolver => and( + eq(externalObjects.providerKey, resolver.providerKey), + resolver.objectType ? eq(externalObjects.objectType, resolver.objectType) : undefined, + )!); + if (opts.pluginWorkerManager) supported.push(pluginRefreshWhere(database)); + return and( + refreshableObjectWhere(), isNotNull(externalObjects.nextRefreshAt), + supported.length ? or(...supported) : sql`false`, + exists(database.select({ id: externalObjectMentions.id }).from(externalObjectMentions).where(and( + eq(externalObjectMentions.companyId, externalObjects.companyId), + eq(externalObjectMentions.objectId, externalObjects.id), + ))), + )!; + } async function isEnabled() { if (typeof opts.enabled === "function") return await opts.enabled(); @@ -422,6 +481,7 @@ export function externalObjectService( nextRefreshAt: now, updatedAt: now, }; + await notifyExternalObjectRefresh(dbOrTx, companyId, { providerKey: detection.providerKey, objectType: detection.objectType, externalId: detection.externalId }); const inserted = await dbOrTx .insert(externalObjects) .values(values) @@ -451,6 +511,9 @@ export function externalObjectService( dbOrTx: any = db, ) { const urls = extractExternalObjectCanonicalUrls(input.text ?? ""); + const previous: Array<{ objectId: string | null }> = await dbOrTx.selectDistinct({ objectId: externalObjectMentions.objectId }) + .from(externalObjectMentions).where(sourceWhere(input)); + for (const row of previous) if (row.objectId) await notifyExternalObjectRefresh(dbOrTx, input.companyId, row.objectId); await dbOrTx.delete(externalObjectMentions).where(sourceWhere(input)); if (urls.length === 0) return; @@ -469,6 +532,7 @@ export function externalObjectService( if (seen.has(sourceKey)) continue; seen.add(sourceKey); const object = await upsertObjectFromDetection(input.companyId, detection, dbOrTx); + await notifyExternalObjectRefresh(dbOrTx, input.companyId, object.id); values.push({ companyId: input.companyId, sourceIssueId: input.sourceIssueId, @@ -520,6 +584,14 @@ export function externalObjectService( return dbOrTx === db ? db.transaction(runSync) : runSync(dbOrTx); } + async function deleteMissingSourceMentions(kind: "comment" | "document", recordId: string, tx: any) { + const source = and(eq(externalObjectMentions.sourceKind, kind), eq(externalObjectMentions.sourceRecordId, recordId)); + const owners: Array<{ companyId: string; objectId: string | null }> = await tx.selectDistinct({ companyId: externalObjectMentions.companyId, objectId: externalObjectMentions.objectId }) + .from(externalObjectMentions).where(source); + for (const owner of owners) if (owner.objectId) await notifyExternalObjectRefresh(tx, owner.companyId, owner.objectId); + await tx.delete(externalObjectMentions).where(source); + } + async function syncComment(commentId: string, dbOrTx: any = db) { const runSync = async (tx: any) => { if (!(await isEnabled())) return; @@ -534,9 +606,7 @@ export function externalObjectService( .where(eq(issueComments.id, commentId)) .then((rows: Array<{ id: string; companyId: string; issueId: string; body: string }>) => rows[0] ?? null); if (!comment) { - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "comment"), eq(externalObjectMentions.sourceRecordId, commentId))); + await deleteMissingSourceMentions("comment", commentId, tx); return; } await replaceSourceMentions({ @@ -568,9 +638,7 @@ export function externalObjectService( .where(eq(documents.id, documentId)) .then((rows: Array<{ documentId: string; companyId: string; issueId: string; key: string; body: string }>) => rows[0] ?? null); if (!document) { - await tx - .delete(externalObjectMentions) - .where(and(eq(externalObjectMentions.sourceKind, "document"), eq(externalObjectMentions.sourceRecordId, documentId))); + await deleteMissingSourceMentions("document", documentId, tx); return; } await replaceSourceMentions({ @@ -780,6 +848,8 @@ export function externalObjectService( companyId: string; actor?: Pick; force?: boolean; + scheduled?: boolean; + signal?: AbortSignal; now?: Date; }; @@ -801,39 +871,50 @@ export function externalObjectService( return { object: toObjectPayload(latest, now), refreshed: false, reason: "refresh_superseded" as const }; } + async function persistRefresh(object: ExternalObjectRecord, token: string, + patch: Partial, scheduled = false, selectedPluginId?: string) { + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx, object.companyId, object.id); + const rows = await tx.update(externalObjects).set(patch) + .where(and(refreshOwnerWhere(object, token), + scheduled ? automaticRefreshWhere(tx as unknown as Db) : undefined, + selectedPluginId ? pluginRefreshWhere(tx as unknown as Db, selectedPluginId) : undefined)).returning(); + if ((scheduled || selectedPluginId) && rows.length === 0) await tx.update(externalObjects) + .set({ refreshStartedAt: null, refreshToken: null }).where(refreshOwnerWhere(object, token)); + return rows; + }); + } + async function resolveObjectRefresh( object: ExternalObjectRecord, input: RefreshObjectInput, now: Date, refreshToken: string, ) { - const pluginResult = await resolveViaPluginProvider(db, opts.pluginWorkerManager, object); - const resolver = pluginResult ? null : resolverRegistry.find(object); - if (!pluginResult && !resolver) { - const [updated] = await db - .update(externalObjects) - .set({ + input.signal?.throwIfAborted(); + const pluginResolution = await observeRefreshCancellation(resolveViaPluginProvider(db, opts.pluginWorkerManager, object), input.signal); + const resolver = pluginResolution ? null : resolverRegistry.find(object); + if (!pluginResolution && !resolver) { + const [updated] = await persistRefresh(object, refreshToken, { liveness: object.isTerminal ? object.liveness : visibleLiveness(object, now) === "fresh" ? "stale" : object.liveness, nextRefreshAt: addSeconds(now, DEFAULT_RETRY_AFTER_SECONDS), refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled); if (!updated) return refreshSupersededResult(object, now); return { object: toObjectPayload(updated ?? object, now), refreshed: false, reason: "no_resolver" as const }; } - const result = pluginResult ?? await resolver!.resolve({ companyId: object.companyId, object }); + const result = pluginResolution?.result ?? await observeRefreshCancellation( + resolver!.resolve({ companyId: object.companyId, object, signal: input.signal }), input.signal); + input.signal?.throwIfAborted(); if (!result.ok) { // A failed refresh means the object's status is currently unknown, not // terminal. Clear a previous terminal verdict so the object keeps being // scheduled for automatic refresh and self-heals (e.g. a GitHub "not // found" that later resolves once repo access improves). - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { liveness: result.liveness, isTerminal: false, lastErrorAt: now, @@ -843,9 +924,7 @@ export function externalObjectService( refreshStartedAt: null, refreshToken: null, updatedAt: now, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); publishLiveEvent({ companyId: object.companyId, @@ -879,14 +958,10 @@ export function externalObjectService( refreshToken: null, updatedAt: now, }; - const [updated] = await db - .update(externalObjects) - .set({ + const [updated] = await persistRefresh(object, refreshToken, { ...patch, lastChangedAt: objectChanged(object, { ...object, ...patch }) ? now : object.lastChangedAt, - }) - .where(refreshOwnerWhere(object, refreshToken)) - .returning(); + }, input.scheduled, pluginResolution?.pluginId); if (!updated) return refreshSupersededResult(object, now); const next = updated ?? object; if (objectChanged(object, next) && input.actor) { @@ -947,6 +1022,8 @@ export function externalObjectService( eq(externalObjects.id, object.id), eq(externalObjects.companyId, object.companyId), dueOrForced, + input.scheduled ? automaticRefreshWhere() : undefined, + input.scheduled ? sql`exists (select 1 from ${companies} where ${companies.id} = ${externalObjects.companyId} and ${companies.status} = 'active')` : undefined, )) .returning(); return claimed ?? null; @@ -969,6 +1046,16 @@ export function externalObjectService( }; } + function observeRefreshCancellation(work: Promise, signal?: AbortSignal): Promise { + if (!signal) return work; + return new Promise((resolve, reject) => { + const abort = () => reject(signal.reason); + work.then(value => { signal.removeEventListener("abort", abort); resolve(value); }, + error => { signal.removeEventListener("abort", abort); reject(error); }); + if (signal.aborted) abort(); else signal.addEventListener("abort", abort, { once: true }); + }); + } + const objectRefreshesInFlight = new Map>>>(); async function refreshObject( @@ -988,7 +1075,7 @@ export function externalObjectService( const refreshKey = `${object.companyId}:${object.id}`; const existingRefresh = objectRefreshesInFlight.get(refreshKey); - if (existingRefresh) return existingRefresh; + if (existingRefresh) return observeRefreshCancellation(existingRefresh, input.signal); const claimed = await claimObjectRefresh(object, input, now); if (!claimed) { @@ -1005,7 +1092,10 @@ export function externalObjectService( } const stopRenewingRefreshLease = startRefreshLeaseRenewal(object, claimed.refreshToken); - const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).finally(() => { + const refresh = resolveObjectRefresh(object, input, now, claimed.refreshToken).catch(async error => { + if (input.signal?.aborted) await persistRefresh(object, claimed.refreshToken!, { refreshStartedAt: null, refreshToken: null }); + throw error; + }).finally(() => { stopRenewingRefreshLease(); objectRefreshesInFlight.delete(refreshKey); }); @@ -1030,73 +1120,66 @@ export function externalObjectService( return results; } - async function refreshDueObjectsUnchecked(companyId: string, limit = 50, now = new Date()) { - const staleRefreshStartedBefore = new Date(now.getTime() - DEFAULT_REFRESH_LEASE_SECONDS * 1000); - // Terminal objects are usually skipped forever, but a GitHub "not found" - // verdict is only trustworthy while the repository is actually visible to - // the configured credentials. GitHub hides inaccessible (often private) - // repos behind a 404, so a not-found object can become resolvable later - // when credentials or repo visibility improve. Re-check objects that were - // frozen as "not found" before the repository probe existed so they - // self-heal instead of freezing. Once the provider has probed the repo and - // confirmed it is visible (data.probeVerified = "true"), the not-found - // verdict is definitive, so stop re-polling that object to avoid burning - // the company's GitHub API budget on deleted references forever. - const probeVerifiedNotFound = sql`coalesce(${externalObjects.data}->>'probeVerified', 'false') = 'true'`; - const refreshable = or( - eq(externalObjects.isTerminal, false), - and( - eq(externalObjects.isTerminal, true), - eq(externalObjects.providerKey, "github"), - eq(externalObjects.statusKey, "not_found"), - not(probeVerifiedNotFound), - ), - )!; - const due = await db - .select({ id: externalObjects.id }) - .from(externalObjects) - .where( - and( - eq(externalObjects.companyId, companyId), - refreshable, - lte(externalObjects.nextRefreshAt, now), - or( - isNull(externalObjects.refreshStartedAt), - lte(externalObjects.refreshStartedAt, staleRefreshStartedBefore), - ), - ), - ) - .limit(limit); - const results = []; - for (const row of due) { - results.push(await refreshObject(row.id, { - companyId, - actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, - now, - })); + function registerRefreshWork( + coordinator: ReturnType, + canRun: () => boolean, + ) { + const cursors = new Map(); + const scopeKey = (scope?: RefreshScope) => JSON.stringify(scope ?? null); + async function load(scope?: RefreshScope) { + if (!(await isEnabled())) { cursors.clear(); return []; } + const key = scopeKey(scope); + const cursor = cursors.get(key); + const rows = await db.select({ id: externalObjects.id, companyId: externalObjects.companyId, + nextRefreshAt: externalObjects.nextRefreshAt, refreshStartedAt: externalObjects.refreshStartedAt, + providerKey: externalObjects.providerKey, objectType: externalObjects.objectType, externalId: externalObjects.externalId }).from(externalObjects) + .innerJoin(companies, eq(companies.id, externalObjects.companyId)) + .where(and( + eq(companies.status, "active"), automaticRefreshWhere(), + scope ? eq(externalObjects.companyId, scope.companyId) : undefined, + scope?.objectId ? eq(externalObjects.id, scope.objectId) : undefined, + scope?.identity ? and(eq(externalObjects.providerKey, scope.identity.providerKey), + eq(externalObjects.objectType, scope.identity.objectType), scope.identity.ignoreExternalIdCase + ? sql`lower(${externalObjects.externalId}) = lower(${scope.identity.externalId})` + : eq(externalObjects.externalId, scope.identity.externalId)) : undefined, + cursor ? gt(externalObjects.id, cursor) : undefined, + )).orderBy(asc(externalObjects.id)).limit(250); + const entries = rows.map(deadline); + if (rows.length === 250) cursors.set(key, rows[rows.length - 1]!.id); + else cursors.delete(key); + return { rows: entries, complete: rows.length < 250 }; } - return results; - } - - async function refreshDueObjects(companyId: string, limit = 50, now = new Date()) { - if (!(await isEnabled())) return []; - return refreshDueObjectsUnchecked(companyId, limit, now); - } - - async function refreshDueObjectsForActiveCompanies(limitPerCompany = 50, now = new Date()) { - if (!(await isEnabled())) return { companies: 0, checked: 0, refreshed: 0 }; - const activeCompanies = await db - .select({ id: companies.id }) - .from(companies) - .where(eq(companies.status, "active")); - let checked = 0; - let refreshed = 0; - for (const company of activeCompanies) { - const results = await refreshDueObjectsUnchecked(company.id, limitPerCompany, now); - checked += results.length; - refreshed += results.filter((result) => result.refreshed).length; + function deadline(object: Pick): RefreshDeadline { + return { id: object.id, companyId: object.companyId, providerKey: object.providerKey, objectType: object.objectType, externalId: object.externalId, + dueAt: Math.max(object.nextRefreshAt!.getTime(), object.refreshStartedAt + ? object.refreshStartedAt.getTime() + DEFAULT_REFRESH_LEASE_SECONDS * 1000 : 0) }; } - return { companies: activeCompanies.length, checked, refreshed }; + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.externalObjectRefresh, + batchSize: 50, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RefreshScope, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId + && (!scope.objectId || entry.id === scope.objectId) + && (!scope.identity || (entry.providerKey === scope.identity.providerKey && entry.objectType === scope.identity.objectType + && (scope.identity.ignoreExternalIdCase ? entry.externalId.toLowerCase() === scope.identity.externalId.toLowerCase() + : entry.externalId === scope.identity.externalId)))), + load, + check: async (entry, signal) => { + if (signal?.aborted || !canRun()) return entry; + const current = await load({ companyId: entry.companyId, objectId: entry.id }); + const scheduled = Array.isArray(current) ? current[0] : current.rows[0]; + if (!scheduled || scheduled.dueAt > Date.now()) return scheduled ?? null; + await refreshObject(entry.id, { + companyId: entry.companyId, + scheduled: true, + signal, + actor: { actorType: "system", actorId: "external-object-resolver", agentId: null, runId: null }, + }); + const updated = await load({ companyId: entry.companyId, objectId: entry.id }); + return (Array.isArray(updated) ? updated[0] : updated.rows[0]) ?? null; + }, + }, canRun); } return { @@ -1112,7 +1195,6 @@ export function externalObjectService( getProjectSummary, refreshObject, refreshIssueObjects, - refreshDueObjects, - refreshDueObjectsForActiveCompanies, + registerRefreshWork, }; } diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index b0cc2f2fdd4..786b3c01207 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -1,4 +1,5 @@ import { notifyGitHubCredentialChange } from "./github-connection-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; import { dispatchGitHubBotCloudEvent } from "./chat-github-cloud-ingress.js"; import { chatEndpoints } from "@paperclipai/db"; @@ -251,6 +252,8 @@ export function githubConnectionEventService( const snapshot = githubSnapshotUpdate(event.payload); if (!snapshot) return; const appliedAt = now(); + await notifyExternalObjectRefresh(database, companyId, { providerKey: "github", objectType: "pull_request", + externalId: snapshot.externalId, ignoreExternalIdCase: true }); await database.update(externalObjects).set({ statusKey: snapshot.statusKey, statusLabel: snapshot.statusLabel, @@ -263,6 +266,8 @@ export function githubConnectionEventService( lastResolvedAt: appliedAt, lastChangedAt: appliedAt, nextRefreshAt: appliedAt, + refreshStartedAt: null, + refreshToken: null, updatedAt: appliedAt, }).where(and( eq(externalObjects.companyId, companyId), diff --git a/server/src/services/github-external-object-provider.ts b/server/src/services/github-external-object-provider.ts index 6dedac97efb..83b47354035 100644 --- a/server/src/services/github-external-object-provider.ts +++ b/server/src/services/github-external-object-provider.ts @@ -15,6 +15,7 @@ type FetchLike = (url: string, init?: RequestInit) => Promise; export interface GitHubExternalObjectProviderOptions { fetch?: FetchLike; + requestTimeoutMs?: number; tokenProvider?: (companyId: string) => Promise | string | null; /** * Resolve the credential and its source. Prefer this over `tokenProvider` when the @@ -243,12 +244,12 @@ type RepoProbeOutcome = { async function probeRepoVisibility( identity: GitHubObjectIdentity, - opts: { fetch: FetchLike; headers?: Record | undefined }, + opts: { fetch: FetchLike; headers?: Record | undefined; signal?: AbortSignal }, ): Promise { try { const probe = await opts.fetch( `${gitHubApiBase(identity.host)}/repos/${encodeURIComponent(identity.owner)}/${encodeURIComponent(identity.repo)}`, - { headers: opts.headers }, + { headers: opts.headers, signal: opts.signal }, ); if (probe.ok) { const body = await safeJson(probe); @@ -427,12 +428,13 @@ export function createGitHubExternalObjectProvider( companyId: string, identity: GitHubObjectIdentity, headers: Record, + signal: AbortSignal, ): Promise { const credentialFingerprint = createHash("sha256").update(headers.authorization ?? "anonymous").digest("hex"); const key = `${companyId}\u0000${credentialFingerprint}\u0000${identity.owner.toLowerCase()}\u0000${identity.repo.toLowerCase()}`; const cached = probeCache.get(key); if (cached && cached.expiresAt > Date.now()) return cached.outcome; - const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const outcome = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (outcome.visibility === "visible" || outcome.visibility === "hidden") { probeCache.set(key, { outcome, expiresAt: Date.now() + GITHUB_OBJECT_TTL_SECONDS * 1000 }); } @@ -464,7 +466,9 @@ export function createGitHubExternalObjectProvider( return { providerKey: "github", objectType, - async resolve({ companyId, object }) { + async resolve({ companyId, object, signal: ownerSignal }) { + const deadline = AbortSignal.timeout(opts.requestTimeoutMs ?? 10_000); + const signal = ownerSignal ? AbortSignal.any([ownerSignal, deadline]) : deadline; const identity = parseGitHubObject(object); if (!identity || identity.objectType !== objectType) { return { @@ -509,7 +513,7 @@ export function createGitHubExternalObjectProvider( // "public" verdict must not authorize a later read of its private contents. // The cache is still used below to classify a 404, where a stale verdict can // only make the object look deleted — it can never expose private data. - const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers }); + const probe = await probeRepoVisibility(identity, { fetch: fetchImpl, headers, signal }); if (probe.visibility === "hidden") { return { ok: false, @@ -554,7 +558,7 @@ export function createGitHubExternalObjectProvider( let response: Response; try { - response = await fetchImpl(url, { headers }); + response = await fetchImpl(url, { headers, signal }); } catch { return { ok: false, @@ -573,7 +577,7 @@ export function createGitHubExternalObjectProvider( // Probe the repository metadata to classify: if the repo itself is visible // the object truly is gone; if it is not, surface credentials-needed // instead of permanently freezing a reference as "Not found". - const probe = await cachedProbeRepoVisibility(companyId, identity, headers); + const probe = await cachedProbeRepoVisibility(companyId, identity, headers, signal); if (probe.visibility === "visible") { return { ok: true, snapshot: notFoundSnapshot(identity, etag) }; } diff --git a/server/src/services/github-pull-request-merge.ts b/server/src/services/github-pull-request-merge.ts index bb92d37d7c2..1cdff01dc1f 100644 --- a/server/src/services/github-pull-request-merge.ts +++ b/server/src/services/github-pull-request-merge.ts @@ -1,5 +1,5 @@ import type { Db } from "@paperclipai/db"; -import { createGitHubExternalObjectProvider } from "./github-external-object-provider.js"; +import { createGitHubExternalObjectProvider, type GitHubExternalObjectProviderOptions } from "./github-external-object-provider.js"; export type GitHubPullRequestReference = { host: "github.com"; @@ -25,11 +25,13 @@ export type PullRequestMergeDetails = { export type PullRequestMergeStateResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export type PullRequestMergeDetailsResolver = ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; export const PULL_REQUEST_CACHE_MAX_ENTRIES = 1_000; @@ -86,14 +88,15 @@ function readRecord(value: unknown): Record | null { : null; } -export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeDetailsResolver { - const resolver = createGitHubExternalObjectProvider(db).resolvers +export function createPullRequestMergeDetailsResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeDetailsResolver { + const resolver = createGitHubExternalObjectProvider(db, options).resolvers .find((candidate) => candidate.objectType === "pull_request") ?? null; - return async (companyId, reference) => { + return async (companyId, reference, signal) => { if (!resolver) return { state: "unknown", headRef: null, headSha: null }; const result = await resolver.resolve({ companyId, + signal, object: { externalId: `${reference.owner}/${reference.repo}#pull/${reference.number}`, sanitizedCanonicalUrl: `https://github.com/${reference.owner}/${reference.repo}/pull/${reference.number}`, @@ -123,7 +126,7 @@ export function createPullRequestMergeDetailsResolver(db: Db): PullRequestMergeD }; } -export function createPullRequestMergeStateResolver(db: Db): PullRequestMergeStateResolver { - const resolveDetails = createPullRequestMergeDetailsResolver(db); - return async (companyId, reference) => (await resolveDetails(companyId, reference)).state; +export function createPullRequestMergeStateResolver(db: Db, options?: GitHubExternalObjectProviderOptions): PullRequestMergeStateResolver { + const resolveDetails = createPullRequestMergeDetailsResolver(db, options); + return async (companyId, reference, signal) => (await resolveDetails(companyId, reference, signal)).state; } diff --git a/server/src/services/grok-bot/workspace-inspection.ts b/server/src/services/grok-bot/workspace-inspection.ts index f1db9a4a851..73cad27c8aa 100644 --- a/server/src/services/grok-bot/workspace-inspection.ts +++ b/server/src/services/grok-bot/workspace-inspection.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, inArray, isNotNull, or, sql } from "drizzle-orm"; import { dotRunnerAssignments, environmentLeases, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { authorizeExternalBotOperator } from "../external-bot-authority.js"; @@ -74,6 +75,7 @@ export async function inspectGrokWorkspaceObligations(db: Db, companyId: string, return { ...rest, externalWorkspaceInspection: inspection }; }; await tx.update(heartbeatRuns).set({ resultJson: retireFileObligation(run.resultJson!) }).where(eq(heartbeatRuns.id, run.id)); + if (actions.length) await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "recovery_action" }); for (const action of actions) { await tx.update(issueRecoveryActions).set({ evidence: retireFileObligation(action.evidence), updatedAt: new Date(), nextAction: "Inspect Grok and verify its connection before sending a new task message. The old work will not be replayed.", diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index ebce78a8266..ced43f5376f 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { readNativeEventEvidenceOutcome } from "./native-runtime/evidence-classifier.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { isNativeResponseEvidence, notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js"; @@ -9,6 +10,8 @@ import type { Db } from "@paperclipai/db"; import { heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; import { nativeSha256 } from "./native-runtime/canonical.js"; +export const RETRY_EXHAUSTION_MESSAGE_PREFIX = "Bounded retry exhausted"; + export interface AppendHeartbeatRunEventInput { companyId: string; runId: string; @@ -74,10 +77,11 @@ export async function appendHeartbeatRunEvent( db: Db, input: AppendHeartbeatRunEventInput, ): Promise { + const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); - if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment) { + if (!input.nativeSource && !input.retryExhaustion && !projectsProgress && !projectsResponse && !projectsReassessment && !projectsRetention) { const allocated = db.$with("allocated_run_event", {}).as(sql` update ${heartbeatRuns} set next_event_seq = ${heartbeatRuns.nextEventSeq} + 1, @@ -129,7 +133,7 @@ export async function appendHeartbeatRunEvent( eq(heartbeatRunEvents.runId, input.runId), eq(heartbeatRunEvents.agentId, input.agentId), eq(heartbeatRunEvents.eventType, "lifecycle"), - sql`${heartbeatRunEvents.message} like 'Bounded retry exhausted%'`, + sql`${heartbeatRunEvents.message} like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`}`, sql`${heartbeatRunEvents.payload} @> ${JSON.stringify(input.retryExhaustion)}::jsonb`, )) .limit(1) @@ -187,6 +191,7 @@ export async function appendHeartbeatRunEvent( if (projectsResponse && run.runtimeMode === "native") { await notifyNativeResponsePresentation(tx, { companyId: run.companyId, runId: input.runId }); } + if (projectsRetention) await notifyAttentionRetentionWork(tx, { companyId: input.companyId, sourceKind: "failed_run" }); const seq = await allocateHeartbeatRunEventSeq( tx as unknown as Db, input.runId, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index eb655fdef25..4405acb742c 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,7 @@ +import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js"; +import { reconcileTaskChatRetryReceipts } from "./task-retry.js"; +import { registerIssueRecoveryWork } from "./recovery/work.js"; +import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js"; import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; @@ -999,6 +1003,7 @@ export function heartbeatService( finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, + nextScheduledRetryAt, retryScheduledRetryNow, scheduleBoundedRetry, } = createHeartbeatRetries(db, { @@ -1333,8 +1338,8 @@ export function heartbeatService( return heartbeatQueue.dispatchPendingNativeStatusWakeups(...args); } - function resumeQueuedRunsAfterDrain() { - const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(); + function resumeQueuedRunsAfterDrain(...args: Parameters) { + const promise = heartbeatQueue.resumeQueuedRunsAfterDrain(...args); activeWakeupPromises.add(promise); void promise.then( () => { activeWakeupPromises.delete(promise); }, @@ -1609,15 +1614,15 @@ export function heartbeatService( void reportRunFailure(db, updated, failureReport); } - async function reconcileStrandedAssignedIssues(opts?: { skipExternalConversationSettlement?: boolean }) { + async function reconcileStrandedAssignedIssues(opts?: Parameters[0]) { return recovery.reconcileStrandedAssignedIssues({ - skipExternalConversationSettlement: opts?.skipExternalConversationSettlement, + ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff(), }); } - async function sweepStaleIssueLocks() { - return recovery.sweepStaleIssueLocks(); + async function sweepStaleIssueLocks(opts?: Parameters[0]) { + return recovery.sweepStaleIssueLocks(opts); } function issueIdFromRunContext(contextSnapshot: unknown) { @@ -1647,10 +1652,7 @@ export function heartbeatService( }); } - async function reconcileTaskWatchdogs(opts?: { - companyId?: string | null; - runId?: string | null; - }) { + async function reconcileTaskWatchdogs(opts?: Parameters[0]) { return taskWatchdogs.reconcileTaskWatchdogs({ ...opts, issueCreatedAtGte: await getWorktreeExecutionCutoff(), @@ -1675,10 +1677,7 @@ export function heartbeatService( return recovery.buildRunOutputSilence(run, now); } - async function reconcileResolvedDependencyWakes(opts?: { - runId?: string | null; - companyId?: string | null; - }) { + async function reconcileResolvedDependencyWakes(opts?: Parameters[0]) { return recovery.reconcileResolvedDependencyWakeBackstop(opts); } @@ -2007,8 +2006,11 @@ export function heartbeatService( const dispatchIssueId = readNonEmptyString(parseObject(run.contextSnapshot).issueId); const resumingAdmittedConversationTurn = !!runOptions.nativeLeaseOwner && typeof run.contextSnapshot?.conversationSessionGeneration === "number"; - if (dispatchIssueId && isConversation(await getIssueExecutionContext(run.companyId, dispatchIssueId)) - && !resumingAdmittedConversationTurn && !(await instanceSettings.getExperimental()).enableAgentChat) { + const conversationIdentity = dispatchIssueId && !resumingAdmittedConversationTurn + ? await db.select({ conversationAgentId: issues.conversationAgentId, conversationUserId: issues.conversationUserId }) + .from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, dispatchIssueId))).then(rows => rows[0] ?? null) + : null; + if (isConversation(conversationIdentity) && !(await instanceSettings.getExperimental()).enableAgentChat) { await setRunStatus(run.id, "cancelled", { finishedAt: new Date(), error: "Agent Chat is disabled", errorCode: "agent_chat_disabled" }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date() }); await releaseIssueExecutionAndPromote((await getRun(run.id))!, { suppressImmediateRecovery: true }); @@ -2204,44 +2206,6 @@ export function heartbeatService( context[PAPERCLIP_EXTERNAL_CHAT_EXECUTION_BOUND_KEY] = true; } const wakeCommentId = deriveCommentId(context, null); - const wakeCommentContext = - issueContext && wakeCommentId - ? await db - .select({ - id: issueComments.id, - body: issueComments.body, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - }) - .from(issueComments) - .where( - and( - eq(issueComments.id, wakeCommentId), - eq(issueComments.issueId, issueContext.id), - eq(issueComments.companyId, agent.companyId), - ), - ) - .then((rows) => { - const row = rows[0] ?? null; - return row?.deletedAt - ? { - ...row, - body: "", - presentation: null, - metadata: null, - } - : row; - }) - : null; let issueAssigneeOverrides = issueContext && issueContext.assigneeAgentId === agent.id ? parseIssueAssigneeAdapterOverrides( @@ -2610,7 +2574,7 @@ export function heartbeatService( continuationSummary, executionContinuation, paperclipWakePayload, safeWakeComments, safeWakeCommentContext, pinnedSkillTest, } = await timePreparation("task_context", () => prepareHeartbeatTaskContext(db, { run, agent, - task: { issueRef, issueContext, wakeCommentContext, context, taskSession }, + task: { issueRef, issueContext, context, taskSession }, policy: { trustPreset, experimentalInstanceSettings, recovering: Boolean(runOptions.nativeRestartRecovery || runOptions.nativeLeaseOwner) }, services: { issuesSvc, getPinnedSkillTestContext }, })); @@ -3724,7 +3688,7 @@ export function heartbeatService( await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) - .where(eq(heartbeatRuns.id, run.id))); + .where(eq(heartbeatRuns.id, run.id)), { companyId: run.companyId }); } nativeOwnershipHeld = nativeOwnershipHeld || @@ -4246,6 +4210,30 @@ export function heartbeatService( computeTaskDrain, applyTaskDrain, + registerDeferredReceiptWork: (coordinator: Parameters[1], canRun: () => boolean) => + registerDeferredReceiptWork(db, coordinator, { + resume: scope => heartbeatQueue.resumeDeferredExecutionReceipts(scope), + settleTaskChat: scope => reconcileTaskChatRetryReceipts(db, { ...scope, settleOnly: true }), + dispatchNative: dispatchPendingNativeStatusWakeups, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), + registerIssueStateRecoveryWork: (coordinator: Parameters[1], canRun: () => boolean) => + registerIssueRecoveryWork(db, coordinator, { + stranded: scope => reconcileStrandedAssignedIssues({ ...scope, skipExternalConversationSettlement: true, skipRelatedRecovery: true }), + orphanBlockers: recovery.reconcileUnassignedBlockingIssues, + recoveryActions: recovery.reconcileActiveRecoveryActions, + dependencies: reconcileResolvedDependencyWakes, + locks: sweepStaleIssueLocks, + watchdogs: reconcileTaskWatchdogs, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), + registerRunDispatchWork: (coordinator: Parameters[0], canRun: () => boolean) => + registerRunDispatchWork(coordinator, { + promoteDueScheduledRetries, + nextScheduledRetryAt, + resumeQueuedRunsAfterDrain, + schedulingSuppressed: async () => (await getSchedulingSuppression(true)).suppressed, + }, () => canRun() && readTaskDrain(new Date()) === null), promoteDueScheduledRetries, retryScheduledRetryNow, diff --git a/server/src/services/heartbeat/agent-status.ts b/server/src/services/heartbeat/agent-status.ts index f034cd06594..23cef4b2f87 100644 --- a/server/src/services/heartbeat/agent-status.ts +++ b/server/src/services/heartbeat/agent-status.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { agents, heartbeatRuns, type Db } from "@paperclipai/db"; import { and, asc, desc, eq, lte, sql } from "drizzle-orm"; @@ -91,6 +92,7 @@ export function createHeartbeatAgentStatus(db: Db) { )).limit(1); const nextStatus = running ? "running" : settledStatus(outcome, options?.keepIdleOnFailure); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const [updated] = await tx.update(agents).set({ status: nextStatus, @@ -140,6 +142,7 @@ export function createHeartbeatAgentStatus(db: Db) { eq(heartbeatRuns.agentId, candidate.id))) .orderBy(desc(heartbeatRuns.finishedAt), desc(heartbeatRuns.createdAt), desc(heartbeatRuns.id)).limit(1); const nextStatus = settledStatus(lastRun?.status); + await notifyAttentionRetentionWork(tx, { companyId: candidate.companyId }); const [agent] = await tx.update(agents).set({ status: nextStatus, errorReason: nextStatus === "error" ? errorReason(lastRun?.error) : null, updatedAt: now, }).where(and(eq(agents.id, candidate.id), eq(agents.companyId, candidate.companyId), diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts new file mode 100644 index 00000000000..7a69bc3be58 --- /dev/null +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -0,0 +1,118 @@ +import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm"; +import { agentWakeupRequests, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; +import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; +type NativeIntentOptions = DeferredExecutionScope & { + onDeferredDeadline(at: number): void; + onError(error: unknown): void; +}; + +export function registerDeferredReceiptWork( + db: Db, + coordinator: ReturnType, + owner: { + resume(scope: DeferredReceiptRecoveryOptions): Promise; + settleTaskChat(scope: DeferredExecutionScope): Promise; + dispatchNative(scope: NativeIntentOptions): Promise; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + function register(kind: "deferred" | "task_chat" | "native") { + const cursors = new Map(); + let enabled = false; + const queue = kind === "deferred" ? DELIVERY_QUEUES.deferredExecutionReceipts + : kind === "task_chat" ? DELIVERY_QUEUES.taskRetryReceiptSettlement : DELIVERY_QUEUES.nativeStatusWakeIntents; + return registerScopedDeadlineWork(coordinator, { + queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), + matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId), + load: async scope => { + const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId, + status: agentWakeupRequests.status, updatedAt: agentWakeupRequests.updatedAt, + claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error, + delayed: sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null + or ${agentWakeupRequests.payload}->'executionWait' is not null`, + interruptedLegacy: sql`exists (select 1 from heartbeat_runs r + where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + }).from(agentWakeupRequests).where(and( + scope ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope?.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + cursor ? gt(agentWakeupRequests.id, cursor) : undefined, + kind === "native" ? and( + eq(agentWakeupRequests.requestedByActorType, "system"), + eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), + inArray(agentWakeupRequests.status, ["queued", "claimed"]), isNull(agentWakeupRequests.runId), + ) : and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + kind === "task_chat" ? and(eq(agentWakeupRequests.reason, "task_retry"), + sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`) : or( + eq(agentWakeupRequests.reason, "task_retry"), + sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, + sql`${agentWakeupRequests.payload}->'executionWait' is not null`, + sql`jsonb_typeof(${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}') = 'array' + and ${agentWakeupRequests.payload}#>'{_paperclipWakeContext,wakeCommentIds}' <> '[]'::jsonb`, + sql`${agentWakeupRequests.payload}->>'mutation' = 'interaction'`, + sql`exists (select 1 from heartbeat_runs r where r.company_id = ${agentWakeupRequests.companyId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + )), + )).orderBy(asc(agentWakeupRequests.id)).limit(100); + const complete = rows.length < 100; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, + dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000 + : kind === "native" && row.error ? row.updatedAt.getTime() + 30_000 + : kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000 + : Date.now(), + })), complete }; + }, + check: async entry => { + if (!enabled) return null; + if (kind !== "task_chat") { + const [current] = await db.select({ status: agentWakeupRequests.status, payload: agentWakeupRequests.payload, + updatedAt: agentWakeupRequests.updatedAt, claimedAt: agentWakeupRequests.claimedAt, error: agentWakeupRequests.error, + interruptedLegacy: sql`exists (select 1 from heartbeat_runs r + where r.company_id = ${agentWakeupRequests.companyId} and r.agent_id = ${agentWakeupRequests.agentId} + and r.runtime_mode = 'legacy' and r.status = 'cancelled' + and r.result_json->>'queuedCommentInterruptQueueId' = ${agentWakeupRequests.id}::text)`, + }).from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, entry.companyId), + eq(agentWakeupRequests.id, entry.id))); + if (!current) return null; + const due = kind === "native" && current.status === "claimed" && current.claimedAt + ? current.claimedAt.getTime() + 60_000 + : kind === "native" && current.status === "queued" && current.error ? current.updatedAt.getTime() + 30_000 + : kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy + && (current.payload?.queuedCommentInterrupt || current.payload?.executionWait) + ? current.updatedAt.getTime() + 30_000 : null; + if (due !== null && due > Date.now()) return { ...entry, dueAt: due }; + } + const errors: unknown[] = []; + let next: number | null = null; + const scope = { companyId: entry.companyId, requestId: entry.id }; + if (kind === "native") await owner.dispatchNative({ ...scope, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); }, + onError: error => { errors.push(error); }, + }); + else if (kind === "task_chat") await owner.settleTaskChat(scope); + else await owner.resume({ ...scope, onError: error => { errors.push(error); } }); + if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); + } + const workers = [register("deferred"), register("task_chat"), register("native")]; + return { ready: Promise.all(workers.map(worker => worker.ready)), + wake() { for (const worker of workers) worker.invalidate(); }, + async stop() { await Promise.all(workers.map(worker => worker.stop())); } }; +} diff --git a/server/src/services/heartbeat/dispatch-work.ts b/server/src/services/heartbeat/dispatch-work.ts new file mode 100644 index 00000000000..5b1f728688a --- /dev/null +++ b/server/src/services/heartbeat/dispatch-work.ts @@ -0,0 +1,58 @@ +import type { RunDispatchScope } from "../run-dispatch-notifications.js"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { registerDeadlineWork } from "../deadline-work.js"; +import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; + +export function registerRunDispatchWork( + coordinator: ReturnType, + owner: { + promoteDueScheduledRetries(): Promise; + nextScheduledRetryAt(): Promise; + resumeQueuedRunsAfterDrain(scope?: RunDispatchScope): Promise; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + let suppressed = false; + const retries = registerDeadlineWork(coordinator, DELIVERY_QUEUES.scheduledRetryDispatch, { + retryMs: 30_000, + continuationMs: 1_000, + notificationCoalesceMs: 1_000, + canRun, + run: async signal => { + suppressed = await owner.schedulingSuppressed(); + if (!signal.aborted && !suppressed && canRun()) await owner.promoteDueScheduledRetries(); + }, + next: () => suppressed ? Promise.resolve(null) : owner.nextScheduledRetryAt(), + }); + const scopes = new Set(["*"]); + const queue = registerDeadlineWork(coordinator, DELIVERY_QUEUES.queuedRunDispatch, { + retryMs: 30_000, + notificationCoalesceMs: 1_000, + canRun, + onSettled: hints => { for (const hint of hints ?? ["*"]) scopes.add(hint); }, + run: async signal => { + const requested = scopes.has("*") ? ["*"] : [...scopes]; + if (requested[0] === "*") scopes.clear(); + else for (const hint of requested) scopes.delete(hint); + for (let index = 0; index < requested.length; index++) { + if (signal.aborted || !canRun()) { + for (const hint of requested.slice(index)) scopes.add(hint); + return; + } + try { + await owner.resumeQueuedRunsAfterDrain(requested[index] === "*" ? undefined : JSON.parse(requested[index]) as RunDispatchScope); + } catch (error) { + for (const hint of requested.slice(index)) scopes.add(hint); + throw error; + } + } + }, + next: async () => null, + }); + return { + ready: Promise.all([retries.ready, queue.ready]), + wake() { scopes.add("*"); retries.wake(); queue.wake(); }, + async stop() { await Promise.all([retries.stop(), queue.stop()]); }, + }; +} diff --git a/server/src/services/heartbeat/execution-configuration.ts b/server/src/services/heartbeat/execution-configuration.ts index 9dd767a3811..fb9626b932f 100644 --- a/server/src/services/heartbeat/execution-configuration.ts +++ b/server/src/services/heartbeat/execution-configuration.ts @@ -139,28 +139,27 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear stripWorkspaceRuntimeFromExecutionRunConfig(mergedConfig); const skillActor: AuthorizationActor = { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }; - const runScopedMentionedSkillKeys = - preconfiguredHome || pinnedSkillTest ? [] : await time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ + const [mentionedSkills, githubIdentity] = await Promise.allSettled([ + preconfiguredHome || pinnedSkillTest ? Promise.resolve([]) : time("configuration_mentions", () => resolveRunScopedMentionedSkillKeys({ db, companyId: agent.companyId, issueId, commentIds: extractWakeCommentIds(context), actor: skillActor, - })); + })), + time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection(db, agent.companyId, { + agentId: agent.id, responsibleUserId, allowStandingDelegation: false, + })), + ]); + if (mentionedSkills.status === "rejected") throw mentionedSkills.reason; + if (githubIdentity.status === "rejected") throw githubIdentity.reason; + const runScopedMentionedSkillKeys = mentionedSkills.value; const runScopedSkillKeys = !pinnedSkillTest && acceptedPlanContinuationWake && !acceptedPlanWakeRoutingDecision?.suppressAcceptedContinuation ? [...runScopedMentionedSkillKeys, ACCEPTED_PLAN_CONVERSION_SKILL_KEY] : runScopedMentionedSkillKeys; - const githubSelection = await time("configuration_github_identity", () => resolveManagedGitHubIdentitySelection( - db, - agent.companyId, - { - agentId: agent.id, - responsibleUserId, - allowStandingDelegation: false, - }, - )); + const githubSelection = githubIdentity.value; const useHostGitHub = !githubSelection.configured && trustPreset.kind === "standard" && @@ -274,6 +273,7 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear ); const nativeRunnerPreparationSpans: NativeRunHistoricalSpan[] = []; const skillsPrepareStartedAtMs = Date.now(); + let skillsPrepareEndedAtMs = skillsPrepareStartedAtMs; let skillDeliveryConfig = effectiveResolvedConfig; const prepareSkills = async () => { @@ -346,24 +346,35 @@ export async function prepareHeartbeatExecutionConfiguration(db: Db, input: Hear throw error; } }; - const runtimeSkillEntries = preconfiguredHome ? [] : await time("configuration_skills", prepareSkills); + const skillsPreparation = (preconfiguredHome ? Promise.resolve([]) : time("configuration_skills", prepareSkills)) + .finally(() => { skillsPrepareEndedAtMs = Date.now(); }); + const assignmentsPreparation = preconfiguredHome ? Promise.resolve([]) : time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const [preparedSkills, preparedAssignments, preparedInstructions] = await Promise.allSettled([ + skillsPreparation, + assignmentsPreparation, + assignmentsPreparation.then(() => time("configuration_connection_instructions", () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }))), + ]); + if (preparedSkills.status === "rejected") throw preparedSkills.reason; + if (preparedAssignments.status === "rejected") throw preparedAssignments.reason; + if (preparedInstructions.status === "rejected") throw preparedInstructions.reason; + const runtimeSkillEntries = preparedSkills.value; nativeRunnerPreparationSpans.push({ name: "skills.prepare", parentName: "task.prepare", startedAtMs: skillsPrepareStartedAtMs, - endedAtMs: Date.now(), + endedAtMs: skillsPrepareEndedAtMs, }); - const connectorAssignments = preconfiguredHome ? [] : await time("configuration_connector_assignments", () => resolveConnectorAssignments(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id, issueId: typeof context.issueId === "string" ? context.issueId : undefined })); + const connectorAssignments = preparedAssignments.value; const connectorSkillConfig = await time("configuration_connector_skills", () => applyConnectorSkills(skillDeliveryConfig, runtimeSkillEntries, connectorAssignments)); // Both CLI adapters and native context materialization use the same resolved set. runtimeSkillEntries.splice(0, runtimeSkillEntries.length, ...connectorSkillConfig.paperclipRuntimeSkills); const connectorDelivery = await time("configuration_connector_delivery", () => prepareConnectorSkillDelivery(connectorSkillConfig, agent.adapterType)); // Always replace this runtime-only field; caller wake data cannot supply skills. context.paperclipWake = { ...parseObject(context.paperclipWake), connectorSkillInstructions: connectorDelivery.instructions }; - let runtimeConfig = await time("configuration_connection_instructions", () => prepareConnectionInstructionDelivery({ - resolve: () => resolveAssignedConnectionInstructionsForRun(db, { companyId: agent.companyId, agentId: agent.id, runId: run.id }), + let runtimeConfig = await prepareConnectionInstructionDelivery({ + resolve: async () => preparedInstructions.value, context, config: connectorDelivery.config, native: agent.adapterType === "paperclip_runner", - })); + }); const agentIdentity = supportsManagedAgentIdentity(agent.adapterType, agent.adapterConfig, run.runtimeMode === "native" ? run.driverKind : undefined) ? await time("configuration_agent_identity", () => agentIdentityService(db).ensureAgentIdentity(agent.companyId, agent.id)) : undefined; diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index a41b486b647..102450cdad7 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,9 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { writeDeferredExecutionWait } from "../deferred-execution-wait.js"; +import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js"; +import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -221,7 +227,7 @@ export interface HeartbeatQueueDependencies extends Pick void; cancelRunInternal: (runId: string, reason?: string, options?: { errorCode?: string }) => Promise; cancelActiveForAgentInternal: (agentId: string, reason: string) => Promise; - resumeExecutionWaitComments: () => Promise; + resumeExecutionWaitComments: (scope?: DeferredReceiptRecoveryOptions) => Promise; resumeQueuedCommentInterrupt: (companyId: string, queueId: string) => Promise; resumeSavedLegacyComments: (companyId: string, queueId: string) => Promise; formatIssueIdentifierLink: (identifier: string | null, fallback: string) => string; @@ -1267,6 +1273,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } async function bindClaimedIssueExecution(tx: Db, ownsIssue: boolean, claimedRun: typeof heartbeatRuns.$inferSelect | null | undefined) { if (!claimedRun || !issueId || !ownsIssue) return; + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: claimedRun.id, executionAgentNameKey: normalizeAgentNameKey(agent.name), @@ -1850,13 +1857,15 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende return cancelled; } - async function resumeQueuedRuns() { + async function resumeDeferredExecutionReceipts(scope: DeferredReceiptRecoveryOptions = {}) { if ((await getSchedulingSuppression()).suppressed) return; - await resumeExecutionWaitComments(); + await resumeExecutionWaitComments(scope); const cutoff = await getWorktreeExecutionCutoff(); const pendingInterrupts = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) .from(agentWakeupRequests).innerJoin(companies, eq(companies.id, agentWakeupRequests.companyId)) - .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(companies.status, "active"), sql`${agentWakeupRequests.payload}->'queuedCommentInterrupt' is not null`, lte(agentWakeupRequests.updatedAt, new Date(Date.now() - 30_000)), @@ -1867,6 +1876,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.status, "deferred_issue_execution"), )); await resumeQueuedCommentInterrupt(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to resume interrupted comment queue"); }); } @@ -1880,7 +1890,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende or(eq(issues.assigneeAgentId, agentWakeupRequests.agentId), sql`${agentWakeupRequests.payload} #>> '{_paperclipWakeContext,wakeReason}' = 'issue_comment_mentioned'`))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.status, "deferred_issue_execution"), isNull(issues.executionRunId), or(and( sql`jsonb_typeof(${agentWakeupRequests.payload} #> '{_paperclipWakeContext,wakeCommentIds}') = 'array'`, @@ -1919,7 +1931,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (latest.runtimeMode !== "legacy" && latest.runtimeMode !== "native") continue; if (wake.idempotencyKey?.startsWith("chat-inbound:")) { await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, - issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(() => { + issueId: String(wake.payload?.issueId), requestId: wake.id }).catch(err => { + scope.onError?.(err); logger.warn({ requestId: wake.id }, "saved chat source could not resume after recovery"); }); continue; @@ -1931,6 +1944,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende latest.errorCode === "execution_reconciliation_required"; if ((latest.status !== "cancelled" || cancelledAdmission) && await getExecutionBlocker(db, wake.companyId, String(wake.payload?.issueId))) { await resumeSavedLegacyComments(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to deliver saved legacy comment after recovery stopped"); }); } @@ -1938,6 +1952,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await releaseIssueExecutionAndPromote(latest, { suppressImmediateRecovery: true, resumeStrandedQueue: true }).then(() => { if (latest.runtimeMode === "native") releasedNativeSources.add(releaseSource); }).catch(err => { + scope.onError?.(err); logger.warn({ err, queueId: wake.id }, "failed to promote stranded comments"); }); } @@ -1955,6 +1970,8 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )) .innerJoin(companies, eq(companies.id, heartbeatRuns.companyId)) .where(and( + scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.status, "deferred_issue_execution"), eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.runtimeMode, "legacy"), @@ -1963,16 +1980,21 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )); for (const run of interruptedQueues) { await releaseIssueExecutionAndPromote(run, { suppressImmediateRecovery: true }).catch((err) => { + scope.onError?.(err); logger.error({ err, runId: run.id }, "failed to retry interrupted comment queue"); }); } + } + + async function resumeQueuedRuns() { + await resumeDeferredExecutionReceipts(); await resumeQueuedRunsAfterDrain(); } // Release admission promptly without first scanning unrelated recovery work. // Every run still passes the ordinary per-agent claim and dispatch gates. - async function resumeQueuedRunsAfterDrain() { + async function resumeQueuedRunsAfterDrain(scope?: { companyId: string }) { if ((await getSchedulingSuppression()).suppressed) return; const cutoff = await getWorktreeExecutionCutoff(); const queuedRuns = await db @@ -1982,6 +2004,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .where( and( eq(heartbeatRuns.status, "queued"), + scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, eq(companies.status, "active"), cutoff ? gte(heartbeatRuns.createdAt, cutoff) : undefined, ), @@ -2504,17 +2527,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // its one durable receipt ID on a skipped wake. if (caseRequest) return { created: false }; if (executionWaitRequestId) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify({ - reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" - ? "This task is paused. Resume it to send your saved message." - : "Waiting for task execution to be enabled. Your message is saved."), - })}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, executionWaitRequestId), - eq(agentWakeupRequests.companyId, agent.companyId), eq(agentWakeupRequests.agentId, agentId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); + await writeDeferredExecutionWait(db, { companyId: agent.companyId, requestId: executionWaitRequestId }, { + reason: skipReason, message: patch.error ?? (skipReason === "issue_tree_hold_active" + ? "This task is paused. Resume it to send your saved message." + : "Waiting for task execution to be enabled. Your message is saved."), + }); return { created: false }; } const request = { @@ -3402,15 +3419,14 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende opts.issueStateGuard?.onExecutionDeferred?.(executionBlocker.cause); const condition = { recoveryActionId: executionBlocker.recoveryActionId, ...continuationWait }; if (executionWaitRequestId) { - await tx.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', ${JSON.stringify(condition)}::jsonb)`, - updatedAt: new Date(), - }).where(eq(agentWakeupRequests.id, executionWaitRequestId)); + await writeDeferredExecutionWait(tx as unknown as Db, { companyId: agent.companyId, + requestId: executionWaitRequestId }, condition, { replace: true }); return { kind: "deferred" as const, admitted: true }; } if (durableRequest || wakeCommentId || hasInteractionContinuationWakeContext(enrichedContextSnapshot) || readNonEmptyString(enrichedContextSnapshot.nativeStatusWakeIntentId)) { + await notifyDeferredExecutionWork(tx, { companyId: agent.companyId }); await tx.insert(agentWakeupRequests).values({ ...durableReceiptFields, companyId: agent.companyId, agentId, source, triggerDetail, reason, @@ -3758,6 +3774,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .from(agents) .where(eq(agents.id, legacyRun.agentId)) .then((rows) => rows[0] ?? null); + await notifyStatusCardIssueChange(tx, issue.companyId); await tx .update(issues) .set({ @@ -3919,6 +3936,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); if (issue.status !== "blocked") await notifyChatResponseAvailability(tx, { companyId: issue.companyId, issueId: issue.id }); + await notifyRoutineExecutionStatus(tx, issue.id, issue.companyId); await tx .update(issues) .set({ @@ -4028,6 +4046,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende })); } if (followUp) { + if (!followUp.consumed) await notifyDeferredExecutionWork(tx, { companyId: agent.companyId }); const assignmentId = followUp.assignmentId; const pending = followUp.consumed ? undefined : pendingFollowUps.find(wake => wake.payload?.[followUpPayloadKey] === assignmentId && @@ -4632,6 +4651,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } const queueOutcome = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: agent.companyId }); await tx.execute( sql`select id from agents where id = ${agentId} and company_id = ${agent.companyId} for update`, ); @@ -4765,6 +4785,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende companyId?: string; limit?: number; staleClaimMs?: number; + requestId?: string; + onDeferredDeadline?: (at: number) => void; + onError?: (error: unknown) => void; } = {}, ) { const now = new Date(); @@ -4777,6 +4800,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende input.companyId ? eq(agentWakeupRequests.companyId, input.companyId) : undefined, + input.requestId ? eq(agentWakeupRequests.id, input.requestId) : undefined, eq(agentWakeupRequests.requestedByActorType, "system"), eq(agentWakeupRequests.requestedByActorId, "native-status-committer"), inArray(agentWakeupRequests.status, ["queued", "claimed"]), @@ -4853,6 +4877,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende candidate.claimedAt && now.getTime() - candidate.claimedAt.getTime() < staleClaimMs ) { + input.onDeferredDeadline?.(candidate.claimedAt.getTime() + staleClaimMs); deferred += 1; continue; } @@ -5027,6 +5052,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (wakeRun) dispatched += 1; else deferred += 1; } catch (error) { + input.onError?.(error); await db .update(agentWakeupRequests) .set({ @@ -5070,6 +5096,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende withChatControlRecoveryGate, trackWakeup, resumeQueuedRuns, + resumeDeferredExecutionReceipts, resumeQueuedRunsAfterDrain, parseHeartbeatPolicy, nextTimerHeartbeatAt, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 3b180d2679d..e506f0f7e62 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1979,7 +1979,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD let settledProcessesDiscovered = false; - async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) { + async function reapOrphanedRuns(opts?: { staleThresholdMs?: number; skipNativeStatusWakeIntents?: boolean; skipScheduledRetryCancellation?: boolean; skipCapturedInstructionCommit?: boolean; skipInstructionDirectoryCleanup?: boolean; skipStoppedInstructionCollection?: boolean; skipCancelledWorkspacePreservation?: boolean; skipObsoletePolicyReviews?: boolean; skipAutomaticCompletionReviews?: boolean; skipUnsafeWorkspaceExports?: boolean; skipNativePendingEffects?: boolean; skipNativeResponsePresentation?: boolean; skipNativeReviewResponse?: boolean; skipNativeStatusReassessment?: boolean; skipRetainedNativeSessionCleanup?: boolean; skipNativeWorkspaceFinalization?: boolean; skipNativeResultCommit?: boolean; skipNativeRunProjection?: boolean; skipNativeRecoveryNotices?: boolean; skipExternalConversationSettlement?: boolean; skipExternalAgentSettlement?: boolean; skipNativeEnvironmentSettlement?: boolean }) { if (!opts?.skipCancelledWorkspacePreservation) await preserveCancelledNativeWorkspaces(db); const staleThresholdMs = opts?.staleThresholdMs ?? 0; const now = new Date(); @@ -2023,7 +2023,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD }); if (!opts?.skipRetainedNativeSessionCleanup) scheduleRetainedNativeSessionCleanup(); await reconcileRetainedWorkspaces(db, environmentRuntime, resumeRemoteStopComments); - await dispatchPendingNativeStatusWakeups().catch((error) => { + if (!opts?.skipNativeStatusWakeIntents) await dispatchPendingNativeStatusWakeups().catch((error) => { logger.warn( { err: error }, "failed to dispatch persisted native status wake intents before orphan reaping", diff --git a/server/src/services/heartbeat/retries.test.ts b/server/src/services/heartbeat/retries.test.ts index a557871c553..3d7e9a1ec34 100644 --- a/server/src/services/heartbeat/retries.test.ts +++ b/server/src/services/heartbeat/retries.test.ts @@ -35,6 +35,7 @@ function callbacks() { getWorktreeExecutionCutoff: vi.fn(async (): Promise => null), applyRunDispatchPostCommitEffects: vi.fn<(effects: PostCommitEffect[]) => void>(), runDispatch: { + nextScheduledRetryAt: vi.fn(async () => null), evaluateScheduledRetryGate: vi.fn(async () => ({ allowed: true })), promoteDueScheduledRetries: vi.fn(async () => ({ promoted: 0, runIds: [], postCommitEffects: [] })), promoteScheduledRetry: vi.fn(async () => ({ outcome: "not_promoted" })), @@ -183,7 +184,9 @@ describe("heartbeat retry module callbacks", () => { }); }); -const support = await getEmbeddedPostgresTestSupport(); +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_HEARTBEAT_RETRIES_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describePostgres = support.supported ? describe : describe.skip; if (!support.supported) console.warn(`Skipping retry PostgreSQL tests: ${support.reason ?? "unsupported host"}`); @@ -193,11 +196,11 @@ describePostgres("heartbeat retry module database wiring", () => { const now = new Date("2026-10-08T12:00:00Z"); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("heartbeat-retries-"); + db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); - afterAll(async () => { await database?.cleanup(); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); afterEach(async () => { await db.delete(issues); diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 946f5e309f8..414dab30ec6 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -1,3 +1,6 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { deriveTaskKeyWithHeartbeatFallback } from "./run-state.js"; import { infrastructureRetryDelayMs } from "../system-recovery-state.js"; import { @@ -77,7 +80,7 @@ type PlanApprovalResumeFailure = { export interface HeartbeatRetryDependencies { hasCaseHandoffForRun: (run: Pick) => Promise; runDispatch: Pick, - "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry">; + "evaluateScheduledRetryGate" | "promoteDueScheduledRetries" | "promoteScheduledRetry" | "nextScheduledRetryAt">; getAgent: (agentId: string) => Promise; getRun: ( runId: string, @@ -997,6 +1000,8 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen retryContextSnapshot.previousRunId = continuation.previousRunId; } + await notifyAttentionRetentionWork(tx, { companyId: run.companyId }); + await notifyRunDispatch(tx, { companyId: run.companyId }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -1178,6 +1183,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen } if (issueId) { + await notifyStatusCardIssueChange(tx, run.companyId); await tx .update(issues) .set({ @@ -1643,6 +1649,10 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }); } + async function nextScheduledRetryAt() { + return runDispatch.nextScheduledRetryAt({ cutoff: await getWorktreeExecutionCutoff() }); + } + async function promoteDueScheduledRetries(now = new Date()) { const cutoff = await getWorktreeExecutionCutoff(); const result = await runDispatch.promoteDueScheduledRetries({ @@ -1750,6 +1760,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen }; const updated = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: scheduled.run.companyId }); const row = await tx .update(heartbeatRuns) .set({ @@ -1901,6 +1912,7 @@ export function createHeartbeatRetries(db: Db, dependencies: HeartbeatRetryDepen finalizeAiConnectionBusyDeferral, finalizeWorkspaceBusyDeferral, promoteDueScheduledRetries, + nextScheduledRetryAt, retryScheduledRetryNow, scheduleBoundedRetry: async ( diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 820f1e50c61..a410d905d6a 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -1,3 +1,5 @@ +import { writeDeferredExecutionWait } from "../deferred-execution-wait.js"; +import type { DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { observeStoppedWorkspaceRecovery } from "../workspace-recovery.js"; import { notifyScheduledRetryCancellation } from "../scheduled-retry-cancellation-notifications.js"; import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js"; @@ -616,12 +618,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont onBlocked: (reason, message) => { wait = { reason, message }; }, }); if (!admitted) { - await db.update(agentWakeupRequests).set({ - payload: sql`jsonb_set(coalesce(${agentWakeupRequests.payload}, '{}'::jsonb), '{executionWait}', - coalesce(${agentWakeupRequests.payload}->'executionWait', '{}'::jsonb) || ${JSON.stringify(wait)}::jsonb)`, - updatedAt: new Date(), - }).where(and(eq(agentWakeupRequests.id, wake.id), eq(agentWakeupRequests.companyId, run.companyId), - eq(agentWakeupRequests.status, "deferred_issue_execution"))); + await writeDeferredExecutionWait(db, { companyId: run.companyId, requestId: wake.id }, wait); continue; } } @@ -777,21 +774,25 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont idempotencyKey: `task-retry-dispatch:${wake.id}` }, wake.id); } - async function resumeExecutionWaitComments() { + async function resumeExecutionWaitComments(scope: DeferredReceiptRecoveryOptions = {}) { if ((await getSchedulingSuppression()).suppressed) return; - await reconcileTaskChatRetryReceipts(db); + await reconcileTaskChatRetryReceipts(db, scope); const retries = await db.select({ id: agentWakeupRequests.id, companyId: agentWakeupRequests.companyId }) .from(agentWakeupRequests).innerJoin(issues, and(eq(issues.companyId, agentWakeupRequests.companyId), sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`, notInArray(issues.status, ["done", "cancelled"]))) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); for (const retry of retries) { await db.update(agentWakeupRequests).set({ updatedAt: new Date() }).where(eq(agentWakeupRequests.id, retry.id)); - await resumeTaskRetry(retry.companyId, retry.id).catch(err => logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred")); + await resumeTaskRetry(retry.companyId, retry.id).catch(err => { + scope.onError?.(err); logger.warn({ err, requestId: retry.id }, "Task retry dispatch deferred"); + }); } const waits = await db.select({ wake: agentWakeupRequests }) .from(agentWakeupRequests) @@ -799,7 +800,9 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont sql`${issues.id}::text = ${agentWakeupRequests.payload}->>'issueId'`, eq(issues.assigneeAgentId, agentWakeupRequests.agentId))) .innerJoin(companies, and(eq(companies.id, issues.companyId), eq(companies.status, "active"))) - .where(and(exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( + .where(and(scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, + exists(db.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, issues.companyId), eq(issueRecoveryActions.sourceIssueId, issues.id), executionBlockerPredicate(), ))), eq(agentWakeupRequests.status, "deferred_issue_execution"), @@ -831,11 +834,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont if (!run || run.companyId !== wake.companyId || run.agentId !== wake.agentId) continue; if (canContinueCancelledRun(run)) { await resumeSavedComments(wake.companyId, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, runId: run.id }, "failed to resume saved input after provider cancellation"); }); continue; } await resumeRemoteStopComments(run, wake.id).catch(err => { + scope.onError?.(err); logger.warn({ err, runId: run.id }, "failed to resume saved execution-wait message"); }); } diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 00a27c1faf2..8224f75208e 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; @@ -1044,7 +1045,8 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl if (previous && isHeartbeatRunTerminalStatus(status)) await notifyRetainedNativeSessionCleanup(tx, { companyId: previous.companyId, runId: previous.id }); if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, previous ? { companyId: previous.companyId, runId: previous.id, + issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined } : undefined); if (previous) await notifyChatRunMilestones(tx, { companyId: previous.companyId, runId: previous.id, issueId: previous.nativeIssueId ?? previous.issueId ?? readNonEmptyString(parseObject(previous.contextSnapshot).issueId) ?? undefined }); return write(tx); @@ -2447,6 +2449,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .returning() .then((rows) => rows[0]); + await notifyRunDispatch(tx, { companyId: run.companyId }); const queuedRun = await tx .insert(heartbeatRuns) .values({ diff --git a/server/src/services/heartbeat/run-preparation.test.ts b/server/src/services/heartbeat/run-preparation.test.ts index 10a447e1a74..7d5a070d91d 100644 --- a/server/src/services/heartbeat/run-preparation.test.ts +++ b/server/src/services/heartbeat/run-preparation.test.ts @@ -14,6 +14,7 @@ import { ConfigurationIncompleteFailure, createHeartbeatRunPreparation, resolveExecutionRunAdapterConfig, + readHeartbeatWakeComments, } from "./run-preparation.js"; // Existing integration suites exercise the real database and dispatch path. @@ -38,6 +39,12 @@ function databaseWithResults(...results: unknown[][]) { return { db: { select } as unknown as Db, select, scopes }; } +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(complete => { resolve = complete; }); + return { promise, resolve }; +} + const emptyRoutineContext = { routineId: null, env: null, responsibleUserId: null }; describe("heartbeat run preparation module", () => { @@ -120,6 +127,42 @@ describe("heartbeat run preparation module", () => { expect(database.select).toHaveBeenCalledTimes(2); }); + it("scopes one selected-comment snapshot for both wake consumers", async () => { + const database = databaseWithResults([{ id: "comment-one", body: "selected wake" }]); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", ["comment-one"])) + .toEqual([{ id: "comment-one", body: "selected wake" }]); + expect(database.scopes[0]).toEqual(expect.arrayContaining(["company-one", "issue-one", "comment-one"])); + expect(await readHeartbeatWakeComments(database.db, "company-one", "issue-one", [])).toEqual([]); + expect(database.select).toHaveBeenCalledOnce(); + }); + + it("settles independent binding checks before surfacing a failure or resolving values", async () => { + const projectCheck = deferred<[]>(); + const firstFailure = new Error("agent binding check failed"); + const secretsSvc = { + resolveAdapterConfigForRuntime: vi.fn(), resolveEnvBindings: vi.fn(), + collectMissingRuntimeBindings: vi.fn(async (_company, _env, context) => { + if (context.consumerType === "agent") throw firstFailure; + return context.consumerType === "project" ? projectCheck.promise : []; + }), + collectMissingAdapterConfigRuntimeBindings: vi.fn(async () => []), + }; + let settled = false; + const opening = resolveExecutionRunAdapterConfig({ + companyId: "company-one", agentId: "agent-one", projectId: "project-one", + environmentId: "environment-one", environmentEnv: { SHARED: "environment" }, + executionRunConfig: { env: { SHARED: "agent" } }, projectEnv: { SHARED: "project" }, secretsSvc, + }); + void opening.then(() => { settled = true; }, () => { settled = true; }); + await vi.waitFor(() => expect(secretsSvc.collectMissingRuntimeBindings).toHaveBeenCalledTimes(3)); + expect(secretsSvc.collectMissingAdapterConfigRuntimeBindings).toHaveBeenCalledOnce(); + expect(settled).toBe(false); + expect(secretsSvc.resolveAdapterConfigForRuntime).not.toHaveBeenCalled(); + expect(secretsSvc.resolveEnvBindings).not.toHaveBeenCalled(); + projectCheck.resolve([]); + await expect(opening).rejects.toBe(firstFailure); + }); + it("keeps missing scoped credentials as a pre-dispatch failure", async () => { const secretsSvc = { resolveAdapterConfigForRuntime: vi.fn(), diff --git a/server/src/services/heartbeat/run-preparation.ts b/server/src/services/heartbeat/run-preparation.ts index bf288599a51..bfbe6c862c1 100644 --- a/server/src/services/heartbeat/run-preparation.ts +++ b/server/src/services/heartbeat/run-preparation.ts @@ -394,76 +394,31 @@ export async function resolveExecutionRunAdapterConfig(input: { // no binding before resolving any secret value. Missing bindings short-circuit // to a configuration-incomplete blocker routed to a human owner instead of a // dispatched-then-failed run (which previously surfaced as opaque setup_failed). - if (typeof input.secretsSvc.collectMissingRuntimeBindings === "function") { - const missingBindings: MissingRuntimeBinding[] = []; - if (environmentEnv && input.environmentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - environmentEnv, - { - consumerType: "environment", - consumerId: input.environmentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } + const collectMissing = input.secretsSvc.collectMissingRuntimeBindings; + if (typeof collectMissing === "function") { + const checks: Array<() => Promise> = []; + const checkEnv = (env: unknown, consumerType: "environment" | "agent" | "project" | "routine", consumerId: string) => { + checks.push(() => collectMissing(input.companyId, env, { + consumerType, consumerId, responsibleUserId: input.responsibleUserId ?? null, + })); + }; + if (environmentEnv && input.environmentId) checkEnv(environmentEnv, "environment", input.environmentId); if (input.agentId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - parseObject(executionRunConfig.env), - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - if ( - typeof input.secretsSvc.collectMissingAdapterConfigRuntimeBindings === - "function" - ) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingAdapterConfigRuntimeBindings( - input.companyId, - executionRunConfig, - input.adapterType ?? null, - { - consumerType: "agent", - consumerId: input.agentId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + checkEnv(parseObject(executionRunConfig.env), "agent", input.agentId); + const collectAdapterMissing = input.secretsSvc.collectMissingAdapterConfigRuntimeBindings; + if (typeof collectAdapterMissing === "function") { + checks.push(() => collectAdapterMissing(input.companyId, executionRunConfig, input.adapterType ?? null, { + consumerType: "agent", consumerId: input.agentId!, responsibleUserId: input.responsibleUserId ?? null, + })); } } - if (projectEnv && input.projectId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - projectEnv, - { - consumerType: "project", - consumerId: input.projectId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); - } - if (routineEnv && input.routineId) { - missingBindings.push( - ...(await input.secretsSvc.collectMissingRuntimeBindings( - input.companyId, - routineEnv, - { - consumerType: "routine", - consumerId: input.routineId, - responsibleUserId: input.responsibleUserId ?? null, - }, - )), - ); + if (projectEnv && input.projectId) checkEnv(projectEnv, "project", input.projectId); + if (routineEnv && input.routineId) checkEnv(routineEnv, "routine", input.routineId); + const results = await Promise.allSettled(checks.map(check => Promise.resolve().then(check))); + const missingBindings: MissingRuntimeBinding[] = []; + for (const result of results) { + if (result.status === "rejected") throw result.reason; + missingBindings.push(...result.value); } if (requiredScopedEnvBinding) { const requiredEnvKeys = new Set(requiredScopedEnvBinding.keys); @@ -2244,12 +2199,45 @@ export async function attestReviewedExternalChatRun(input: { throw new Error("reviewed_chat_execution_binding_not_ready"); } +export async function readHeartbeatWakeComments(db: Db, companyId: string, issueId: string | null, commentIds: string[]) { + if (commentIds.length === 0) return []; + return db + .select({ + id: issueComments.id, + companyId: issueComments.companyId, + issueId: issueComments.issueId, + body: issueComments.body, + origin: issueComments.origin, + authorType: issueComments.authorType, + authorAgentId: issueComments.authorAgentId, + authorUserId: issueComments.authorUserId, + presentation: issueComments.presentation, + metadata: issueComments.metadata, + deletedAt: issueComments.deletedAt, + deletedByType: issueComments.deletedByType, + deletedByAgentId: issueComments.deletedByAgentId, + deletedByUserId: issueComments.deletedByUserId, + deletedByRunId: issueComments.deletedByRunId, + sourceTrust: issueComments.sourceTrust, + createdAt: issueComments.createdAt, + }) + .from(issueComments) + .where( + and( + eq(issueComments.companyId, companyId), + issueId ? eq(issueComments.issueId, issueId) : undefined, + inArray(issueComments.id, commentIds), + ), + ); +} + export async function buildPaperclipWakePayload(input: { db: Db; companyId: string; agentId?: string | null; runId?: string | null; contextSnapshot: Record; + wakeCommentRows?: Awaited>; continuationSummary?: { key: string; title: string | null; @@ -2314,36 +2302,9 @@ export async function buildPaperclipWakePayload(input: { ) return null; - const commentRows = - commentIds.length === 0 - ? [] - : await input.db - .select({ - id: issueComments.id, - issueId: issueComments.issueId, - body: issueComments.body, - origin: issueComments.origin, - authorType: issueComments.authorType, - authorAgentId: issueComments.authorAgentId, - authorUserId: issueComments.authorUserId, - presentation: issueComments.presentation, - metadata: issueComments.metadata, - deletedAt: issueComments.deletedAt, - deletedByType: issueComments.deletedByType, - deletedByAgentId: issueComments.deletedByAgentId, - deletedByUserId: issueComments.deletedByUserId, - deletedByRunId: issueComments.deletedByRunId, - sourceTrust: issueComments.sourceTrust, - createdAt: issueComments.createdAt, - }) - .from(issueComments) - .where( - and( - eq(issueComments.companyId, input.companyId), - issueId ? eq(issueComments.issueId, issueId) : undefined, - inArray(issueComments.id, commentIds), - ), - ); + const commentRows = input.wakeCommentRows + ? input.wakeCommentRows.filter(row => row.companyId === input.companyId && (!issueId || row.issueId === issueId) && commentIds.includes(row.id)) + : await readHeartbeatWakeComments(input.db, input.companyId, issueId, commentIds); const commentsById = new Map( commentRows.map((comment) => [comment.id, comment]), diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index b5b64abc948..bdbbfa07752 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -966,7 +966,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); const workspaceFinalizeStatus = hasWorkspaceRestoreFailure(adapterResult.resultJson) ? "failed" : "succeeded"; await recordWorkspaceFinalize(workspaceFinalizeStatus); if (adapterResult.nativeFinalization) { @@ -1042,7 +1042,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "running"), ), - )); + ), { companyId: run.companyId }); if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/heartbeat/scheduling.ts b/server/src/services/heartbeat/scheduling.ts index dece8b41ae9..b0e7f308297 100644 --- a/server/src/services/heartbeat/scheduling.ts +++ b/server/src/services/heartbeat/scheduling.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyIssueReviewState } from "../issue-review-state-notifications.js"; import { insertIssueComment } from "../issue-comment-writes.js"; @@ -595,6 +596,7 @@ export function createHeartbeatScheduling( try { await db.transaction(async (tx) => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); const released = await tx.update(issues) .set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)).returning({ id: issues.id }); @@ -628,6 +630,7 @@ export function createHeartbeatScheduling( // failed, and never clobber a monitor that was replaced in the meantime. await db.transaction(async tx => { await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, claimed.companyId); await tx.update(issues).set({ monitorWakeRequestedAt: null, updatedAt: input.now }) .where(issueMonitorClaimCondition(claimed)); }); @@ -827,6 +830,7 @@ export function createHeartbeatScheduling( } const consumed = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, claimed.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); return tx.update(issues) .set({ @@ -977,6 +981,7 @@ export function createHeartbeatScheduling( const staleClaimThreshold = new Date(now.getTime() - 5 * 60 * 1000); const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, issue.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) @@ -1050,6 +1055,7 @@ export function createHeartbeatScheduling( for (const due of dueMonitors) { if (!canContinue()) break; const claimed = await db.transaction(async (tx) => { + await notifyStatusCardIssueChange(tx, due.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); const [updated] = await tx .update(issues) diff --git a/server/src/services/heartbeat/task-context.ts b/server/src/services/heartbeat/task-context.ts index 85bdd65ca61..9c1fb122dd4 100644 --- a/server/src/services/heartbeat/task-context.ts +++ b/server/src/services/heartbeat/task-context.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import type { Db, agents, agentTaskSessions, heartbeatRuns, issueComments } from "@paperclipai/db"; +import type { Db, agents, agentTaskSessions, heartbeatRuns } from "@paperclipai/db"; import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared"; import type { PaperclipTurnContext } from "@paperclipai/adapter-utils/server-utils"; import { parseObject } from "../../adapters/utils.js"; @@ -15,8 +15,9 @@ import { getTaskPlanContext } from "../task-plan-context.js"; import { caseExecutionMarkdown, getCaseExecutionContext } from "../case-context.js"; import type { TrustPresetResolution } from "../trust-preset-resolver.js"; import { liveVoiceExecutionGuidance } from "../voice/voice-execution-guidance.js"; -import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; +import { buildPaperclipWakePayload, ConfigurationIncompleteFailure, readHeartbeatWakeComments, PAPERCLIP_WAKE_PAYLOAD_KEY, type createHeartbeatRunPreparation } from "./run-preparation.js"; import { buildPaperclipTaskMarkdown } from "./task-markdown.js"; +import { deriveCommentId, extractWakeCommentIds } from "../../modules/run-dispatch/index.js"; import { workspacePreferenceService } from "../workspace-preferences.js"; type RunPreparation = ReturnType; @@ -30,8 +31,6 @@ export interface HeartbeatTaskContextInput { "id" | "identifier" | "title" | "status" | "priority" | "workMode" | "conversationAgentId" | "description" | "projectId"> | null; issueContext: IssueContext | null; - wakeCommentContext: Pick | null; context: Record; taskSession: Pick | null; }; @@ -53,35 +52,9 @@ function readNonEmptyString(value: unknown): string | null { /** Rebuilds task prompt fields on the run's context; acquires no execution resources. */ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskContextInput) { const { run, agent } = input; - const { issueRef, issueContext, wakeCommentContext, context, taskSession } = input.task; + const { issueRef, issueContext, context, taskSession } = input.task; const { trustPreset, experimentalInstanceSettings } = input.policy; const { issuesSvc, getPinnedSkillTestContext } = input.services; - const continuationSummary = issueRef && !isConversation(issueContext) - ? await getIssueContinuationSummaryDocument(db, issueRef.id) - : null; - const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; - const safeContinuationSummary = - continuationSummary && !exposeLowTrustRaw - ? redactQuarantinedBodyForHigherTrust(continuationSummary) - : continuationSummary; - const safeWakeCommentContext = - wakeCommentContext && !exposeLowTrustRaw - ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) - : wakeCommentContext; - const issueAncestors = issueRef - ? await issuesSvc.getAncestors(issueRef.id) - : []; - if (continuationSummary) { - context.paperclipContinuationSummary = { - key: safeContinuationSummary!.key, - title: safeContinuationSummary!.title, - body: safeContinuationSummary!.body, - sourceTrust: safeContinuationSummary!.sourceTrust ?? null, - updatedAt: safeContinuationSummary!.updatedAt.toISOString(), - }; - } else { - delete context.paperclipContinuationSummary; - } const admittedScope = parseObject(context.paperclipAgentJwtScope); if (!input.policy.recovering || (admittedScope.kind !== "standard" && admittedScope.kind !== "skill_test")) { const savedIssue = parseObject(context.paperclipIssue); @@ -93,10 +66,58 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo : { kind: "standard" }; } const isSkillTest = parseObject(context.paperclipAgentJwtScope).kind === "skill_test"; - const pinnedSkillTestContext = - isSkillTest && issueRef - ? await getPinnedSkillTestContext(agent.companyId, issueRef.id) - : null; + const wakeCommentId = deriveCommentId(context, null); + const commentIds = [...new Set([...extractWakeCommentIds(context), ...(wakeCommentId ? [wakeCommentId] : [])])]; + const [summary, ancestors, pinnedSkillTestRead, wakeComments] = await Promise.allSettled([ + issueRef && !isConversation(issueContext) ? getIssueContinuationSummaryDocument(db, issueRef.id) : Promise.resolve(null), + issueRef ? issuesSvc.getAncestors(issueRef.id) : Promise.resolve([]), + isSkillTest && issueRef ? getPinnedSkillTestContext(agent.companyId, issueRef.id) : Promise.resolve(null), + readHeartbeatWakeComments(db, agent.companyId, readNonEmptyString(context.issueId), commentIds), + ]); + if (summary.status === "rejected") throw summary.reason; + if (ancestors.status === "rejected") throw ancestors.reason; + if (pinnedSkillTestRead.status === "rejected") throw pinnedSkillTestRead.reason; + if (wakeComments.status === "rejected") throw wakeComments.reason; + const wakeCommentRows = wakeComments.value; + const selectedWakeComment = issueRef && wakeCommentId ? wakeCommentRows.find(comment => comment.id === wakeCommentId && comment.issueId === issueRef.id) ?? null : null; + const wakeCommentContext = selectedWakeComment ? { + id: selectedWakeComment.id, + body: selectedWakeComment.deletedAt ? "" : selectedWakeComment.body, + authorType: selectedWakeComment.authorType, + authorAgentId: selectedWakeComment.authorAgentId, + authorUserId: selectedWakeComment.authorUserId, + presentation: selectedWakeComment.deletedAt ? null : selectedWakeComment.presentation, + metadata: selectedWakeComment.deletedAt ? null : selectedWakeComment.metadata, + deletedAt: selectedWakeComment.deletedAt, + deletedByType: selectedWakeComment.deletedByType, + deletedByAgentId: selectedWakeComment.deletedByAgentId, + deletedByUserId: selectedWakeComment.deletedByUserId, + deletedByRunId: selectedWakeComment.deletedByRunId, + sourceTrust: selectedWakeComment.sourceTrust, + } : null; + const continuationSummary = summary.value; + const issueAncestors = ancestors.value; + const pinnedSkillTestContext = pinnedSkillTestRead.value; + const exposeLowTrustRaw = trustPreset.kind === "low_trust_review"; + const safeContinuationSummary = + continuationSummary && !exposeLowTrustRaw + ? redactQuarantinedBodyForHigherTrust(continuationSummary) + : continuationSummary; + const safeWakeCommentContext = + wakeCommentContext && !exposeLowTrustRaw + ? sanitizeQuarantinedCommentForHigherTrust(wakeCommentContext) + : wakeCommentContext; + if (continuationSummary) { + context.paperclipContinuationSummary = { + key: safeContinuationSummary!.key, + title: safeContinuationSummary!.title, + body: safeContinuationSummary!.body, + sourceTrust: safeContinuationSummary!.sourceTrust ?? null, + updatedAt: safeContinuationSummary!.updatedAt.toISOString(), + }; + } else { + delete context.paperclipContinuationSummary; + } if (isSkillTest && !pinnedSkillTestContext) { throw new ConfigurationIncompleteFailure("The skill test's saved version is unavailable.", { configurationIncomplete: { reason: "skill_unavailable", actionUrl: "/skills" }, @@ -132,6 +153,7 @@ export async function prepareHeartbeatTaskContext(db: Db, input: HeartbeatTaskCo agentId: agent.id, runId: run.id, contextSnapshot: context, + wakeCommentRows, continuationSummary, issueSummary: issueRef ? { diff --git a/server/src/services/instance-settings.ts b/server/src/services/instance-settings.ts index 0208653a86f..e6f69242d2a 100644 --- a/server/src/services/instance-settings.ts +++ b/server/src/services/instance-settings.ts @@ -1,3 +1,6 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifySkillVersionRetentionReconstruction } from "./skill-version-retention-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; @@ -13,6 +16,7 @@ import { notifyInstructionCommit } from "./instruction-commit-notifications.js"; import { notifyScheduledRetryCancellation } from "./scheduled-retry-cancellation-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import type { Db } from "@paperclipai/db"; import { companies, instanceSettings } from "@paperclipai/db"; @@ -625,11 +629,15 @@ export function instanceSettingsService(db: Db, options: InstanceSettingsService } const now = new Date(); const [updated] = await db.transaction(async tx => { + if (patch.enableStatusCards !== undefined) await notifyStatusCardEvaluation(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.mcpEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, "*"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.grokDispatch); + if (patch.enableExternalObjects !== undefined) await notifyExternalObjectRefresh(tx); if (patch.enableWorktreeRunExecution !== undefined) await notifyScheduledRetryCancellation(tx); if (patch.enableWorktreeRunExecution !== undefined) { + await notifyRunDispatch(tx); + await notifyDeferredExecutionEligibility(tx); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalAgentSettlement(tx); await notifyInstructionCommit(tx); diff --git a/server/src/services/issue-comment-writes.ts b/server/src/services/issue-comment-writes.ts index e24e78e9855..32cd4407dd6 100644 --- a/server/src/services/issue-comment-writes.ts +++ b/server/src/services/issue-comment-writes.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import type { PgInsertValue } from "drizzle-orm/pg-core"; import { issueComments, type Db } from "@paperclipai/db"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -6,6 +8,9 @@ type WriteDb = Db | Parameters[0]>[0]; export function insertIssueComment(db: WriteDb, input: PgInsertValue & { companyId: string }) { return db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await notifyIssueRecovery(tx, { companyId: input.companyId, + issueId: typeof input.issueId === "string" ? input.issueId : undefined }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.dotEvents, `company:${input.companyId}`); return tx.insert(issueComments).values(input).returning(); }); diff --git a/server/src/services/issue-prefix.ts b/server/src/services/issue-prefix.ts index 5e31f9bd7ff..649a4aaae9b 100644 --- a/server/src/services/issue-prefix.ts +++ b/server/src/services/issue-prefix.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import type { PgColumn } from "drizzle-orm/pg-core"; import type { Db } from "@paperclipai/db"; @@ -127,6 +128,7 @@ export async function rekeyCompanyIssueIdentifiers( // overload, which returns NULL for a non-pattern argument. const tailStart = sql`${fromPrefix.length + 1}::int`; + await notifyStatusCardIssueChange(tx, companyId); const rekeyedIssues = await tx .update(issues) .set({ identifier: sql`${toPrefix} || substring(${issues.identifier} from ${tailStart})` }) diff --git a/server/src/services/issue-recovery-actions.ts b/server/src/services/issue-recovery-actions.ts index 5fb28bcb6d8..e276dc88f89 100644 --- a/server/src/services/issue-recovery-actions.ts +++ b/server/src/services/issue-recovery-actions.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -322,6 +324,8 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ): Promise { try { const created = await db.transaction(async (tx) => { + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionDisposition); await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionContinuation); @@ -392,7 +396,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], tx => tx.update(issueRecoveryActions).set({ nextAction: input.nextAction, updatedAt: now }) .where(and(eq(issueRecoveryActions.id, existing.id), inArray(issueRecoveryActions.status, [...ACTIVE_RECOVERY_ACTION_STATUSES]), - sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning()); + sql`${issueRecoveryActions.evidence}->'workspaceOwnerStop' is null`)).returning(), { companyId: input.companyId }); if (updated) return toReadModel(updated); } return existing; @@ -413,6 +417,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { Math.min(nextAttemptCount, effectiveMaxAttempts), ); const [exhausted] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -467,13 +472,14 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!exhausted) { return retryUpsertSourceScoped(input, retryCount); } return toReadModel(exhausted); } const [updated] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); if ((!input.preserveExistingOwner && existing.cause !== input.cause) || (input.evidence?.runId !== undefined && input.evidence.runId !== existing.evidence?.runId)) { @@ -539,7 +545,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { ), ) .returning(); - }); + }, { companyId: input.companyId }); if (!updated) { return retryUpsertSourceScoped(input, retryCount); } @@ -548,13 +554,14 @@ export function issueRecoveryActionService(db: DbOrTransaction) { try { const [created] = await writeExecutionControl(db, [DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.executionContinuation], async tx => { + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx .insert(issueRecoveryActions) .values(buildInsertValues(input, ownerType, now)) .returning(); - }); + }, { companyId: input.companyId }); return toReadModel(created!); } catch (error) { if (!isUniqueRecoveryActionConflict(error)) throw error; @@ -595,6 +602,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { const pending = await tx.select({ id: issueRecoveryActions.id }).from(issueRecoveryActions) .where(and(...predicates)).limit(1); if (!pending.length) return []; + await notifyIssueRecovery(tx, { companyId: input.companyId }); await notifyNativeRecoveryNotices(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.sourceIssueId }); return tx @@ -608,7 +616,7 @@ export function issueRecoveryActionService(db: DbOrTransaction) { }) .where(and(...predicates)) .returning(); - }); + }, { companyId: input.companyId }); return updated ? toReadModel(updated) : null; } diff --git a/server/src/services/issue-recovery-notifications.ts b/server/src/services/issue-recovery-notifications.ts new file mode 100644 index 00000000000..7b709960831 --- /dev/null +++ b/server/src/services/issue-recovery-notifications.ts @@ -0,0 +1,16 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** An issue hint is reserved for local facts such as comments. Lifecycle, + * graph, run and eligibility changes retain company scope because their + * consequences can include ancestors and shared execution references. */ +export type IssueRecoveryScope = { companyId: string; issueId?: string }; + +export async function notifyIssueRecovery(transaction: object, scope?: IssueRecoveryScope) { + const companyHint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; + const issueHint = scope ? JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId }) : "*"; + for (const queue of [DELIVERY_QUEUES.orphanBlockers, DELIVERY_QUEUES.activeRecoveryActions, DELIVERY_QUEUES.strandedAssignments, DELIVERY_QUEUES.resolvedDependencies, + DELIVERY_QUEUES.taskWatchdogs, DELIVERY_QUEUES.staleIssueLocks]) { + await notifyDeliveryWork(transaction, queue, + queue === DELIVERY_QUEUES.strandedAssignments || queue === DELIVERY_QUEUES.staleIssueLocks ? issueHint : companyHint); + } +} diff --git a/server/src/services/issue-review-state-notifications.ts b/server/src/services/issue-review-state-notifications.ts index 679b7e37475..d1dff39e1ee 100644 --- a/server/src/services/issue-review-state-notifications.ts +++ b/server/src/services/issue-review-state-notifications.ts @@ -1,9 +1,15 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeReviewResponse } from "./native-review-response-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; export async function notifyIssueReviewState(tx: object, scope: { companyId: string; issueId?: string; runId?: string; agentId?: string }) { + await notifyRoutineExecutionStatus(tx, scope.issueId, scope.companyId); + await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyRunDispatch(tx, scope); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyExternalConversationSettlement(tx, scope); await notifyNativeReviewResponse(tx, scope); diff --git a/server/src/services/issue-thread-interactions.test.ts b/server/src/services/issue-thread-interactions.test.ts index 24e1489006f..5ad905a8a10 100644 --- a/server/src/services/issue-thread-interactions.test.ts +++ b/server/src/services/issue-thread-interactions.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { getTableName } from "drizzle-orm"; +import { PgDialect } from "drizzle-orm/pg-core"; +import { getTableName, type SQL } from "drizzle-orm"; const mockCreateChild = vi.fn(); @@ -48,8 +49,18 @@ function createFakeDb(args: { const toolActionRequestUpdates: Array> = []; const inserts: Array<{ table: string; values: Record }> = []; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { + execute: vi.fn(async (query: SQL) => { + const statement = new PgDialect().sqlToQuery(query); + expect(transactionDepth).toBeGreaterThan(0); + expect(statement.params).toContain(interactionRow.companyId); + expect(statement.params).toContain(interactionRow.issueId); + if (statement.sql.includes("origin_kind")) return [{ id: interactionRow.issueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: interactionRow.issueId }]; + throw new Error(`Unexpected interaction fixture query: ${statement.sql}`); + }), select: vi.fn(() => { selectCallCount += 1; const rows = selectCallCount === 1 ? [interactionRow] : (args.parentRows ?? []); @@ -84,7 +95,10 @@ function createFakeDb(args: { inserts.push({ table: getTableName(table as never), values }); }, })), - transaction: async (callback: (tx: typeof db) => Promise) => callback(db), + transaction: async (callback: (tx: typeof db) => Promise) => { + transactionDepth++; + try { return await callback(db); } finally { transactionDepth--; } + }, }; return { @@ -177,6 +191,7 @@ describe("issueThreadInteractionService", () => { }; let selectCallCount = 0; + let transactionDepth = 0; const db: any = { select: vi.fn(() => { selectCallCount += 1; diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 730a82db3a6..29b181d9164 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,5 @@ +import { notifyMergeConfirmationWork, type MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyIssueReviewState } from "./issue-review-state-notifications.js"; import { notifyChatConfirmationResolution } from "./chat-confirmation-notifications.js"; @@ -203,6 +205,7 @@ export type IssueThreadInteractionServiceOptions = { resolvePullRequestState?: ( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ) => Promise; wakeup?: InteractionWakeup; pullRequestCacheTtlMs?: number; @@ -376,7 +379,7 @@ type ResolvedInteractionResult = { }; type IssueThreadInteractionRow = typeof issueThreadInteractions.$inferSelect; -type IssueTouchDb = Pick; +type IssueTouchDb = Pick; function isNativeCompletionReview( row: Pick, @@ -501,6 +504,7 @@ type IssueResolutionContext = { companyId: string; status: string; executionRunId: string | null; + hiddenAt?: Date | null; workMode: string; assigneeAgentId: string | null; assigneeUserId: string | null; @@ -801,11 +805,11 @@ function hydrateInteraction( } } -async function touchIssue(db: IssueTouchDb, issueId: string) { - await db - .update(issues) - .set({ updatedAt: new Date() }) - .where(eq(issues.id, issueId)); +async function touchIssue(db: IssueTouchDb, issueId: string, companyId: string) { + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx.update(issues).set({ updatedAt: new Date() }).where(eq(issues.id, issueId)); + }); } function isTerminalIssueStatus(status: string) { @@ -2013,7 +2017,7 @@ async function expireStaleRequestConfirmationTarget( "Interaction has already been resolved", ); } - await touchIssue(tx, args.row.issueId); + await touchIssue(tx, args.row.issueId, args.row.companyId); const expired = hydrateInteraction(updated); await enqueueTerminalIssueInteractionChatPublications(tx, expired); return expired; @@ -2043,10 +2047,11 @@ export function issueThreadInteractionService( async function resolvePullRequestState( companyId: string, reference: GitHubPullRequestReference, + signal?: AbortSignal, ): Promise { if (opts.resolvePullRequestState) - return opts.resolvePullRequestState(companyId, reference); - return defaultPullRequestStateResolver?.(companyId, reference) ?? "unknown"; + return opts.resolvePullRequestState(companyId, reference, signal); + return defaultPullRequestStateResolver?.(companyId, reference, signal) ?? "unknown"; } async function resolvePullRequestStates( @@ -2055,6 +2060,7 @@ export function issueThreadInteractionService( companyId: string; reference: GitHubPullRequestReference; }>, + signal?: AbortSignal, ) { const states = new Map(); const pending = entries.slice(); @@ -2062,11 +2068,13 @@ export function issueThreadInteractionService( await Promise.all( Array.from({ length: workerCount }, async () => { while (pending.length > 0) { + signal?.throwIfAborted(); const entry = pending.shift(); if (!entry) return; const state = await resolvePullRequestState( entry.companyId, entry.reference, + signal, ); states.set(entry.key, state); } @@ -2074,6 +2082,38 @@ export function issueThreadInteractionService( ); return states; } + async function mergedPullRequestConfirmationPage(scope?: MergeConfirmationScope, after?: string) { + const rows = await db.select({ interaction: issueThreadInteractions, issue: { + id: issues.id, companyId: issues.companyId, projectId: issues.projectId, goalId: issues.goalId, + status: issues.status, assigneeAgentId: issues.assigneeAgentId, + } }).from(issueThreadInteractions).innerJoin(issues, and( + eq(issueThreadInteractions.issueId, issues.id), eq(issueThreadInteractions.companyId, issues.companyId), + )).where(and( + eq(issueThreadInteractions.kind, "request_confirmation"), eq(issueThreadInteractions.status, "pending"), + isNull(issues.hiddenAt), sql`${issues.status} NOT IN ('done', 'cancelled')`, + scope ? eq(issueThreadInteractions.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(issueThreadInteractions.issueId, scope.issueId) : undefined, + scope?.interactionId ? eq(issueThreadInteractions.id, scope.interactionId) : undefined, + after ? sql`${issueThreadInteractions.id} > ${after}::uuid` : undefined, + sql`NOT EXISTS (SELECT 1 FROM ${toolActionRequests} WHERE ${toolActionRequests.interactionId} = ${issueThreadInteractions.id})`, + )).orderBy(asc(issueThreadInteractions.id)).limit(100); + return { rows: rows.flatMap(row => { + const references = getMergeConfirmationPullRequestReferences(row.interaction); + return references.length ? [{ ...row, references }] : []; + }), after: rows.at(-1)?.interaction.id, complete: rows.length < 100 }; + } + + async function mergedPullRequestConfirmationCandidates(scope?: MergeConfirmationScope) { + const candidates: Awaited>["rows"] = []; + let after: string | undefined; + while (true) { + const page = await mergedPullRequestConfirmationPage(scope, after); + candidates.push(...page.rows); + if (page.complete) return candidates; + after = page.after; + } + } + async function getIdempotentInteraction(args: { issueId: string; companyId: string; @@ -2271,6 +2311,7 @@ export function issueThreadInteractionService( companyId: issues.companyId, status: issues.status, executionRunId: issues.executionRunId, + hiddenAt: issues.hiddenAt, workMode: issues.workMode, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -2309,6 +2350,10 @@ export function issueThreadInteractionService( "Interaction has already been resolved", ); } + if (args.actor.systemId === "system:pr-merged" && + (issueContext.hiddenAt || getMergeConfirmationPullRequestReferences(lockedCurrent).length === 0)) { + throw conflict("Merge confirmation is no longer actionable"); + } await assertRequestConfirmationResolutionAllowedUnderLock( tx as unknown as Db, issueContext, @@ -2468,7 +2513,7 @@ export function issueThreadInteractionService( }; } } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } // Snapshot the continuation target only after every lifecycle mutation @@ -2682,7 +2727,7 @@ export function issueThreadInteractionService( tx, ); } else { - await touchIssue(tx, args.issue.id); + await touchIssue(tx, args.issue.id, args.issue.companyId); } await enqueueTerminalIssueInteractionChatPublications( tx as unknown as Db, @@ -2864,7 +2909,7 @@ export function issueThreadInteractionService( await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(previous)); } } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); inserted = true; await enqueueIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(row)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.connection); @@ -2908,7 +2953,7 @@ export function issueThreadInteractionService( ) .returning(); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); return hydrateInteraction(updated) as ConnectionIntentInteraction; }, resolveConnectionIntent: async ( @@ -2972,13 +3017,15 @@ export function issueThreadInteractionService( return row; }); if (!updated) throw interactionAlreadyResolvedError(); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const interaction = hydrateInteraction( updated, ) as ConnectionIntentInteraction; await emitInteractionResolvedTelemetry(db, interaction); return interaction; }, + mergedPullRequestConfirmationPage, + mergedPullRequestConfirmationCandidates, sweepMergedPullRequestConfirmations: async ( mergedHints: Array<{ companyId: string; @@ -2986,53 +3033,21 @@ export function issueThreadInteractionService( repo: string; number: number; }> = [], + scope?: MergeConfirmationScope, + signal?: AbortSignal, mutationOptions: InteractionResolutionMutationOptions = {}, ) => { - const rows = await db - .select({ - interaction: issueThreadInteractions, - issue: { - id: issues.id, - companyId: issues.companyId, - projectId: issues.projectId, - goalId: issues.goalId, - status: issues.status, - assigneeAgentId: issues.assigneeAgentId, - }, - }) - .from(issueThreadInteractions) - .innerJoin(issues, eq(issueThreadInteractions.issueId, issues.id)) - .where( - and( - eq(issueThreadInteractions.kind, "request_confirmation"), - eq(issueThreadInteractions.status, "pending"), - mergedHints.length ? inArray(issues.companyId, [...new Set(mergedHints.map(hint => hint.companyId))]) : undefined, - ), - ); - - const candidates = rows.flatMap((row) => { - const references = getMergeConfirmationPullRequestReferences( - row.interaction, - ); - return references.length > 0 ? [{ ...row, references }] : []; - }); - if (candidates.length === 0) { - return { checked: rows.length, candidates: 0, accepted: 0, woken: 0 }; - } - - const candidateIds = candidates.map(({ interaction }) => interaction.id); - const linkedToolActions = await db - .select({ interactionId: toolActionRequests.interactionId }) - .from(toolActionRequests) - .where(inArray(toolActionRequests.interactionId, candidateIds)); - const toolActionInteractionIds = new Set( - linkedToolActions - .map((row) => row.interactionId) - .filter((value): value is string => Boolean(value)), - ); - const eligible = candidates.filter( - ({ interaction }) => !toolActionInteractionIds.has(interaction.id), - ); + const scopes = scope ? [scope] : mergedHints.length + ? [...new Set(mergedHints.map(hint => hint.companyId))].map(companyId => ({ companyId })) + : [undefined]; + const candidates: Awaited> = []; + for (const candidateScope of scopes) candidates.push(...await mergedPullRequestConfirmationCandidates(candidateScope)); + const eligible = mergedHints.length ? candidates.filter(candidate => mergedHints.some(hint => + hint.companyId === candidate.issue.companyId && candidate.references.some(reference => + hint.owner.toLowerCase() === reference.owner.toLowerCase() && + hint.repo.toLowerCase() === reference.repo.toLowerCase() && hint.number === reference.number), + )) : candidates; + if (eligible.length === 0) return { checked: 0, candidates: 0, accepted: 0, woken: 0 }; const checkedAt = now().getTime(); const cacheTtlMs = opts.pullRequestCacheTtlMs ?? 5 * 60 * 1000; @@ -3066,7 +3081,10 @@ export function issueThreadInteractionService( const refreshedStates = await resolvePullRequestStates([ ...uniqueReferences.values(), - ]); + ], signal); + if (scope && [...refreshedStates.values()].some(state => state === "unknown")) { + throw new Error("Merge confirmation provider state is unavailable"); + } for (const [key, state] of refreshedStates) { setBoundedPullRequestCacheEntry(pullRequestStateCache, key, { state, @@ -3077,6 +3095,7 @@ export function issueThreadInteractionService( let accepted = 0; let woken = 0; for (const candidate of eligible) { + if (signal?.aborted) break; const allMerged = candidate.references.every((reference) => { const key = `${candidate.issue.companyId}:${reference.owner.toLowerCase()}/${reference.repo.toLowerCase()}#${reference.number}`; return pullRequestStateCache.get(key)?.state === "merged"; @@ -3114,12 +3133,16 @@ export function issueThreadInteractionService( if (resolved.interaction.status !== "accepted") continue; accepted += 1; - const wakeIssue = resolved.continuationIssue ?? candidate.issue; + const wakeIssue = resolved.continuationIssue ?? await db.select({ + id: issues.id, assigneeAgentId: issues.assigneeAgentId, status: issues.status, + }).from(issues).where(and(eq(issues.id, candidate.issue.id), + eq(issues.companyId, candidate.issue.companyId), isNull(issues.hiddenAt))).then(rows => rows[0]); const shouldWake = resolved.interaction.continuationPolicy === "wake_assignee" || resolved.interaction.continuationPolicy === "wake_assignee_on_accept"; if ( !opts.wakeup || + !wakeIssue || !shouldWake || !wakeIssue.assigneeAgentId || isTerminalIssueStatus(wakeIssue.status) @@ -3164,7 +3187,7 @@ export function issueThreadInteractionService( } return { - checked: rows.length, + checked: candidates.length, candidates: eligible.length, accepted, woken, @@ -3357,7 +3380,7 @@ export function issueThreadInteractionService( }) .where(eq(issueThreadInteractions.id, current.id)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); return row; }); return hydrateInteraction(updated); @@ -3420,10 +3443,8 @@ export function issueThreadInteractionService( if (updated) cancelled.push(updated); } - for (const issueId of new Set( - cancelled.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(cancelled.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, cancelled); return cancelled; @@ -3512,10 +3533,8 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - for (const issueId of new Set( - expired.map((interaction) => interaction.issueId), - )) { - await touchIssue(db, issueId); + for (const [issueId, companyId] of new Map(expired.map(interaction => [interaction.issueId, interaction.companyId]))) { + await touchIssue(db, issueId, companyId); } await emitResolvedInteractionsTelemetry(db, expired); } @@ -3915,6 +3934,7 @@ export function issueThreadInteractionService( } else { await notifyNativeStatusReassessment(tx, { companyId: issue.companyId, issueId: issue.id }); } + if (data.kind === "request_confirmation") await notifyMergeConfirmationWork(tx, { companyId: issue.companyId, issueId: issue.id }); const [row] = await tx .insert(issueThreadInteractions) .values({ @@ -4058,7 +4078,7 @@ export function issueThreadInteractionService( return interaction; } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); if (superseded.length > 0) { await emitResolvedInteractionsTelemetry( db, @@ -4363,7 +4383,7 @@ export function issueThreadInteractionService( .where(eq(issueThreadInteractions.id, interactionId)) .returning(); - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); current.status = updated.status; @@ -4555,7 +4575,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(tx, issue.id); + await touchIssue(tx, issue.id, issue.companyId); await mutationOptions.afterResolveInTransaction?.(tx, hydrateInteraction(updated)); if (complete) await enqueueTerminalIssueInteractionChatPublications(tx as unknown as Db, hydrateInteraction(updated)); return { @@ -4628,7 +4648,7 @@ export function issueThreadInteractionService( throw interactionAlreadyResolvedError(); } - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const rejected = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, rejected); return rejected; @@ -4726,7 +4746,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -4958,7 +4978,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5055,7 +5075,7 @@ export function issueThreadInteractionService( } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5195,7 +5215,7 @@ export function issueThreadInteractionService( if (updated) expired.push(hydrateInteraction(updated)); } if (expired.length > 0) { - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); await emitResolvedInteractionsTelemetry(db, expired); } return expired; @@ -5300,7 +5320,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const withdrawn = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, withdrawn); return withdrawn; @@ -5423,7 +5443,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const answered = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, answered); return answered; @@ -5559,7 +5579,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const skipped = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, skipped); return skipped; @@ -5636,7 +5656,7 @@ export function issueThreadInteractionService( return row; }); - await touchIssue(db, issue.id); + await touchIssue(db, issue.id, issue.companyId); const cancelled = hydrateInteraction(updated); await emitInteractionResolvedTelemetry(db, cancelled); return cancelled; diff --git a/server/src/services/issue-title.ts b/server/src/services/issue-title.ts index af6716e3aa4..4894a7b070e 100644 --- a/server/src/services/issue-title.ts +++ b/server/src/services/issue-title.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { setIssueTitleSchema, type SetIssueTitle } from "@paperclipai/shared"; @@ -76,6 +78,8 @@ export async function setIssueTitle( || (issue.title === title && !issue.titleNeedsGeneration)) { return { result: await saveReceipt({ id: issue.id, title: issue.title, titleNeedsGeneration: issue.titleNeedsGeneration, changed: false }), publication: null }; } + await notifyAttentionRetentionWork(tx, { companyId }); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ title, titleNeedsGeneration: false, updatedAt: new Date() }).where(predicate); await issueReferenceService(tx).syncIssue(issueId, tx); const { publication } = await persistActivity(tx, { diff --git a/server/src/services/issue-tree-control.ts b/server/src/services/issue-tree-control.ts index 0d7633f846a..3f2cf140237 100644 --- a/server/src/services/issue-tree-control.ts +++ b/server/src/services/issue-tree-control.ts @@ -1,3 +1,7 @@ +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; @@ -758,6 +762,8 @@ export function issueTreeControlService(db: Db) { const releaseReason = input.reason ?? "Subtree resume applied."; const { hold: resumeHold } = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -836,6 +842,8 @@ export function issueTreeControlService(db: Db) { } const { hold, members } = await db.transaction(async (tx) => { + await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); const [createdHold] = await tx .insert(issueTreeHolds) .values({ @@ -1000,17 +1008,20 @@ export function issueTreeControlService(db: Db) { const now = new Date(); const releasedCancelHoldIds = activeCancelHolds.map((hold) => hold.id); const updatedIssues = await db.transaction(async (tx) => { + await notifyDeferredExecutionEligibility(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); const restored: TreeStatusUpdateResult["updatedIssues"] = []; for (const [status, issueIdsForStatus] of issueIdsByStatus) { if (issueIdsForStatus.length === 0) continue; for (const issueId of issueIdsForStatus) { + await notifyRoutineExecutionStatus(tx, issueId, companyId); await notifyNativeStatusReassessment(tx, { companyId, issueId }); await notifyTerminalWorkspaceWork(tx, `issue:${issueId}`); await notifyChatResponseAvailability(tx, { companyId, issueId }); if (status === "blocked" || status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); } + await notifyStatusCardIssueChange(tx, companyId); const rows = await tx .update(issues) .set({ @@ -1177,7 +1188,10 @@ export function issueTreeControlService(db: Db) { throw conflict("Issue tree hold is already released"); } - const [updated] = await db + const [updated] = await db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId }); + return tx .update(issueTreeHolds) .set({ status: "released", @@ -1195,6 +1209,7 @@ export function issueTreeControlService(db: Db) { }) .where(and(eq(issueTreeHolds.id, holdId), eq(issueTreeHolds.companyId, companyId))) .returning(); + }); const members = await db .select() diff --git a/server/src/services/issue-visibility.ts b/server/src/services/issue-visibility.ts index 44a04a089e6..b54832f29ee 100644 --- a/server/src/services/issue-visibility.ts +++ b/server/src/services/issue-visibility.ts @@ -1,5 +1,5 @@ import { nonIdleSlackIssueCondition } from "./slack-conversation-state.js"; -import { and, isNull, type SQL } from "drizzle-orm"; +import { and, isNull, isNotNull, or, type SQL } from "drizzle-orm"; import { issues } from "@paperclipai/db"; export function visibleIssueCondition(): SQL { @@ -14,3 +14,9 @@ export function visibleIssueSql(alias = "issues") { export function executionIssueCondition(): SQL { return and(visibleIssueCondition(), isNull(issues.conversationAgentId), nonIdleSlackIssueCondition())!; } + +export function attentionIssueCondition(includeConversations = false): SQL { + return includeConversations + ? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))! + : executionIssueCondition(); +} diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 42c65bb5d02..569a386d3f9 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,12 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; +import { notifyMergeConfirmationWork } from "./merged-pr-confirmation-notifications.js"; +import { notifyRoutineExecutionStatus } from "./routine-execution-status-notifications.js"; +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { notifyNativeRecoveryNotices } from "./native-recovery-notice-notifications.js"; import { notifyNativeRunProjection } from "./native-run-projection-notifications.js"; @@ -2537,33 +2546,37 @@ async function listPendingFinalizeBlockerIssueIds( ), ); - if (lockSources) { + const rows: Array<{ issueId: string | null; executionWorkspaceId: string | null; + phase: string; status: string; startedAt: Date }> = []; + for (const chunk of chunkList(executionWorkspaceIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + if (lockSources) { - await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) - .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, executionWorkspaceIds))) - .for("update", { noWait: true }); - } - const query = dbOrTx - .select({ - issueId: workspaceOperations.issueId, - executionWorkspaceId: workspaceOperations.executionWorkspaceId, - phase: workspaceOperations.phase, - status: workspaceOperations.status, - startedAt: workspaceOperations.startedAt, - }) - .from(workspaceOperations) - .where( - and( - eq(workspaceOperations.companyId, companyId), - inArray( - workspaceOperations.executionWorkspaceId, - executionWorkspaceIds, + await dbOrTx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) + .where(and(eq(executionWorkspaces.companyId, companyId), inArray(executionWorkspaces.id, chunk))) + .for("update", { noWait: true }); + } + const query = dbOrTx + .select({ + issueId: workspaceOperations.issueId, + executionWorkspaceId: workspaceOperations.executionWorkspaceId, + phase: workspaceOperations.phase, + status: workspaceOperations.status, + startedAt: workspaceOperations.startedAt, + }) + .from(workspaceOperations) + .where( + and( + eq(workspaceOperations.companyId, companyId), + inArray( + workspaceOperations.executionWorkspaceId, + chunk, + ), ), - ), - ); + ); - const rows = await (lockSources ? query.for("share", { noWait: true }) : query); + rows.push(...await (lockSources ? query.for("share", { noWait: true }) : query)); + } const latestAttributedByBlockerWorkspace = new Map< string, @@ -3431,6 +3444,7 @@ type IssueBlockerAttentionEdge = { blockerIssueId: string; }; type IssueBlockerAttentionQueryRow = IssueBlockerAttentionNode & { + executionWorkspaceId?: string | null; issueId: string | null; blockerIssueId: string; }; @@ -3819,11 +3833,15 @@ async function terminalExplicitBlockersByRoot( return terminalByRoot; } -async function listIssueBlockerAttentionMap( +async function listIssueBlockerAttentionMap(dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[]) { + return (await readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows)).attention; +} + +async function readIssueBlockerAttentionBatch( dbOrTx: any, companyId: string, issueRows: IssueBlockerAttentionInputNode[], -): Promise> { +) { const roots = issueRows.filter( (row) => row.companyId === companyId && row.status === "blocked", ); @@ -3833,36 +3851,42 @@ async function listIssueBlockerAttentionMap( attentionMap.set(row.id, createIssueBlockerAttention()); } } - if (roots.length === 0) return attentionMap; + const deferredRootIds = new Set(roots.slice(BLOCKER_ATTENTION_MAX_NODES).map(root => root.id)); + let edgeCount = 0, fetchedRows = 0; + const result = () => ({ attention: attentionMap, deferredRootIds, nodeCount: nodesById.size, edgeCount, fetchedRows }); + if (roots.length === 0) return { attention: attentionMap, deferredRootIds, nodeCount: 0, edgeCount, fetchedRows }; const nodesById = new Map(); const edgesByIssueId = new Map(); - for (const root of roots) nodesById.set(root.id, { ...root }); + // Returned rows are already bounded by the residual fetch budget. Keep their + // positive live-work evidence even when they cannot enter the traversal graph. + const observedNodes = new Map(); + const observedChildrenByParent = new Map>(); + const admittedRoots = roots.slice(0, BLOCKER_ATTENTION_MAX_NODES); + for (const root of admittedRoots) nodesById.set(root.id, { ...root }); - let frontier = roots.map((root) => root.id); - let truncated = false; + type Traversal = { rootId: string; nodes: Set; expanded: Set; frontier: string[]; truncated: boolean }; + const traversals = new Map(admittedRoots.map(root => [root.id, { + rootId: root.id, nodes: new Set([root.id]), expanded: new Set(), frontier: [root.id], truncated: false, + }])); + const expanded = new Set(); + const queried = new Set(); const pendingFinalizeBlockerIssueIds = new Set(); - for ( - let depth = 0; - frontier.length > 0 && depth < BLOCKER_ATTENTION_MAX_DEPTH; - depth += 1 - ) { - const nextFrontier = new Set(); + for (let depth = 0; depth < BLOCKER_ATTENTION_MAX_DEPTH; depth += 1) { + const active = [...traversals.values()].filter(traversal => !traversal.truncated && traversal.frontier.length > 0); + if (active.length === 0) break; + const unread = new Set(); + for (const traversal of active) for (const id of traversal.frontier) if (!expanded.has(id)) unread.add(id); + const frontier = [...unread]; for (const chunk of chunkList( - [...new Set(frontier)], + frontier, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE, )) { - const readinessByIssueId = await listIssueDependencyReadinessMap( - dbOrTx, - companyId, - chunk, - ); - for (const readiness of readinessByIssueId.values()) { - for (const blockerIssueId of readiness.pendingFinalizeBlockerIssueIds) { - pendingFinalizeBlockerIssueIds.add(blockerIssueId); - } - } + const remaining = Math.min(BLOCKER_ATTENTION_MAX_NODES - nodesById.size, + BLOCKER_ATTENTION_MAX_NODES * 2 - edgeCount); + if (remaining <= 0) break; + const rowLimit = remaining + 1; const explicitBlockerRowsPromise: Promise< IssueBlockerAttentionQueryRow[] > = dbOrTx @@ -3876,6 +3900,7 @@ async function listIssueBlockerAttentionMap( title: issues.title, status: issues.status, executionRunId: issues.executionRunId, + executionWorkspaceId: issues.executionWorkspaceId, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, }) @@ -3888,7 +3913,7 @@ async function listIssueBlockerAttentionMap( inArray(issueRelations.relatedIssueId, chunk), eq(issues.companyId, companyId), ), - ); + ).limit(rowLimit); const childRowsPromise: Promise = dbOrTx .select({ issueId: issues.parentId, @@ -3913,68 +3938,83 @@ async function listIssueBlockerAttentionMap( BLOCKER_ATTENTION_CHILD_TERMINAL_STATUSES, ), ), - ); + ).limit(rowLimit); const [explicitBlockerRows, childRows] = await Promise.all([ explicitBlockerRowsPromise, childRowsPromise, ]); + for (const id of chunk) queried.add(id); + fetchedRows += explicitBlockerRows.length + childRows.length; + const pendingFinalize = await listPendingFinalizeBlockerIssueIds(dbOrTx, companyId, + explicitBlockerRows.flatMap(row => row.status === "done" && row.executionWorkspaceId + ? [{ blockerIssueId: row.blockerIssueId, executionWorkspaceId: row.executionWorkspaceId }] : [])); + for (const id of pendingFinalize) pendingFinalizeBlockerIssueIds.add(id); + let complete = explicitBlockerRows.length < rowLimit && childRows.length < rowLimit; const unresolvedExplicitBlockerRows = explicitBlockerRows.filter( (row) => row.status !== "done" || pendingFinalizeBlockerIssueIds.has(row.blockerIssueId), ); - appendBlockerAttentionEdges(edgesByIssueId, [ - ...unresolvedExplicitBlockerRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ...childRows - .filter( - (row): row is IssueBlockerAttentionQueryRow & { issueId: string } => - row.issueId !== null, - ) - .map((row) => ({ - issueId: row.issueId, - blockerIssueId: row.blockerIssueId, - })), - ]); - for (const row of [...unresolvedExplicitBlockerRows, ...childRows]) { - if (!row.issueId || nodesById.has(row.blockerIssueId)) continue; - nodesById.set(row.blockerIssueId, { - id: row.blockerIssueId, - companyId: row.companyId, - parentId: row.parentId, - identifier: row.identifier, - title: row.title, - status: row.status, - executionRunId: row.executionRunId, - assigneeAgentId: row.assigneeAgentId, - assigneeUserId: row.assigneeUserId, - }); - nextFrontier.add(row.blockerIssueId); + if (!row.issueId) continue; + observedNodes.set(row.blockerIssueId, row); + const observed = observedChildrenByParent.get(row.issueId) ?? new Set(); + observed.add(row.blockerIssueId); + observedChildrenByParent.set(row.issueId, observed); + if (!nodesById.has(row.blockerIssueId)) { + if (nodesById.size >= BLOCKER_ATTENTION_MAX_NODES) { complete = false; continue; } + nodesById.set(row.blockerIssueId, { + id: row.blockerIssueId, companyId: row.companyId, parentId: row.parentId, identifier: row.identifier, + title: row.title, status: row.status, executionRunId: row.executionRunId, + assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, + }); + } + const edges = edgesByIssueId.get(row.issueId) ?? []; + if (edges.some(edge => edge.blockerIssueId === row.blockerIssueId)) continue; + if (edgeCount >= BLOCKER_ATTENTION_MAX_NODES * 2) { complete = false; continue; } + edges.push({ issueId: row.issueId, blockerIssueId: row.blockerIssueId }); + edgesByIssueId.set(row.issueId, edges); + edgeCount++; } + if (complete) for (const id of chunk) expanded.add(id); + } - if (nodesById.size > BLOCKER_ATTENTION_MAX_NODES) { - truncated = true; - break; + for (const traversal of active) { + if (traversal.frontier.some(id => !expanded.has(id))) { + for (const id of traversal.frontier) if (queried.has(id)) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) if (nodesById.has(edge.blockerIssueId)) traversal.nodes.add(edge.blockerIssueId); + } + traversal.truncated = true; + traversal.frontier = []; + deferredRootIds.add(traversal.rootId); + continue; + } + const nextFrontier = new Set(); + for (const id of traversal.frontier) { + traversal.expanded.add(id); + for (const edge of edgesByIssueId.get(id) ?? []) { + if (traversal.nodes.has(edge.blockerIssueId) || !nodesById.has(edge.blockerIssueId)) continue; + traversal.nodes.add(edge.blockerIssueId); + nextFrontier.add(edge.blockerIssueId); + } + } + traversal.truncated = traversal.nodes.size > BLOCKER_ATTENTION_MAX_NODES; + traversal.frontier = traversal.truncated ? [] : [...nextFrontier]; } - frontier = [...nextFrontier]; } - if (frontier.length > 0) truncated = true; + for (const traversal of traversals.values()) { + if (traversal.frontier.length > 0) traversal.truncated = true; + } - const nodeIds = [...nodesById.keys()]; + const evidenceNodes = new Map([...nodesById, ...observedNodes]); + const nodeIds = [...evidenceNodes.keys()]; const activeIssueIds = new Set(); const agentIds = new Set(); const issueIdByExecutionRunId = new Map(); - for (const node of nodesById.values()) { + for (const node of evidenceNodes.values()) { if (node.assigneeAgentId) agentIds.add(node.assigneeAgentId); if (node.executionRunId) issueIdByExecutionRunId.set(node.executionRunId, node.id); @@ -4032,7 +4072,7 @@ async function listIssueBlockerAttentionMap( } } - const explicitWaitCandidateIds = [...nodesById.values()] + const explicitWaitCandidateIds = [...evidenceNodes.values()] .filter((node) => node.status !== "done") .map((node) => node.id); const explicitWaitingIssueIds = new Set(); @@ -4107,23 +4147,26 @@ async function listIssueBlockerAttentionMap( ownerType: string; ownerAgentId: string | null; ownerUserId: string | null; - }> = await dbOrTx - .select({ - id: issueRecoveryActions.id, - sourceIssueId: issueRecoveryActions.sourceIssueId, - status: issueRecoveryActions.status, - ownerType: issueRecoveryActions.ownerType, - ownerAgentId: issueRecoveryActions.ownerAgentId, - ownerUserId: issueRecoveryActions.ownerUserId, - }) - .from(issueRecoveryActions) - .where( - and( - eq(issueRecoveryActions.companyId, companyId), - inArray(issueRecoveryActions.status, ["active", "escalated"]), - inArray(issueRecoveryActions.sourceIssueId, explicitWaitCandidateIds), - ), - ); + }> = []; + for (const chunk of chunkList(explicitWaitCandidateIds, ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + recoveryActionRows.push(...await dbOrTx + .select({ + id: issueRecoveryActions.id, + sourceIssueId: issueRecoveryActions.sourceIssueId, + status: issueRecoveryActions.status, + ownerType: issueRecoveryActions.ownerType, + ownerAgentId: issueRecoveryActions.ownerAgentId, + ownerUserId: issueRecoveryActions.ownerUserId, + }) + .from(issueRecoveryActions) + .where( + and( + eq(issueRecoveryActions.companyId, companyId), + inArray(issueRecoveryActions.status, ["active", "escalated"]), + inArray(issueRecoveryActions.sourceIssueId, chunk), + ), + )); + } const recoveryActionIds = recoveryActionRows.map((row) => row.id); const liveRecoveryActionIds = new Set(); for (const chunk of chunkList( @@ -4186,22 +4229,11 @@ async function listIssueBlockerAttentionMap( } } - const agentRows: IssueBlockerAttentionAgentRow[] = - agentIds.size > 0 - ? await dbOrTx - .select({ - id: agents.id, - companyId: agents.companyId, - status: agents.status, - }) - .from(agents) - .where( - and( - eq(agents.companyId, companyId), - inArray(agents.id, [...agentIds]), - ), - ) - : []; + const agentRows: IssueBlockerAttentionAgentRow[] = []; + for (const chunk of chunkList([...agentIds], ISSUE_LIST_RELATED_QUERY_CHUNK_SIZE)) { + agentRows.push(...await dbOrTx.select({ id: agents.id, companyId: agents.companyId, status: agents.status }) + .from(agents).where(and(eq(agents.companyId, companyId), inArray(agents.id, chunk)))); + } const agentsById = new Map(agentRows.map((agent) => [agent.id, agent])); type PathClassification = { @@ -4214,6 +4246,7 @@ async function listIssueBlockerAttentionMap( const classifyPath = ( nodeId: string, seen: Set, + truncated: boolean, ): PathClassification => { const sample = blockerSampleIdentifier(nodesById.get(nodeId)); if (truncated || seen.has(nodeId)) { @@ -4318,7 +4351,7 @@ async function listIssueBlockerAttentionMap( const nextSeen = new Set(seen); nextSeen.add(nodeId); const classified = downstream.map((edge) => - classifyPath(edge.blockerIssueId, nextSeen), + classifyPath(edge.blockerIssueId, nextSeen, truncated), ); const stalledChild = classified.find( (result) => result.stalled || result.sampleStalledBlockerIdentifier, @@ -4384,18 +4417,22 @@ async function listIssueBlockerAttentionMap( }; }; - const pathHasLiveWork = (nodeId: string, seen: Set): boolean => { - if (seen.has(nodeId)) return false; + const nodeHasLiveWork = (node: IssueBlockerAttentionNode) => + node.status === "in_progress" || activeIssueIds.has(node.id) || + explicitWaitingIssueIds.has(node.id) || Boolean(node.assigneeUserId); + const observedChildHasLiveWork = (parentId: string) => + [...observedChildrenByParent.get(parentId) ?? []].some(id => { + const node = observedNodes.get(id); + return node?.companyId === companyId && nodeHasLiveWork(node); + }); + const pathHasLiveWork = (nodeId: string, seen: Set, traversal: Traversal): boolean => { + if (seen.has(nodeId) || !traversal.nodes.has(nodeId)) return false; const node = nodesById.get(nodeId); if (!node || node.companyId !== companyId) return false; - if ( - node.status === "in_progress" || - activeIssueIds.has(node.id) || - explicitWaitingIssueIds.has(node.id) || - Boolean(node.assigneeUserId) - ) - return true; + if (nodeHasLiveWork(node)) return true; + if (!traversal.expanded.has(nodeId)) return false; + if (observedChildHasLiveWork(nodeId)) return true; const nextSeen = new Set(seen); nextSeen.add(nodeId); return (edgesByIssueId.get(node.id) ?? []).some((edge) => { @@ -4405,7 +4442,7 @@ async function listIssueBlockerAttentionMap( !pendingFinalizeBlockerIssueIds.has(edge.blockerIssueId) ) return false; - return pathHasLiveWork(edge.blockerIssueId, nextSeen); + return pathHasLiveWork(edge.blockerIssueId, nextSeen, traversal); }); }; @@ -4418,6 +4455,7 @@ async function listIssueBlockerAttentionMap( }; for (const root of roots) { + const traversal = traversals.get(root.id)!; const topLevelEdges = (edgesByIssueId.get(root.id) ?? []).filter((edge) => { const blocker = nodesById.get(edge.blockerIssueId); return ( @@ -4439,7 +4477,7 @@ async function listIssueBlockerAttentionMap( const classified = topLevelEdges.map((edge) => ({ edge, - result: classifyPath(edge.blockerIssueId, new Set([root.id])), + result: classifyPath(edge.blockerIssueId, new Set([root.id]), traversal.truncated), })); const coveredBlockerCount = classified.filter( (entry) => entry.result.covered, @@ -4521,8 +4559,8 @@ async function listIssueBlockerAttentionMap( stalledEntry?.result.sampleStalledBlockerIdentifier ?? sampleStalledFromChain ?? null, - blockingTreeLive: topLevelEdges.some((edge) => - pathHasLiveWork(edge.blockerIssueId, new Set([root.id])), + blockingTreeLive: observedChildHasLiveWork(root.id) || topLevelEdges.some((edge) => + pathHasLiveWork(edge.blockerIssueId, new Set([root.id]), traversal), ), directBlockerIssueId: sampleEntry?.edge.blockerIssueId ?? null, terminalBlockerIssueId, @@ -4537,7 +4575,7 @@ async function listIssueBlockerAttentionMap( ); } - return attentionMap; + return result(); } type IssueReviewAttentionInput = Pick; @@ -7597,7 +7635,7 @@ export function issueService(db: Db) { companyId: string, blockedByIssueIds: string[], actor: { agentId?: string | null; userId?: string | null } = {}, - dbOrTx: any = db, + dbOrTx: any, ) { const deduped = [...new Set(blockedByIssueIds)]; if (deduped.some((candidate) => candidate === issueId)) { @@ -7626,6 +7664,9 @@ export function issueService(db: Db) { await assertNoBlockingCycles(companyId, issueId, deduped, dbOrTx); } + await notifyIssueAttentionRetentionWork(dbOrTx, { companyId, issueId }); + await notifyIssueRecovery(dbOrTx, { companyId }); + await notifyDeferredExecutionEligibility(dbOrTx, { companyId }); await dbOrTx .delete(issueRelations) .where( @@ -7729,6 +7770,7 @@ export function issueService(db: Db) { } const now = new Date(); + await notifyStatusCardIssueChange(tx, lockedIssue.companyId); const adopted = await tx .update(issues) .set({ @@ -7782,13 +7824,14 @@ export function issueService(db: Db) { sql`select ${heartbeatRuns.id} from ${heartbeatRuns} where ${heartbeatRuns.id} = ${input.actorRunId} for update`, ); const actorRun = await tx - .select({ status: heartbeatRuns.status }) + .select({ status: heartbeatRuns.status, companyId: heartbeatRuns.companyId }) .from(heartbeatRuns) .where(eq(heartbeatRuns.id, input.actorRunId)) .then((rows) => rows[0] ?? null); if (!actorRun || TERMINAL_HEARTBEAT_RUN_STATUSES.has(actorRun.status)) return null; + await notifyStatusCardIssueChange(tx, actorRun.companyId); const now = new Date(); const adopted = await tx .update(issues) @@ -9371,6 +9414,9 @@ export function issueService(db: Db) { return listIssueDependencyReadinessMap(dbOrTx, companyId, issueIds); }, + listBlockerAttentionBatch: async (companyId: string, issueRows: IssueBlockerAttentionInputNode[], dbOrTx: any = db) => + readIssueBlockerAttentionBatch(dbOrTx, companyId, issueRows), + listBlockerAttention: async ( companyId: string, issueRows: IssueBlockerAttentionInputNode[], @@ -9674,27 +9720,32 @@ export function issueService(db: Db) { }); if (blockParentUntilDone) { - const existingBlockers = await db - .select({ blockerIssueId: issueRelations.issueId }) - .from(issueRelations) - .where( - and( - eq(issueRelations.companyId, parent.companyId), - eq(issueRelations.relatedIssueId, parent.id), - eq(issueRelations.type, "blocks"), - ), + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and( + eq(issues.companyId, parent.companyId), eq(issues.id, parent.id))).for("update"); + const existingBlockers = await tx + .select({ blockerIssueId: issueRelations.issueId }) + .from(issueRelations) + .where( + and( + eq(issueRelations.companyId, parent.companyId), + eq(issueRelations.relatedIssueId, parent.id), + eq(issueRelations.type, "blocks"), + ), + ); + await syncBlockedByIssueIds( + parent.id, + parent.companyId, + [ + ...new Set([ + ...existingBlockers.map((row) => row.blockerIssueId), + child.id, + ]), + ], + { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, + tx, ); - await syncBlockedByIssueIds( - parent.id, - parent.companyId, - [ - ...new Set([ - ...existingBlockers.map((row) => row.blockerIssueId), - child.id, - ]), - ], - { agentId: actorAgentId ?? null, userId: actorUserId ?? null }, - ); + }); [child] = await withIssueRelationSummaries( parent.companyId, [child], @@ -10605,7 +10656,13 @@ export function issueService(db: Db) { : values.parentId ? `issue:${values.parentId}` : undefined); } if (values.monitorNextCheckAt) await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: values.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: values.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + values.id ??= randomUUID(); + await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: values.id, parentId: values.parentId, originKind: values.originKind, originId: values.originId }); + await notifyStatusCardIssueChange(tx, companyId); + if (values.originKind === "routine_execution") await notifyRoutineExecutionStatus(tx, values.id, companyId); const [issue] = await tx.insert(issues).values(values).returning(); await attachOwnedDraftImages(tx, issue, issue.description, { userId: issueData.createdByUserId, agentId: issueData.createdByAgentId }); if (issue.visibility === "private" && !inheritedPrivateSubtree && issueData.createdByAgentId) { @@ -10890,6 +10947,7 @@ export function issueService(db: Db) { } } + await notifyStatusCardIssueChange(tx, companyId); await insertRowsInChunks(tx, issues, issueRows); await insertRowsInChunks(tx, issueLabels, labelRows); }); @@ -10906,6 +10964,7 @@ export function issueService(db: Db) { const censorUsernameInLogs = (await instanceSettings.getGeneral()) .censorUsernameInLogs; await db.transaction(async (tx) => { + for (const companyId of new Set(rows.map(row => row.companyId))) await notifyStatusCardIssueChange(tx, companyId); const commentRows = rows.map((row) => { const createdAt = row.createdAt ? new Date(row.createdAt) : null; return { @@ -11486,14 +11545,27 @@ export function issueService(db: Db) { (blockedByIssueIds !== undefined || issueData.unblockDescriptor !== undefined)))) { patch.statusVersion = sql`${issues.statusVersion} + 1` as unknown as number; } + if (patch.status !== undefined || patch.completedAt !== undefined || patch.cancelledAt !== undefined) { + await notifyNativeHarnessBackupExpiration(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + await notifyStatusCardIssueChange(tx, receiptExisting.companyId); if ((patch.status !== undefined && patch.status !== receiptExisting.status) || patch.completedAt !== undefined || patch.cancelledAt !== undefined || issueData.parentId !== undefined || issueData.executionWorkspaceId !== undefined) { await notifyTerminalWorkspaceWork(tx, `issue:${id}`); } + if (patch.status !== undefined || patch.hiddenAt !== undefined) { + await notifyMergeConfirmationWork(tx, { companyId: receiptExisting.companyId, issueId: id }); + } + await notifyIssueAttentionRetentionWork(tx, { companyId: receiptExisting.companyId, issueId: id, parentId: patch.parentId, originKind: patch.originKind, originId: patch.originId }); if (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined - || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined) { + || patch.hiddenAt !== undefined || patch.conversationAgentId !== undefined || patch.parentId !== undefined + || patch.executionState !== undefined || patch.executionRunId !== undefined || patch.checkoutRunId !== undefined + || blockedByIssueIds !== undefined) { + await notifyRunDispatch(tx, { companyId: receiptExisting.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: receiptExisting.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); } if (patch.monitorNextCheckAt !== undefined || (receiptExisting.monitorNextCheckAt && (patch.status !== undefined || patch.assigneeAgentId !== undefined || patch.assigneeUserId !== undefined))) { @@ -11542,6 +11614,9 @@ export function issueService(db: Db) { await notifyNativeWorkspaceFinalization(tx, { companyId: receiptExisting.companyId, issueId: id }); await notifyNativeStatusReassessment(tx, { companyId: receiptExisting.companyId, issueId: id }); } + if (patch.originKind !== undefined || patch.originRunId !== undefined || patch.originId !== undefined) { + await notifyRoutineExecutionStatus(tx, id); + } const updated = await tx .update(issues) .set(patch) @@ -11919,7 +11994,9 @@ export function issueService(db: Db) { ); if (settings?.environmentId !== environmentId) continue; - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, companyId); + await tx .update(issues) .set({ executionWorkspaceSettings: { @@ -11929,6 +12006,7 @@ export function issueService(db: Db) { updatedAt: new Date(), }) .where(eq(issues.id, row.id)); + }); cleared += 1; } @@ -11952,12 +12030,18 @@ export function issueService(db: Db) { await requestRestorePointDeletion(tx, { sourceIssueId: id }); const [owner] = await tx.select({ companyId: issues.companyId }).from(issues).where(eq(issues.id, id)); if (owner) { + await notifyIssueAttentionRetentionWork(tx, { companyId: owner.companyId, issueId: id }); await notifyExternalConversationSettlement(tx, { companyId: owner.companyId }); + await notifyExternalObjectRefresh(tx, owner.companyId); await notifyApprovalWork(tx, owner.companyId); + await notifyStatusCardIssueChange(tx, owner.companyId); } await notifyDecisionTargetIssueRemoval(tx, id); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, owner ? { companyId: owner.companyId } : undefined); + if (owner) await notifyDeferredExecutionEligibility(tx, owner); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); removedIssue = await tx .delete(issues) .where(eq(issues.id, id)) @@ -12084,8 +12168,13 @@ export function issueService(db: Db) { ) : isNull(issues.executionRunId); const updated = await db.transaction(async tx => { + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set({ @@ -12136,7 +12225,9 @@ export function issueService(db: Db) { current.executionRunId === checkoutRunId) && checkoutRunId ) { - const adopted = await db + const adopted = await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, issueCompany.companyId); + return tx .update(issues) .set({ checkoutRunId, @@ -12157,6 +12248,7 @@ export function issueService(db: Db) { ) .returning() .then((rows) => rows[0] ?? null); + }); if (adopted) return adopted; } @@ -12213,8 +12305,13 @@ export function issueService(db: Db) { } const staleExecutionRunId = current.executionRunId; const adopted = await db.transaction(async tx => { + await notifyIssueAttentionRetentionWork(tx, { companyId: issueCompany.companyId, issueId: id }); + await notifyStatusCardIssueChange(tx, issueCompany.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: issueCompany.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: issueCompany.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); return tx .update(issues) .set(adoptionSet) @@ -12468,8 +12565,12 @@ export function issueService(db: Db) { existing.status === "in_progress" ? "todo" : existing.status; await notifyTerminalWorkspaceWork(tx, `issue:${id}`); await notifyNativeStatusReassessment(tx, { companyId: existing.companyId, issueId: id }); + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set({ @@ -12520,6 +12621,8 @@ export function issueService(db: Db) { patch.assigneeAgentId = null; } + await notifyIssueAttentionRetentionWork(tx, { companyId: existing.companyId, issueId: id }); + await notifyStatusCardIssueChange(tx, existing.companyId); await notifyTerminalWorkspaceWork(tx, `issue:${id}`); if (existing.checkoutRunId || existing.executionRunId || options.clearAssignee) { await notifyExternalConversationSettlement(tx, { companyId: existing.companyId, issueId: id }); @@ -12529,7 +12632,10 @@ export function issueService(db: Db) { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: existing.companyId, issueId: id }); } await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyRunDispatch(tx, { companyId: existing.companyId }); + await notifyDeferredExecutionEligibility(tx, { companyId: existing.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, id); const updated = await tx .update(issues) .set(patch) @@ -12577,12 +12683,12 @@ export function issueService(db: Db) { return created; }, - deleteLabel: async (id: string) => - db - .delete(labels) - .where(eq(labels.id, id)) - .returning() - .then((rows) => rows[0] ?? null), + deleteLabel: async (id: string) => db.transaction(async tx => { + const [label] = await tx.select({ companyId: labels.companyId }).from(labels).where(eq(labels.id, id)); + if (!label) return null; + await notifyStatusCardIssueChange(tx, label.companyId); + return tx.delete(labels).where(eq(labels.id, id)).returning().then(rows => rows[0] ?? null); + }), listComments: async ( issueId: string, @@ -12726,6 +12832,7 @@ export function issueService(db: Db) { .from(issueComments).where(eq(issueComments.id, commentId)); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); const [comment] = await tx .delete(issueComments) .where(eq(issueComments.id, commentId)) @@ -12765,6 +12872,7 @@ export function issueService(db: Db) { .for("update"); if (!source) return null; await notifyExternalConversationSettlement(tx, source); + await notifyStatusCardIssueChange(tx, source.companyId); await notifyChatInboundWakeNotices(tx, source); const now = new Date(); const [comment] = await tx @@ -12832,7 +12940,7 @@ export function issueService(db: Db) { }, dbOrTx: any = db, ): Promise { - if (dbOrTx === db && (actor.runId || actor.userId)) { + if (dbOrTx === db) { const append = () => measureRunTransaction(db, "response", async (tx) => { // Serialize run-authored comments on the issue so a provider retry @@ -13075,6 +13183,7 @@ export function issueService(db: Db) { currentUserRedactionOptions.enabled, ); } + await notifyStatusCardIssueChange(dbOrTx, issue.companyId); comment = shouldUpgradeExternalAuthorization || shouldUpgradeAttachmentAuthorization @@ -13789,6 +13898,7 @@ export function issueService(db: Db) { .then((rows) => rows[0] ?? null); if (!existing) return null; + await notifyIssueRecovery(tx, { companyId: existing.companyId }); await tx.delete(issueAttachments).where(eq(issueAttachments.id, id)); await tx.delete(assets).where(eq(assets.id, existing.assetId)); return existing; diff --git a/server/src/services/legacy-controller-lease.ts b/server/src/services/legacy-controller-lease.ts index 55a8439debc..22fa078f459 100644 --- a/server/src/services/legacy-controller-lease.ts +++ b/server/src/services/legacy-controller-lease.ts @@ -77,14 +77,18 @@ async function renewOnLivenessPool( }); } -export async function hasLiveLegacyController(db: Db, run: Run): Promise { - if (run.runtimeMode === "native" || !run.controllerBootId) return false; - const [owner] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( +export async function liveLegacyControllerExpiresAt(db: Db, run: Run): Promise { + if (run.runtimeMode === "native" || !run.controllerBootId) return null; + const [owner] = await db.select({ expiresAt: heartbeatRuns.controllerLeaseExpiresAt }).from(heartbeatRuns).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "running"), gt(heartbeatRuns.controllerLeaseExpiresAt, sql`clock_timestamp()`), )); - return Boolean(owner); + return owner?.expiresAt ?? null; +} + +export async function hasLiveLegacyController(db: Db, run: Run): Promise { + return (await liveLegacyControllerExpiresAt(db, run)) !== null; } export type RevokedLegacyControllerState = Pick; diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 427704de63f..c462cdf1fca 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -135,7 +135,7 @@ export async function terminalizeLegacyExecution(input: { }; if (issueId) await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: current.companyId, issueId }); await notifyInstructionCollection(tx, { companyId: current.companyId, runId: current.id }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: current.companyId }); let [updated] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/services/merged-pr-confirmation-notifications.ts b/server/src/services/merged-pr-confirmation-notifications.ts new file mode 100644 index 00000000000..34df4ecffb0 --- /dev/null +++ b/server/src/services/merged-pr-confirmation-notifications.ts @@ -0,0 +1,7 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type MergeConfirmationScope = { companyId: string; issueId?: string; interactionId?: string }; + +export function notifyMergeConfirmationWork(tx: object, scope: MergeConfirmationScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.mergedPullRequestConfirmations, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, interactionId: scope.interactionId })); +} diff --git a/server/src/services/merged-pr-confirmation-work.ts b/server/src/services/merged-pr-confirmation-work.ts new file mode 100644 index 00000000000..6a754d5a88a --- /dev/null +++ b/server/src/services/merged-pr-confirmation-work.ts @@ -0,0 +1,38 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { issueThreadInteractionService } from "./issue-thread-interactions.js"; +import type { MergeConfirmationScope } from "./merged-pr-confirmation-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +export const MERGE_CONFIRMATION_RECONCILE_MS = 5 * 60_000; + +export function registerMergeConfirmationWork( + coordinator: ReturnType, + service: Pick, + "mergedPullRequestConfirmationCandidates" | "mergedPullRequestConfirmationPage" | "sweepMergedPullRequestConfirmations">, + canRun: () => boolean, +) { + const cursors = new Map(); + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.mergedPullRequestConfirmations, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as MergeConfirmationScope, + matches: (entry, scope?: MergeConfirmationScope) => !scope || (entry.companyId === scope.companyId && + (!scope.issueId || entry.issueId === scope.issueId) && (!scope.interactionId || entry.id === scope.interactionId)), + load: async (scope?: MergeConfirmationScope) => { + const key = scope ? JSON.stringify(scope) : "*"; + const page = await service.mergedPullRequestConfirmationPage(scope, cursors.get(key)); + if (page.complete) cursors.delete(key); + else cursors.set(key, page.after!); + return { complete: page.complete, rows: page.rows.map(({ interaction }) => ({ + id: interaction.id, companyId: interaction.companyId, issueId: interaction.issueId, dueAt: Date.now(), + })) }; + }, + check: async (entry, signal) => { + const scope = { companyId: entry.companyId, issueId: entry.issueId, interactionId: entry.id }; + await service.sweepMergedPullRequestConfirmations([], scope, signal); + if (signal?.aborted) return entry; + const remaining = await service.mergedPullRequestConfirmationCandidates(scope); + return remaining.length ? { ...entry, dueAt: Date.now() + MERGE_CONFIRMATION_RECONCILE_MS } : null; + }, + }, canRun); +} diff --git a/server/src/services/native-environment-settlement-notifications.ts b/server/src/services/native-environment-settlement-notifications.ts index 5769b81015a..0c55810b6a8 100644 --- a/server/src/services/native-environment-settlement-notifications.ts +++ b/server/src/services/native-environment-settlement-notifications.ts @@ -1,8 +1,10 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export type NativeEnvironmentSettlementScope = { companyId: string; runId?: string; issueId?: string }; +export type NativeEnvironmentSettlementScope = { companyId: string; runId?: string; issueId?: string; leaseId?: string }; export async function notifyNativeEnvironmentSettlement(tx: object, scope?: NativeEnvironmentSettlementScope) { + await notifyNativeHarnessBackupExpiration(tx, scope); if (scope) await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeEnvironmentSettlement, scope ? JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId }) : "*"); diff --git a/server/src/services/native-finalization-notifications.ts b/server/src/services/native-finalization-notifications.ts index 72d4cbcac49..1feb1ba9703 100644 --- a/server/src/services/native-finalization-notifications.ts +++ b/server/src/services/native-finalization-notifications.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { notifyNativeEnvironmentSettlement } from "./native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../modules/external-agents/index.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; @@ -14,6 +15,7 @@ import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notificatio export type NativeFinalizationWorkScope = { companyId: string; runId?: string; issueId?: string }; export async function notifyNativeFinalizationWork(transaction: object, scope: NativeFinalizationWorkScope) { + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); if (scope.issueId || scope.runId) { await notifyNativeEnvironmentSettlement(transaction, scope); await notifyExternalAgentSettlement(transaction, scope); diff --git a/server/src/services/native-harness-backup-expiration-notifications.ts b/server/src/services/native-harness-backup-expiration-notifications.ts new file mode 100644 index 00000000000..e334487a82c --- /dev/null +++ b/server/src/services/native-harness-backup-expiration-notifications.ts @@ -0,0 +1,26 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type NativeHarnessBackupExpirationScope = { + companyId: string; scopeDigest?: string; leaseId?: string; runId?: string; issueId?: string; agentId?: string; sessionId?: string; +}; + +export function nativeHarnessBackupScopeHint(scope: NativeHarnessBackupExpirationScope) { + return JSON.stringify({ companyId: scope.companyId, scopeDigest: scope.scopeDigest, leaseId: scope.leaseId, + runId: scope.runId, issueId: scope.issueId, agentId: scope.agentId, sessionId: scope.sessionId }); +} + +export function notifyNativeHarnessBackupExpiration(tx: object, scope?: NativeHarnessBackupExpirationScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeHarnessBackupExpiration, scope ? nativeHarnessBackupScopeHint(scope) : "*"); +} + +const listeners = new WeakMap void>>(); +export function onNativeHarnessBackupChange(db: object, listener: (scope: NativeHarnessBackupExpirationScope) => void) { + let owned = listeners.get(db); + if (!owned) { owned = new Set(); listeners.set(db, owned); } + owned.add(listener); + return () => { owned.delete(listener); if (!owned.size) listeners.delete(db); }; +} + +export function noteNativeHarnessBackupChange(db: object, scope: NativeHarnessBackupExpirationScope) { + for (const listener of listeners.get(db) ?? []) listener(scope); +} diff --git a/server/src/services/native-runtime/chat-attachment-reuse.test.ts b/server/src/services/native-runtime/chat-attachment-reuse.test.ts index ccd4bbe30d6..d0de5ab04c9 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.test.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.test.ts @@ -73,7 +73,7 @@ describe("native same-conversation chat attachment reuse", () => { beforeAll(async () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CHAT_REUSE_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-chat-reuse-"); - db = createDb(external ?? temporary!.connectionString); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); const storageRoot = await mkdtemp( path.join(tmpdir(), "paperclip-chat-reuse-"), ); diff --git a/server/src/services/native-runtime/chat-attachment-reuse.ts b/server/src/services/native-runtime/chat-attachment-reuse.ts index cc7990e9b7a..6cc44299ee7 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, desc, eq, inArray, isNull, sql } from "drizzle-orm"; @@ -1582,7 +1583,9 @@ export async function prepareReusedChatAttachment(input: { if (!workProduct) { throw new Error("paperclip_runner_chat_attachment_origin_not_persisted"); } - await input.db + await input.db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await tx .update(issueWorkProducts) .set({ title: input.title, @@ -1602,6 +1605,7 @@ export async function prepareReusedChatAttachment(input: { eq(issueWorkProducts.createdByRunId, input.binding.runId), ), ); + }); const comment = await issueService(input.db).addComment( input.binding.issueId, `Prepared ${input.title} for this response.`, diff --git a/server/src/services/native-runtime/native-failed-workspace-recovery.ts b/server/src/services/native-runtime/native-failed-workspace-recovery.ts index 5c3268f6afc..a1c6c914b33 100644 --- a/server/src/services/native-runtime/native-failed-workspace-recovery.ts +++ b/server/src/services/native-runtime/native-failed-workspace-recovery.ts @@ -65,7 +65,7 @@ export async function preserveNativeFailedWorkspace(db: Db, selected: Pick; issueIds: Set; leaseIds: Set }; +type Entry = { id: string; companyId: string; agentId: string; dueAt: number; dependencies: Dependencies }; +type Reference = "company" | "agent" | "session" | "run" | "issue" | "lease"; + +export function registerNativeHarnessBackupExpirationWork( + coordinator: ReturnType, db: Db, + options: NativeHarnessBackupExpirationOptions, canRun: () => boolean, +) { + const service = nativeHarnessBackupExpirationService(db, { ...options, duty: "backups" }); + const cursors = new Map(); + const sources = new Map(); + const index = new Map>(); + const key = (companyId: string, kind: Reference, id = "") => JSON.stringify([companyId, kind, id]); + const sessionKey = (agentId: string, sessionId: string) => JSON.stringify([agentId, sessionId]); + function references(source: Dependencies) { + return [key(source.companyId, "company"), key(source.companyId, "agent", source.agentId), + key(source.companyId, "session", sessionKey(source.agentId, source.sessionId)), + ...[...source.runIds].map(id => key(source.companyId, "run", id)), + ...[...source.issueIds].map(id => key(source.companyId, "issue", id)), + ...[...source.leaseIds].map(id => key(source.companyId, "lease", id))]; + } + function add(reference: string, id: string) { + let roots = index.get(reference); + if (!roots) { roots = new Set(); index.set(reference, roots); } + roots.add(id); + } + function remember(id: string, current?: NativeHarnessBackupDependencies) { + const previous = sources.get(id); + if (previous) for (const reference of references(previous)) { + const roots = index.get(reference)!; + roots.delete(id); if (!roots.size) index.delete(reference); + } + if (!current) { sources.delete(id); return undefined; } + const source = { companyId: current.companyId, agentId: current.agentId, + sessionId: current.sessionId, runIds: new Set(current.runIds), issueIds: new Set(current.issueIds), leaseIds: new Set(current.leaseIds) }; + sources.set(id, source); + for (const reference of references(source)) add(reference, id); + return source; + } + function selected(scope: NativeHarnessBackupExpirationScope): Set { + if (scope.scopeDigest) return new Set([scope.scopeDigest]); + const reference = scope.leaseId ? key(scope.companyId, "lease", scope.leaseId) + : scope.runId ? key(scope.companyId, "run", scope.runId) + : scope.sessionId && scope.agentId ? key(scope.companyId, "session", sessionKey(scope.agentId, scope.sessionId)) + : scope.issueId ? key(scope.companyId, "issue", scope.issueId) + : scope.agentId ? key(scope.companyId, "agent", scope.agentId) : key(scope.companyId, "company"); + return new Set(index.get(reference) ?? []); + } + async function discoverReferences(scope: NativeHarnessBackupExpirationScope) { + if (scope.scopeDigest || (!scope.runId && !scope.leaseId)) return; + const resolved = await service.resolveBackupScope(scope); + const roots = selected(scope); + if (resolved.scopeDigest) roots.add(resolved.scopeDigest); + const related = resolved.sessionId && resolved.agentId + ? selected({ companyId: scope.companyId, sessionId: resolved.sessionId, agentId: resolved.agentId }) + : resolved.issueId ? selected({ companyId: scope.companyId, issueId: resolved.issueId }) : []; + for (const id of related) roots.add(id); + for (const id of roots) { + const source = sources.get(id); + if (!source || source.companyId !== scope.companyId) continue; + if (scope.runId) { source.runIds.add(scope.runId); add(key(scope.companyId, "run", scope.runId), id); } + if (scope.leaseId) { source.leaseIds.add(scope.leaseId); add(key(scope.companyId, "lease", scope.leaseId), id); } + } + } + async function reconcile(id: string): Promise { + const source = await service.reconcileBackup(id); + const dependencies = remember(id, source?.dependencies); + return source && dependencies && source.dueAt !== null ? { ...source, dependencies, dueAt: source.dueAt } : null; + } + function matches(entry: Entry, scope?: NativeHarnessBackupExpirationScope) { + if (!scope) return true; + if (entry.companyId !== scope.companyId) return false; + if (scope.scopeDigest) return entry.id === scope.scopeDigest; + if (scope.leaseId) return entry.dependencies.leaseIds.has(scope.leaseId); + if (scope.runId) return entry.dependencies.runIds.has(scope.runId); + if (scope.sessionId) return entry.dependencies.sessionId === scope.sessionId + && (!scope.agentId || entry.agentId === scope.agentId); + if (scope.issueId) return entry.dependencies.issueIds.has(scope.issueId); + return !scope.agentId || entry.agentId === scope.agentId; + } + const worker = registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.nativeHarnessBackupExpiration, batchSize: options.maxRemovals ?? 25, + decodeScope: (hint: string): NativeHarnessBackupExpirationScope | undefined => hint === "*" ? undefined : JSON.parse(hint), + matches, + load: async (scope, signal) => { + const hint = scope ? nativeHarnessBackupScopeHint(scope) : "*"; + if (scope && !cursors.has(hint)) await discoverReferences(scope); + const limit = options.maxEntries ?? 100; + const indexed = scope ? [...selected(scope)].filter(id => !cursors.get(hint) || id > cursors.get(hint)!).sort() : null; + const page = indexed ? { names: indexed.slice(0, limit), complete: indexed.length <= limit } + : await service.backupScopes(cursors.get(hint), limit); + const rows: Entry[] = []; + for (const name of page.names) { + if (signal?.aborted || !canRun()) throw new Error("Backup expiration discovery paused"); + const entry = await reconcile(name); + if (entry && (!scope || entry.companyId === scope.companyId)) rows.push(entry); + } + if (page.complete) cursors.delete(hint); else cursors.set(hint, page.names.at(-1)!); + return { rows, complete: page.complete }; + }, + check: async (entry, signal) => { + if (signal?.aborted || !canRun()) throw new Error("Backup expiration paused"); + const current = await reconcile(entry.id); + if (!current || current.companyId !== entry.companyId) return null; + if (current.dueAt > Date.now()) return current; + const result = await service.expireBackup(entry.id); + if (result.candidates || result.failed) logger.info({ ...result, dryRun: options.dryRun ?? false }, "native harness backup expiration"); + if (options.dryRun) return null; + const remaining = await reconcile(entry.id); + if (!remaining) return null; + if (result.skipped.session_active) throw new Error("Backup session is active"); + return remaining; + }, + }, canRun); + const unsubscribe = onNativeHarnessBackupChange(db, scope => worker.invalidate(nativeHarnessBackupScopeHint(scope))); + const release = onRetainedNativeSessionCleanupRelease(db, (_scope, source) => worker.invalidate(nativeHarnessBackupScopeHint(source))); + return { ...worker, async stop() { unsubscribe(); release(); await worker.stop(); } }; +} diff --git a/server/src/services/native-runtime/native-harness-backup-expiration.ts b/server/src/services/native-runtime/native-harness-backup-expiration.ts index 8ea9a242629..1d6a43169b9 100644 --- a/server/src/services/native-runtime/native-harness-backup-expiration.ts +++ b/server/src/services/native-runtime/native-harness-backup-expiration.ts @@ -17,6 +17,7 @@ import { nativeHarnessStateBase } from "./native-harness-backup-stamp.js"; import { nativeSessionScopeDigestIsBusy } from "./native-session-executor.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; +import type { NativeHarnessBackupExpirationScope } from "../native-harness-backup-expiration-notifications.js"; export const NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS = 60 * 60 * 1000; export const NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS = 2 * 60 * 1000; @@ -146,7 +147,9 @@ async function readSessionIdentity(root: string): Promise MAX_CONTROL_PLANE_STATE_BYTES) return null; const state = JSON.parse(await fs.readFile(statePath, "utf8")) as { identity?: Record }; + if (!state || typeof state !== "object" || Array.isArray(state)) return null; const identity = state.identity ?? {}; + if (!identity || typeof identity !== "object" || Array.isArray(identity)) return null; const runId = identity.runId; const normalizedSessionId = identity.normalizedSessionId; const environmentLeaseId = identity.environmentLeaseId; @@ -160,17 +163,22 @@ async function readSessionIdentity(root: string): Promise[0]>[0]; async function sessionRetentionBlocker( tx: Tx, - input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean }, -): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[] }> { + input: { scopeDigest: string | null; identity: SessionIdentity; closedBefore: Date; checkpoint?: boolean; lock?: boolean }, +): Promise<{ reason: string } | { companyId: string; agentId: string; runIds: string[]; issueIds: string[]; closedAt: number }> { const [owner] = await tx .select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId }) .from(heartbeatRuns) @@ -197,7 +205,7 @@ async function sessionRetentionBlocker( if (input.checkpoint && runs.some(run => run.finishedAt! >= input.closedBefore)) return { reason: "recent_run" }; const issueIds = [...new Set(runs.map((run) => run.issueId).filter((id): id is string => Boolean(id)))]; if (issueIds.length === 0) return { reason: "issue_unknown" }; - const issueRows = await tx + const issueQuery = tx .select({ id: issues.id, status: issues.status, @@ -205,8 +213,8 @@ async function sessionRetentionBlocker( closedAt: sql`coalesce(${issues.completedAt}, ${issues.cancelledAt})`, }) .from(issues) - .where(and(eq(issues.companyId, owner.companyId), inArray(issues.id, issueIds))) - .for("share"); + .where(and(eq(issues.companyId, owner.companyId), inArray(issues.id, issueIds))); + const issueRows = await (input.lock === false ? issueQuery : issueQuery.for("share")); if (issueRows.length !== issueIds.length) return { reason: "issue_unknown" }; for (const issue of issueRows) { if (!CLOSED_ISSUE_STATUSES.includes(issue.status)) return { reason: "issue_open" }; @@ -246,7 +254,7 @@ async function sessionRetentionBlocker( .where(and(inArray(environmentLeases.status, LIVE_LEASE_STATUSES), or(...leaseReferences))) .limit(1); if (lease) return { reason: "live_lease" }; - return { ...owner, runIds }; + return { ...owner, runIds, issueIds, closedAt: issueRows.reduce((latest, issue) => Math.max(latest, new Date(issue.closedAt!).getTime()), Number.NEGATIVE_INFINITY) }; } async function hasLiveRun(db: Db, owner: { companyId: string; agentId: string }, sessionId: string) { @@ -344,7 +352,7 @@ export function nativeHarnessBackupExpirationService( async function settleLeftovers(backupRoot: string) { const current = path.join(backupRoot, "current"); - for (const name of await fs.readdir(backupRoot).catch(() => [] as string[])) { + for (const name of (await fs.readdir(backupRoot).catch(missingPath)) ?? []) { const target = path.join(backupRoot, name); if (name.startsWith(RETIRING_PREFIX) && UUID_PATTERN.test(name.slice(RETIRING_PREFIX.length))) { if (!(await isRealDirectory(current))) await fs.rename(target, current); @@ -379,54 +387,49 @@ export function nativeHarnessBackupExpirationService( const identity = await readSessionIdentity(root); if (!identity) return skip("identity_unknown"); - const decision = await db.transaction(async (tx) => { - const blocker = await sessionRetentionBlocker(tx, { - scopeDigest, - identity, - closedBefore: options.closedBefore, - }); - if ("reason" in blocker) return blocker; - if (options.dryRun) return { owner: blocker, cleared: 0 }; - await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); - const cleared = await tx - .delete(agentTaskSessions) - .where( - and( - eq(agentTaskSessions.companyId, blocker.companyId), - eq(agentTaskSessions.agentId, blocker.agentId), - sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${identity.normalizedSessionId}`, - ), - ) - .returning({ id: agentTaskSessions.id }); - return { owner: blocker, cleared: cleared.length }; - }); - if ("reason" in decision) return skip(decision.reason); - const bytes = await allocatedBytes(current); - result.candidates++; - result.candidateBytes += bytes; - if (options.dryRun) return; - result.clearedTaskSessions += decision.cleared; - if (options.isSessionBusy(scopeDigest)) return skip("session_active"); const id = randomUUID(); const retiring = path.join(backupRoot, `${RETIRING_PREFIX}${id}`); const trash = path.join(backupRoot, `${EXPIRED_PREFIX}${id}`); - await fs.rename(current, retiring); - let liveRun: boolean; + let retired = false; + let decision: { reason: string } | { cleared: number; bytes: number }; try { - liveRun = await hasLiveRun(db, decision.owner, identity.normalizedSessionId); - } catch (err) { - await fs.rename(retiring, current); - throw err; - } - if (liveRun) { - await fs.rename(retiring, current); - return skip("live_run"); + decision = await db.transaction(async (tx) => { + const blocker = await sessionRetentionBlocker(tx, { + scopeDigest, identity, closedBefore: options.closedBefore, + }); + if ("reason" in blocker) return blocker; + if ((await fs.stat(current)).mtime >= options.closedBefore) return { reason: "recent_backup" }; + if (options.dryRun) return { cleared: 0, bytes }; + if (options.isSessionBusy(scopeDigest)) return { reason: "session_active" }; + await notifyExternalConversationSettlement(tx, { companyId: blocker.companyId }); + await fs.rename(current, retiring); + retired = true; + if (await hasLiveRun(tx as unknown as Db, blocker, identity.normalizedSessionId)) { + await fs.rename(retiring, current); + retired = false; + return { reason: "live_run" }; + } + const cleared = await tx.delete(agentTaskSessions).where(and( + eq(agentTaskSessions.companyId, blocker.companyId), + eq(agentTaskSessions.agentId, blocker.agentId), + sql`${agentTaskSessions.sessionParamsJson}->>'sessionId' = ${identity.normalizedSessionId}`, + )).returning({ id: agentTaskSessions.id }); + return { cleared: cleared.length, bytes }; + }); + } catch (error) { + if (retired) await fs.rename(retiring, current); + throw error; } + if ("reason" in decision) return skip(decision.reason); + result.candidates++; + result.candidateBytes += decision.bytes; + if (options.dryRun) return; + result.clearedTaskSessions += decision.cleared; await fs.rename(retiring, trash); await removeBackupTree(trash); result.expired++; - result.reclaimedBytes += bytes; + result.reclaimedBytes += decision.bytes; } async function removeExpiredLeftovers(parent: string, context: EvidenceContext, result: NativeHarnessBackupExpirationResult) { @@ -745,5 +748,91 @@ export function nativeHarnessBackupExpirationService( }); } - return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration }; + async function backupDependencies(scopeDigest: string, identity: SessionIdentity): Promise { + const [owner] = await db.select({ companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(eq(heartbeatRuns.id, identity.runId)); + if (!owner) return null; + const runs = await db.select({ id: heartbeatRuns.id, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, owner.companyId), + eq(heartbeatRuns.agentId, owner.agentId), eq(heartbeatRuns.nativeSessionId, identity.normalizedSessionId))); + const runIds = [...new Set([identity.runId, ...runs.map(run => run.id)])]; + const issueIds = [...new Set([owner.issueId, ...runs.map(run => run.issueId)].filter((id): id is string => Boolean(id)))]; + const leases = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, owner.companyId), or(inArray(environmentLeases.heartbeatRunId, runIds), + sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256' = ${scopeDigest}`, + identity.environmentLeaseId ? eq(environmentLeases.id, identity.environmentLeaseId) : undefined))); + return { companyId: owner.companyId, agentId: owner.agentId, sessionId: identity.normalizedSessionId, + runIds, issueIds, leaseIds: leases.map(lease => lease.id) }; + } + + async function resolveBackupScope(scope: NativeHarnessBackupExpirationScope) { + let runId = scope.runId; + let issueId = scope.issueId; + let scopeDigest = scope.scopeDigest; + if (scope.leaseId) { + const [lease] = await db.select({ runId: environmentLeases.heartbeatRunId, issueId: environmentLeases.issueId, + scopeDigest: sql`${environmentLeases.metadata}->'nativeHarnessBackup'->>'sessionScopeSha256'`, + }).from(environmentLeases).where(and(eq(environmentLeases.companyId, scope.companyId), eq(environmentLeases.id, scope.leaseId))); + runId = lease?.runId ?? runId; issueId = lease?.issueId ?? issueId; + scopeDigest = lease?.scopeDigest && SCOPE_PATTERN.test(lease.scopeDigest) ? lease.scopeDigest : scopeDigest; + } + const [run] = runId ? await db.select({ agentId: heartbeatRuns.agentId, sessionId: heartbeatRuns.nativeSessionId, + issueId: sql`coalesce(${heartbeatRuns.nativeIssueId}, ${heartbeatRuns.issueId})`, + }).from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.id, runId))) : []; + return { runId, issueId: run?.issueId ?? issueId, agentId: run?.agentId ?? scope.agentId, + sessionId: run?.sessionId ?? scope.sessionId, scopeDigest }; + } + + async function reconcileBackup(scopeDigest: string) { + const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); + const root = path.join(base, scopeDigest); + if (!SCOPE_PATTERN.test(scopeDigest) || !(await isRealDirectory(root))) return null; + const backupRoot = path.join(root, "failover-backups"); + if (!(await isRealDirectory(backupRoot))) return null; + if (!defaults.dryRun) await settleLeftovers(backupRoot); + const current = path.join(backupRoot, "current"); + const stat = await fs.lstat(current).catch(missingPath); + if (!stat?.isDirectory() || stat.isSymbolicLink()) return null; + const identity = await readSessionIdentity(root); + if (!identity) return null; + const dependencies = await backupDependencies(scopeDigest, identity); + if (!dependencies) return null; + const source = { id: scopeDigest, companyId: dependencies.companyId, agentId: dependencies.agentId, dependencies }; + if (await fs.lstat(path.join(root, "cleanup-activation.json")).then(() => true, () => false)) { + return { ...source, dueAt: null }; + } + const blocker = await db.transaction(tx => sessionRetentionBlocker(tx, { + scopeDigest, identity, closedBefore: new Date(8_000_000_000_000_000), lock: false, + })); + if ("reason" in blocker) return { ...source, dueAt: null }; + const retentionMs = Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS; + return { ...source, dueAt: Math.max(stat!.mtime.getTime(), blocker.closedAt) + retentionMs + 1 }; + } + + async function backupScopes(after?: string, limit = 100) { + const base = path.resolve(defaults.stateBase ?? nativeHarnessStateBase()); + const names = (await fs.readdir(base).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return [] as string[]; + throw error; + })).filter(name => SCOPE_PATTERN.test(name) && (!after || name > after)).sort(); + return { names: names.slice(0, limit), complete: names.length <= limit }; + } + + async function expireBackup(scopeDigest: string) { + if (!SCOPE_PATTERN.test(scopeDigest)) throw new Error("Invalid native harness backup scope"); + const result = emptyResult(); + const root = path.join(path.resolve(defaults.stateBase ?? nativeHarnessStateBase()), scopeDigest); + const dryRun = defaults.dryRun ?? false; + await expireSession(root, scopeDigest, { + dryRun, + isSessionBusy: defaults.isSessionBusy ?? nativeSessionScopeDigestIsBusy, + closedBefore: new Date(Date.now() - Math.max(1, defaults.closedForDays ?? NATIVE_HARNESS_BACKUP_EXPIRATION_DEFAULT_DAYS) * DAY_MS), + }, result); + return result; + } + + return { tick, noteEvidence, nextEvidenceAt, tickCheckpoints, registerCheckpointExpiration, + reconcileBackup, backupScopes, expireBackup, resolveBackupScope }; } diff --git a/server/src/services/native-runtime/native-harness-backup-publication.test.ts b/server/src/services/native-runtime/native-harness-backup-publication.test.ts new file mode 100644 index 00000000000..dd9d940d9df --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-publication.test.ts @@ -0,0 +1,44 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { onNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; +import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; + +describe("native harness backup publication", () => { + let backupRoot = ""; + let unsubscribe: (() => void) | undefined; + afterEach(async () => { + unsubscribe?.(); unsubscribe = undefined; + if (backupRoot) await fs.rm(backupRoot, { recursive: true, force: true }); + }); + async function fixture() { + backupRoot = await fs.mkdtemp(path.join(os.tmpdir(), "native-backup-publication-")); + const current = path.join(backupRoot, "current"), pendingRoot = path.join(backupRoot, "pending"); + await fs.mkdir(current); await fs.mkdir(pendingRoot); + await fs.writeFile(path.join(current, "manifest.json"), "saved"); + await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement"); + return { db: {}, companyId: "company", scopeDigest: "scope", backupRoot, pendingRoot, current }; + } + it("restores the existing backup when its lease stamp fails without publishing a false change", async () => { + const input = await fixture(); const changes: unknown[] = []; + unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + await expect(publishNativeHarnessBackup({ ...input, stamp: async current => { + expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); + throw new Error("stamp transaction rejected"); + } })).rejects.toThrow("stamp transaction rejected"); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("saved"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([]); + }); + it("publishes the completed replacement once and removes its predecessor", async () => { + const input = await fixture(); const changes: unknown[] = []; + unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + await publishNativeHarnessBackup({ ...input, stamp: async current => { + expect(changes).toEqual([]); + expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); + } }); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("replacement"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); + expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]); + }); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-publication.ts b/server/src/services/native-runtime/native-harness-backup-publication.ts new file mode 100644 index 00000000000..820303584b0 --- /dev/null +++ b/server/src/services/native-runtime/native-harness-backup-publication.ts @@ -0,0 +1,28 @@ +import { existsSync, renameSync } from "node:fs"; +import { resolve } from "node:path"; +import { noteNativeHarnessBackupChange } from "../native-harness-backup-expiration-notifications.js"; +import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; + +export async function publishNativeHarnessBackup(input: { + db: object; companyId: string; scopeDigest: string; backupRoot: string; pendingRoot: string; + stamp?: (currentRoot: string) => Promise; +}) { + const currentRoot = resolve(input.backupRoot, "current"); + const previousRoot = resolve(input.backupRoot, "previous"); + removeNativeHarnessBackup(previousRoot); + let movedCurrent = false; + if (existsSync(currentRoot)) { + renameSync(currentRoot, previousRoot); + movedCurrent = true; + } + try { + renameSync(input.pendingRoot, currentRoot); + await input.stamp?.(currentRoot); + } catch (error) { + if (existsSync(currentRoot)) removeNativeHarnessBackup(currentRoot); + if (movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot)) renameSync(previousRoot, currentRoot); + throw error; + } + removeNativeHarnessBackup(previousRoot); + noteNativeHarnessBackupChange(input.db, { companyId: input.companyId, scopeDigest: input.scopeDigest }); +} diff --git a/server/src/services/native-runtime/native-provider-capacity-retry.ts b/server/src/services/native-runtime/native-provider-capacity-retry.ts index 5b54bd0cdfc..81ef00e1fee 100644 --- a/server/src/services/native-runtime/native-provider-capacity-retry.ts +++ b/server/src/services/native-runtime/native-provider-capacity-retry.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { agentWakeupRequests, heartbeatRuns, type Db } from "@paperclipai/db"; @@ -49,6 +51,8 @@ export async function materializeNativeProviderCapacityRetry(input: { scheduledRetryAt: dueAt.toISOString(), executionRetryAccounting: accountingForScheduledRetry(run, NATIVE_PROVIDER_OVERLOADED_CODE, attempt), }; + await notifyAttentionRetentionWork(input.tx, { companyId: input.companyId }); + await notifyRunDispatch(input.tx, { companyId: run.companyId }); const [wake] = await input.tx.insert(agentWakeupRequests).values({ companyId: run.companyId, agentId: run.agentId, source: "automation", triggerDetail: "system", reason: NATIVE_PROVIDER_OVERLOADED_CODE, status: "queued", runId: retryId, diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 8ac4d9bd918..b54f6019f3b 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -899,7 +899,7 @@ export async function claimNativeRestartRecoveries(input: { recycledProviderPids: providerProcesses.recycledPids, stderrTail: row.run.stderrExcerpt, }); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: row.run.companyId }); await tx .update(nativeRunFinalizations) .set({ diff --git a/server/src/services/native-runtime/native-review-participant.ts b/server/src/services/native-runtime/native-review-participant.ts index 573574dd19d..16787a140b4 100644 --- a/server/src/services/native-runtime/native-review-participant.ts +++ b/server/src/services/native-runtime/native-review-participant.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { agents, heartbeatRuns, @@ -240,6 +242,8 @@ export async function claimNativeReviewExecutionLock(db: Db, input: { if (!run || !["queued", "running"].includes(run.status) || !admitted || !expected || admitted.nativeReviewInteractionId !== expected.nativeReviewInteractionId || admitted.nativeReviewDecisionId !== expected.nativeReviewDecisionId) return false; + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); + await notifyStatusCardIssueChange(tx, input.companyId); const claimed = await tx.update(issues).set({ executionRunId: input.runId, executionAgentNameKey: input.agentNameKey, diff --git a/server/src/services/native-runtime/native-routine-status.test.ts b/server/src/services/native-runtime/native-routine-status.test.ts index 668de83a4d5..415622ea2c3 100644 --- a/server/src/services/native-runtime/native-routine-status.test.ts +++ b/server/src/services/native-runtime/native-routine-status.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; @@ -8,13 +9,14 @@ import { routineService } from "../routines.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { NATIVE_STATUS_ARBITER_POLICY_VERSION, type NativeStatusDecision } from "./status-arbiter.js"; -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_ROUTINE_STATUS_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping native routine status tests: ${support.reason}`); (support.supported ? describe : describe.skip)("native routine history status projection", () => { let db: ReturnType, database: Awaited>; const companyId = randomUUID(), agentId = randomUUID(); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("native-routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); await db.insert(companies).values({ id: companyId, name: "Routine status fixture", issuePrefix: "NRS" }); await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", adapterType: "paperclip_runner", status: "running" }); }, 60_000); diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 202d2b4eac3..7112a6afcff 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -401,7 +401,7 @@ async function recordRetryableFailure(input: { if (phase === "terminal_failure" && failureCode === "native_workspace_sync_out_unsafe_archive") { await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); } - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: input.run.companyId }); await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); await tx .update(nativeRunFinalizations) @@ -1659,7 +1659,7 @@ export async function finalizeNativeRun(input: { if (input.projectRunStatus) await notifyCancelledWorkspacePreservation(tx, { companyId: run.companyId, runId: run.id }); if (input.projectRunStatus) await notifyUnsafeWorkspaceExportRunChange(tx, run); if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); const [after] = await projectRun(tx); return after ? { before, after } : null; diff --git a/server/src/services/native-runtime/native-run-projection.ts b/server/src/services/native-runtime/native-run-projection.ts index 5708272da2c..2542b8bb1e3 100644 --- a/server/src/services/native-runtime/native-run-projection.ts +++ b/server/src/services/native-runtime/native-run-projection.ts @@ -138,7 +138,7 @@ export async function projectCommittedRun(input: { await notifyCancelledWorkspacePreservation(tx, { companyId: input.run.companyId, runId: input.run.id }); await notifyUnsafeWorkspaceExportRunChange(tx, input.run); await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: input.run.companyId }); await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); const [after] = await tx .update(heartbeatRuns) diff --git a/server/src/services/native-runtime/native-runner-file-handoff.test.ts b/server/src/services/native-runtime/native-runner-file-handoff.test.ts index 0abae30539f..e302c432190 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.test.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js"; import { nativeCompletionFeedback } from "./native-completion-feedback.js"; import { createHash, randomUUID } from "node:crypto"; @@ -61,10 +62,9 @@ describe("native runner file handoff", () => { const runId = "00000000-0000-4000-8000-000000009104"; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase( - "native-runner-file-handoff-", - ); - db = createDb(temporary.connectionString); + const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_HANDOFF_TEST_DATABASE_URL"); + if (!external) temporary = await startEmbeddedPostgresTestDatabase("native-runner-file-handoff-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 }); temporaryRoot = await mkdtemp( path.join(tmpdir(), "paperclip-native-file-handoff-"), ); diff --git a/server/src/services/native-runtime/native-runner-file-handoff.ts b/server/src/services/native-runtime/native-runner-file-handoff.ts index 6f2c32beec8..420380500ea 100644 --- a/server/src/services/native-runtime/native-runner-file-handoff.ts +++ b/server/src/services/native-runtime/native-runner-file-handoff.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "../issue-recovery-notifications.js"; import { createHash, randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import { constants } from "node:fs"; @@ -1265,7 +1266,9 @@ export async function prepareNativeRunnerFileHandoff(input: { ) { throw new Error("paperclip_runner_file_handoff_origin_not_persisted"); } - await input.db + await input.db.transaction(async tx => { + await notifyIssueRecovery(tx, { companyId: input.binding.companyId }); + await tx .update(issueWorkProducts) .set({ title: verified.title, updatedAt: new Date() }) .where( @@ -1276,6 +1279,7 @@ export async function prepareNativeRunnerFileHandoff(input: { eq(issueWorkProducts.createdByRunId, input.binding.runId), ), ); + }); const comment = await issueService(input.db).addComment( input.binding.issueId, `Prepared ${verified.title} for this response.`, diff --git a/server/src/services/native-runtime/native-safe-replacement.ts b/server/src/services/native-runtime/native-safe-replacement.ts index 7fb789f3304..da485883cb0 100644 --- a/server/src/services/native-runtime/native-safe-replacement.ts +++ b/server/src/services/native-runtime/native-safe-replacement.ts @@ -267,7 +267,7 @@ export async function reconcileSafeNativeReplacements( current.failureDetail?.replacementDenied ) return; - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx .update(nativeRunFinalizations) .set({ @@ -409,7 +409,7 @@ export async function reconcileSafeNativeReplacements( idempotencyKey: `native-safe-replacement:${run.id}`, }) .returning(); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: run.companyId }); await tx.insert(heartbeatRuns).values({ id: successorRunId, companyId: run.companyId, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 4dabeda6954..8fa7964005c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -4302,6 +4302,17 @@ describe("retained native cleanup activation", () => { releaseCommit = release; }); const db = { + execute: async (query: SQL) => { + // Retention hints read the fixture's run/issue relationship inside the + // same cleanup transaction; this fixture has one issue and no ancestors. + const statement = new PgDialect().sqlToQuery(query); + expect(transactionOpen).toBe(true); + expect(statement.params).toContain(run.companyId); + if (statement.sql.includes("from heartbeat_runs")) return [{ id: run.nativeIssueId }]; + if (statement.sql.includes("origin_kind")) return [{ id: run.nativeIssueId, originKind: null, originId: null }]; + if (statement.sql.includes("with recursive affected")) return [{ issueId: run.nativeIssueId }]; + throw new Error(`Unexpected cleanup fixture query: ${statement.sql}`); + }, select: () => ({ from: (table: unknown) => { const rows = @@ -5103,7 +5114,10 @@ describe("retained native cleanup activation", () => { } const outcome = await pendingOutcome; if (mode === "legacy_busy_copy") { - expect(maintenanceReleased).toHaveBeenCalledExactlyOnceWith({ companyId: run.companyId, runId: run.id }); + expect(maintenanceReleased).toHaveBeenCalledExactlyOnceWith( + { companyId: run.companyId, runId: run.id }, + { companyId: run.companyId, runId: run.id }, + ); unsubscribeMaintenance?.(); } if (canonicalSource) { @@ -6425,7 +6439,10 @@ describe("native startup cancellation fence", () => { })).rejects.toThrow("native_cancellation_pending_recovery"); expect(cancel).toHaveBeenCalledOnce(); expect(submit).not.toHaveBeenCalled(); - expect(released).toHaveBeenCalledWith({ companyId: execution.binding.companyId, agentId: execution.binding.agentId }); + expect(released).toHaveBeenCalledWith( + { companyId: execution.binding.companyId, agentId: execution.binding.agentId }, + { companyId: execution.binding.companyId, runId: execution.binding.runId, issueId: execution.binding.issueId }, + ); } finally { unsubscribe(); } }); }); @@ -9872,9 +9889,10 @@ describe("native warm session supervision", () => { ); expect(released).not.toHaveBeenCalled(); releaseClose(); - await vi.waitFor(() => expect(released).toHaveBeenCalledExactlyOnceWith({ - companyId: base.binding.companyId, agentId: base.binding.agentId, - })); + await vi.waitFor(() => expect(released).toHaveBeenCalledExactlyOnceWith( + { companyId: base.binding.companyId, agentId: base.binding.agentId }, + { companyId: base.binding.companyId, runId: base.binding.runId, issueId: base.binding.issueId }, + )); } finally { releaseClose(); unsubscribe(); } }); @@ -12453,7 +12471,10 @@ describe("runnerd provider runtime wiring", () => { releaseEnvironment(); releaseAssigned(); const result = await preparation; - expect(released).toHaveBeenCalledExactlyOnceWith({ companyId: execution.binding.companyId, agentId: execution.binding.agentId }); + expect(released).toHaveBeenCalledExactlyOnceWith( + { companyId: execution.binding.companyId, agentId: execution.binding.agentId }, + { companyId: execution.binding.companyId, runId: execution.binding.runId, issueId: execution.binding.issueId }, + ); if (failure !== "success") { expect(result).toMatchObject({ error: { message: failure === "assigned" ? "assigned failed" : "environment failed" } }); expect(state.createBackend).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index aa8e8f7fcd0..d68bce7191d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { rehydratePinnedSkillAssets } from "./runtime-context.js"; import { createNativeUsageDelivery } from "./native-usage-delivery.js"; import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; @@ -8091,7 +8092,8 @@ export async function executePaperclipNativeSession(input: { } releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }); + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }); } } @@ -9473,7 +9475,8 @@ async function executePaperclipNativeSessionWithinScope( warmNativeSessions.set(warmSessionId, { onReleased: () => releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }), + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }), pluginTaskId: usesPluginTask && input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox" ? input.runnerExecutionTarget.sandboxLeaseAcquisition?.providerLeaseId : undefined, runId: input.execution.binding.runId, @@ -9691,7 +9694,7 @@ async function executePaperclipNativeSessionWithinScope( eq(nativeRunFinalizations.runId, input.execution.binding.runId), eq(nativeRunFinalizations.leaseOwner, leaseOwner), ), - )); + ), { companyId: input.execution.binding.companyId }); const failedAtMs = Date.now(); const executionFailureMessage = redactIdentityText( @@ -9743,7 +9746,7 @@ async function executePaperclipNativeSessionWithinScope( eq(nativeRunFinalizations.leaseOwner, leaseOwner), eq(nativeRunFinalizations.attempt, attempt), isNull(nativeRunFinalizations.resultId), - )).returning({ runId: nativeRunFinalizations.runId })); + )).returning({ runId: nativeRunFinalizations.runId }), { companyId: input.execution.binding.companyId }); if (settled) { await stoppedLeaseRenewal; if (warmSessionId !== null && lifecyclePolicy.mode === "warm") { @@ -9830,7 +9833,7 @@ async function executePaperclipNativeSessionWithinScope( ), ) .for("update"); - await notifyExecutionRunWork(tx); + await notifyExecutionRunWork(tx, { companyId: input.execution.binding.companyId }); await notifyNativeFinalizationWork(tx, input.execution.binding); const updated = await tx .update(nativeRunFinalizations) @@ -11814,7 +11817,8 @@ export async function createRunnerdBackend(input: { initializingSessionToolAuthorities.delete(sessionScopeId); releaseRetainedNativeSessionCleanup(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, - }); + }, { companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + issueId: input.execution.binding.issueId }); } } @@ -12859,7 +12863,7 @@ async function createRunnerdBackendWithinSessionClaim( }) .where(eq(environmentLeases.id, leaseId)) .returning({ id: environmentLeases.id }) - .then((rows) => rows[0] ?? null), { companyId: leaseRow.companyId, runId: leaseRow.heartbeatRunId ?? undefined }); + .then((rows) => rows[0] ?? null), { companyId: leaseRow.companyId, runId: leaseRow.heartbeatRunId ?? undefined, leaseId }); if (!updated) throw new Error("runner_harness_backup_lease_missing"); }; @@ -13278,40 +13282,15 @@ async function createRunnerdBackendWithinSessionClaim( }); renameSync(temporaryManifest, manifestPath); - const currentRoot = resolve(backupRoot, "current"); - const previousRoot = resolve(backupRoot, "previous"); - removeNativeHarnessBackup(previousRoot); - let movedCurrent = false; - if (existsSync(currentRoot)) { - renameSync(currentRoot, previousRoot); - movedCurrent = true; - } - try { - renameSync(pendingRoot, currentRoot); - if ( - remoteTarget?.transport === "sandbox" && - remoteTarget.leaseId - ) { + await publishNativeHarnessBackup({ + db: input.db, companyId: input.execution.binding.companyId, + scopeDigest: createHash("sha256").update(sessionScopeId).digest("hex"), backupRoot, pendingRoot, + stamp: remoteTarget?.transport === "sandbox" && remoteTarget.leaseId ? async currentRoot => { await recordHarnessBackupStampForCurrentLease({ - root: currentRoot, - manifest, - bytes: backupSpanAttributes.bytesTransferred, + root: currentRoot, manifest, bytes: backupSpanAttributes.bytesTransferred, }); - } - } catch (error) { - if (existsSync(currentRoot)) { - removeNativeHarnessBackup(currentRoot); - } - if ( - movedCurrent && - existsSync(previousRoot) && - !existsSync(currentRoot) - ) { - renameSync(previousRoot, currentRoot); - } - throw error; - } - removeNativeHarnessBackup(previousRoot); + } : undefined, + }); } finally { removeNativeHarnessBackup(pendingRoot); } diff --git a/server/src/services/native-runtime/native-superseded-retry.ts b/server/src/services/native-runtime/native-superseded-retry.ts index e98672826b0..1bb6df9bd50 100644 --- a/server/src/services/native-runtime/native-superseded-retry.ts +++ b/server/src/services/native-runtime/native-superseded-retry.ts @@ -39,7 +39,7 @@ export async function retireSupersededNativeRetry(db: Db, input: { )).limit(1); if (!source) return false; - await notifyExecutionRunWork(db); + await notifyExecutionRunWork(db, { companyId: run.companyId }); await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", nextAttemptAt: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: "blocked", failureCode: "native_retry_superseded", failureDetail: { ...coordinator.failureDetail, diff --git a/server/src/services/native-runtime/native-workspace-export-recovery.ts b/server/src/services/native-runtime/native-workspace-export-recovery.ts index 41e51f2ddc7..0c3aac34799 100644 --- a/server/src/services/native-runtime/native-workspace-export-recovery.ts +++ b/server/src/services/native-runtime/native-workspace-export-recovery.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { alias } from "drizzle-orm/pg-core"; import { notifyUnsafeWorkspaceExportRecovery, type UnsafeWorkspaceExportScope } from "../unsafe-workspace-export-notifications.js"; @@ -107,6 +108,7 @@ async function recoverCandidate(db: Db, candidate: { runId: string; companyId: s nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...run.resultJson, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now, }).where(eq(heartbeatRuns.id, run.id)).returning(); + await notifyStatusCardIssueChange(tx, run.companyId); await tx.update(issues).set({ executionRunId: run.id, updatedAt: now }).where(eq(issues.id, issue.id)); await appendHeartbeatRunEvent(tx as unknown as Db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, eventType: "workspace_export_omitted", stream: "system", level: "info", diff --git a/server/src/services/native-runtime/native-workspace-export-retry.ts b/server/src/services/native-runtime/native-workspace-export-retry.ts index 47de8c73cf4..5a79c9e782e 100644 --- a/server/src/services/native-runtime/native-workspace-export-retry.ts +++ b/server/src/services/native-runtime/native-workspace-export-retry.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; import { randomUUID } from "node:crypto"; @@ -141,6 +142,7 @@ export async function retryNativeWorkspaceExport(input: { await tx.update(heartbeatRuns).set({ status: "running", finishedAt: null, error: null, errorCode: null, nativePhase: "result_accepted", nativePhaseUpdatedAt: now, resultJson: { ...current.run.resultJson, workspaceExportRetry: retry, finalizationPhase: "result_accepted", failureCode: null, originalFailureCode: null, nextAttemptAt: null }, updatedAt: now }).where(eq(heartbeatRuns.id, current.run.id)); + await notifyStatusCardIssueChange(tx, current.run.companyId); await tx.update(issues).set({ executionRunId: current.run.id, updatedAt: now }).where(eq(issues.id, current.issue.id)); // Reclaim the same lease only for control-plane copyback. The normal // finalization settlement will stop/retain it again on success or failure. diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index 59270a2bb65..53063f345e9 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -131,7 +131,7 @@ describe("native workspace finalization recovery", () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_WORKSPACE_FINALIZER_TEST_DATABASE_URL"); if (!external) temporary = await startEmbeddedPostgresTestDatabase("paperclip-native-workspace-finalizer-"); connectionString = external ?? temporary!.connectionString; - db = createDb(connectionString); + db = createDb(connectionString, { maxConnections: 6 }); await db.insert(companies).values([ { id: companyId, name: "Native workspace owner", issuePrefix: "NWO" }, @@ -452,7 +452,7 @@ describe("native workspace finalization recovery", () => { const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Live copyback" }); const live = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async (ownership) => { await ownership.assertHeld(); - expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString), runId: seeded.runId })).toBeNull(); + expect(await resumeNativeWorkspaceFinalization({ db: createDb(connectionString, { maxConnections: 6 }), runId: seeded.runId })).toBeNull(); expect(await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, seeded.runId))).toEqual([]); return "copied"; }); @@ -476,7 +476,7 @@ describe("native workspace finalization recovery", () => { try { await entered; await db.execute(sql`select pg_terminate_backend(pid) from pg_locks where locktype = 'advisory' and database = (select oid from pg_database where datname = current_database()) and pid <> pg_backend_pid()`); - const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "must not run"); + const second = await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "must not run"); expect(second).toEqual({ acquired: false, reason: "busy" }); const [held] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.runId)); expect(held.runnerProfileJson?.nativeWorkspaceFinalizationOwner).toBeTruthy(); @@ -489,7 +489,7 @@ describe("native workspace finalization recovery", () => { it("recovers its own joined copyback receipt after the application pool disconnects during cleanup", async () => { const seeded = await seedRun({ executionWorkspaceId: randomUUID(), title: "Reconnect joined copyback" }); - const interruptedDb = createDb(connectionString); + const interruptedDb = createDb(connectionString, { maxConnections: 6 }); let joined = false; await expect(withNativeWorkspaceFinalizationOwnership({ db: interruptedDb, companyId, runId: seeded.runId }, async () => { await fs.writeFile(path.join(workspaceRoot, "joined-copyback.txt"), "durable work"); @@ -507,7 +507,7 @@ describe("native workspace finalization recovery", () => { .toEqual({ acquired: false, reason: "owner_unverified" }); } await db.update(heartbeatRuns).set({ runnerProfileJson: beforeReconnect.runnerProfileJson }).where(eq(heartbeatRuns.id, seeded.runId)); - expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString), companyId, runId: seeded.runId }, async () => "resumed")) + expect(await withNativeWorkspaceFinalizationOwnership({ db: createDb(connectionString, { maxConnections: 6 }), companyId, runId: seeded.runId }, async () => "resumed")) .toEqual({ acquired: true, value: "resumed" }); }); diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.test.ts b/server/src/services/native-runtime/native-workspace-reconstruction.test.ts index fcff7eb58ec..8afb9820bee 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.test.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.test.ts @@ -38,7 +38,7 @@ describe("reconstruction after confirmed native workspace loss", () => { const external = resolveExternalTestDatabaseUrl("PAPERCLIP_NATIVE_RECONSTRUCTION_TEST_DATABASE_URL"); if (external) { await applyPendingMigrations(external); - db = createDb(external); + db = createDb(external, { maxConnections: 6 }); return; } temporary = await startEmbeddedPostgresTestDatabase("workspace-reconstruction-"); @@ -144,17 +144,21 @@ describe("reconstruction after confirmed native workspace loss", () => { expect(await queue.nextTimerHeartbeatAt(agent)).toBeNull(); const events: string[] = []; const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.agentTimers, event => events.push(event)); + const routineEvents: string[] = []; + const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event)); try { await db.transaction(async tx => { await reconstructNativeWorkspace({ ...f, db: tx as unknown as Db }); expect(events).toContain("intent"); expect(events).not.toContain("settled"); + expect(routineEvents).toContain("intent"); expect(routineEvents).not.toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("blocked"); }); expect(events).toContain("settled"); + expect(routineEvents).toContain("settled"); expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("todo"); expect(await queue.nextTimerHeartbeatAt(agent)).toBe(agent.createdAt.getTime() + 600_000); - } finally { unsubscribe(); } + } finally { unsubscribe(); unsubscribeRoutine(); } }); it("pins a historical cancelled workspace through public heartbeat startup without replaying its turn", async () => { diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 5273b30216a..bc34cc3bb3c 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -1,3 +1,6 @@ +import { notifyRoutineExecutionStatus } from "../routine-execution-status-notifications.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "../unsafe-workspace-export-notifications.js"; import { and, eq, gt, inArray, ne, notInArray, or, sql } from "drizzle-orm"; @@ -179,6 +182,7 @@ export async function reconstructNativeWorkspace(input: { ...(source.interveningExecutions.length ? { interveningExecutions: source.interveningExecutions } : {}) } }; await markExecutionReconciliation(db, source.action, decision, input.actorId); const now = new Date(); + await notifyAttentionRetentionWork(tx, { companyId: input.companyId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: input.companyId, issueId: input.issueId }); await tx.update(issueRecoveryActions).set({ status: "resolved", outcome: "handed_back", resolvedAt: now, updatedAt: now, resolutionNote: "workspace_reconstruction_authorized", nextAction: "Continue from saved sources. The original unexported edits are lost; verify the selected attachments before rebuilding. Do not repeat completed external actions or reset the original work limits.", @@ -187,6 +191,8 @@ export async function reconstructNativeWorkspace(input: { }).where(eq(issueRecoveryActions.id, source.action.id)); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyRoutineExecutionStatus(tx, input.issueId, input.companyId); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ status: "todo", unblockDescriptor: null, blockedTransitionAt: null, blockedOwnerNotifiedAt: null, updatedAt: now }).where(eq(issues.id, issue.id)); const publication = await persistActivity(db, { companyId: input.companyId, actorType: "user", actorId: input.actorId, diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts index 02917045861..6f357598dbf 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.ts @@ -1,4 +1,6 @@ +import { notifyStatusCardIssueChange } from "../status-card-notifications.js"; import { nativeRunKeyScope, nativeSkillTestToolAllowed, assertNativeSkillTestTool } from "./native-skill-test-scope.js"; +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { requiredQualityReviewSchema } from "@paperclipai/shared"; import { priorRecoveryApiReceipt } from "../recovery-effect-receipts.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -2067,8 +2069,10 @@ export class PaperclipRunnerToolAuthority { replayMonitor = summarizeIssueMonitor(target, normalizeIssueExecutionPolicy(target.executionPolicy)); return { taskId: target.id, status: target.status, monitor: replayMonitor }; } + await notifyAttentionRetentionWork(tx, { companyId: this.binding.companyId }); const patch = prepareIssueMonitorUpdate(target, input.monitor, actor); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); + await notifyStatusCardIssueChange(tx, this.binding.companyId); const [updated] = await tx.update(issues).set({ ...patch, updatedAt: new Date() }) .where(and(eq(issues.id, target.id), eq(issues.companyId, this.binding.companyId))).returning(); if (!updated) throw notFound("Task not found"); diff --git a/server/src/services/native-runtime/runner-api.integration.test.ts b/server/src/services/native-runtime/runner-api.integration.test.ts index 4e1831b734a..fdb5b7f70a2 100644 --- a/server/src/services/native-runtime/runner-api.integration.test.ts +++ b/server/src/services/native-runtime/runner-api.integration.test.ts @@ -497,7 +497,7 @@ else if(m.id!==undefined) send({id:m.id,result:{}}); const [agent] = await server.db.select().from(agents).where(eq(agents.id, fixture.agentId)); const context = structuredClone(run.contextSnapshot ?? {}); await prepareHeartbeatTaskContext(server.db, { run, agent, - task: { issueRef: issue, issueContext: null, wakeCommentContext: null, context, taskSession: null }, + task: { issueRef: issue, issueContext: null, context, taskSession: null }, policy: { recovering: true, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} }, services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: createHeartbeatRunPreparation(server.db).getPinnedSkillTestContext }, diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index a43868dd25f..3ea3096b710 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,3 +1,4 @@ +import { notifyNativeStatusWakeWork } from "../deferred-execution-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { lockCurrentNativeWorkAssessmentEvidence } from "./work-assessments.js"; import { hasNativeWorkspaceFinalizationOwner } from "./native-workspace-finalization-recovery-state.js"; @@ -478,6 +479,7 @@ async function enqueueWake(input: { .limit(1) .then((rows) => rows[0] ?? null); if (existing) return existing.id; + await notifyNativeStatusWakeWork(input.tx, { companyId: input.companyId }); const inserted = await input.tx .insert(agentWakeupRequests) .values({ diff --git a/server/src/services/native-runtime/superseded-external-review-requests.ts b/server/src/services/native-runtime/superseded-external-review-requests.ts index f04db57f7d7..9697b436ed3 100644 --- a/server/src/services/native-runtime/superseded-external-review-requests.ts +++ b/server/src/services/native-runtime/superseded-external-review-requests.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "../retention-work-notifications.js"; import { and, eq, isNull, ne, sql } from "drizzle-orm"; import { completionContracts, heartbeatRuns, issues, issueThreadInteractions, statusDecisionEffects, statusDecisions, workAssessments, type Db } from "@paperclipai/db"; import { hasPublishedGitHubReview } from "../chat-channels/github/review-wait.js"; @@ -70,6 +71,7 @@ export async function supersedeExternalReviewRequestsInTransaction( publications: ActivityPublication[], ) { for (const { interaction } of candidates) { + await notifyAttentionRetentionWork(tx, { companyId: binding.companyId }); const reason = "A later published review no longer requests human input."; const [retired] = await tx.update(issueThreadInteractions).set({ status: "cancelled", result: { version: 1, outcome: "withdrawn", reason }, resolvedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/native-status-reassessment-notifications.ts b/server/src/services/native-status-reassessment-notifications.ts index fb845523bb4..c49f600ce5f 100644 --- a/server/src/services/native-status-reassessment-notifications.ts +++ b/server/src/services/native-status-reassessment-notifications.ts @@ -1,8 +1,14 @@ +import { notifyIssueAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type NativeStatusReassessmentScope = { companyId: string; issueId?: string; runId?: string; agentId?: string }; -export function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { +export async function notifyNativeStatusReassessment(tx: object, scope: NativeStatusReassessmentScope) { + await notifyIssueAttentionRetentionWork(tx, { companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId }); + await notifyStatusCardIssueChange(tx, scope.companyId); + await notifyIssueRecovery(tx, scope ? { companyId: scope.companyId } : undefined); return notifyDeliveryWork(tx, DELIVERY_QUEUES.nativeStatusReassessment, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId, runId: scope.runId, agentId: scope.agentId })); } diff --git a/server/src/services/plugin-managed-agents.ts b/server/src/services/plugin-managed-agents.ts index e2e2c965e8c..91a81fb5120 100644 --- a/server/src/services/plugin-managed-agents.ts +++ b/server/src/services/plugin-managed-agents.ts @@ -1,3 +1,4 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; import { createAgentLifecycle } from "./agent-lifecycle.js"; import { and, eq, isNull, ne } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -340,6 +341,7 @@ export function pluginManagedAgentService( // and its canonical history. Transfer only its proven stable-key marker to // the replacement installation, atomically with the new scoped bindings. await upsertBinding(companyId, declaration, agentId, {}, adapterType, tx); + await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" }); await tx.update(agents).set({ metadata: managedMetadata(options.pluginId, options.pluginKey, declaration, agent.metadata), updatedAt: new Date(), diff --git a/server/src/services/plugin-registry.ts b/server/src/services/plugin-registry.ts index 239c3c50490..62d9b7cde70 100644 --- a/server/src/services/plugin-registry.ts +++ b/server/src/services/plugin-registry.ts @@ -1,3 +1,4 @@ +import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js"; import { isDeepStrictEqual } from "node:util"; import { writePluginJobChange } from "./plugin-job-work-notifications.js"; import { asc, eq, isNull, ne, sql, and } from "drizzle-orm"; @@ -239,12 +240,15 @@ export function pluginRegistryService(db: Db) { setClause.categories = data.manifest.categories; } - return db - .update(plugins) - .set(setClause) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set(setClause) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Status --------------------------------------------------------- @@ -254,16 +258,19 @@ export function pluginRegistryService(db: Db) { const plugin = await getById(id); if (!plugin) throw notFound("Plugin not found"); - return db - .update(plugins) - .set({ - status: input.status, - lastError: input.lastError ?? null, - updatedAt: new Date(), - }) - .where(eq(plugins.id, id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.transaction(async tx => { + await notifyExternalObjectRefresh(tx); + return tx + .update(plugins) + .set({ + status: input.status, + lastError: input.lastError ?? null, + updatedAt: new Date(), + }) + .where(eq(plugins.id, id)) + .returning() + .then((rows) => rows[0] ?? null); + }); }, // ----- Uninstall / Remove -------------------------------------------- @@ -280,6 +287,7 @@ export function pluginRegistryService(db: Db) { if (!plugin) throw notFound("Plugin not found"); return writePluginJobChange(db, async db => { + await notifyExternalObjectRefresh(db); if (removeData) { // Hard delete – plugin_config cascades via FK onDelete return db @@ -343,6 +351,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -385,6 +394,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); @@ -467,6 +477,7 @@ export function pluginRegistryService(db: Db) { return db.transaction(async tx => { const db = tx as unknown as Db; await notifyDeliveryWork(db, DELIVERY_QUEUES.pluginLifecycle); + await notifyExternalObjectRefresh(db, companyId); const plugin = await getById(pluginId, db); if (!plugin) throw notFound("Plugin not found"); diff --git a/server/src/services/plugin-workspaces.ts b/server/src/services/plugin-workspaces.ts index e2383dcfc74..ca04b081e48 100644 --- a/server/src/services/plugin-workspaces.ts +++ b/server/src/services/plugin-workspaces.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { effectiveProjectWorkspacePolicy, isWorkspaceProvider } from "./project-workspace-provider.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { randomUUID, createHash } from "node:crypto"; @@ -418,6 +419,7 @@ export async function preparePluginWorkspace( }, }) .returning(); + await notifyStatusCardIssueChange(tx, task.companyId); await tx .update(issues) .set({ diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 28cae661e9b..45891a218a6 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyChatInboundWakeNotices } from "./chat-inbound-wake-notice-notifications.js"; import { listProjectWorkspaceProviders, resolveProjectWorkspaceProvider } from "./project-workspace-provider.js"; import { retireRemovedPluginWorkspaceDependents } from "./plugin-workspaces.js"; @@ -1013,6 +1014,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool const row = await db.transaction(async (tx) => { if (updates.visibility !== undefined) { + await notifyStatusCardIssueChange(tx, existingProject.companyId); await notifyChatInboundWakeNotices(tx, { companyId: existingProject.companyId }); await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`issue-privacy-tree:${existingProject.companyId}`}, 0))`); } @@ -1086,6 +1088,7 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update"); await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); + await notifyStatusCardIssueChange(tx, project.companyId); const [row] = await tx.delete(projects).where(eq(projects.id, id)).returning(); return row ? { ...row, urlKey: deriveProjectUrlKey(row.name, row.id) } : null; }), diff --git a/server/src/services/provider-fresh-workspace.ts b/server/src/services/provider-fresh-workspace.ts index fcc184e15f4..98a478a375d 100644 --- a/server/src/services/provider-fresh-workspace.ts +++ b/server/src/services/provider-fresh-workspace.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { assertTaskWorkspaceExecutionAccess } from "./task-workspace-access.js"; import { isDeepStrictEqual } from "node:util"; import { createHash, randomUUID } from "node:crypto"; @@ -152,6 +153,7 @@ export async function reserveFreshPluginTaskWorkspace(input: { providerType: "adapter_managed", cwd: `/workspaces/${input.fresh.attemptId}`, metadata: { runnerTaskFreshWorkspace: input.fresh, runnerTaskWorkspaceOwner: input.owner }, }); + await notifyStatusCardIssueChange(tx, input.companyId); await tx.update(issues).set({ executionWorkspaceId: input.fresh.attemptId, executionWorkspacePreference: "reuse_existing", workspaceBindingRevision: task.workspaceBindingRevision + 1, workspacePendingSelection: task.workspacePendingSelection?.expectedBindingRevision === task.workspaceBindingRevision diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 991fa8f63b5..4b2ec5c0672 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,4 @@ +import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeStatusReassessment } from "../native-status-reassessment-notifications.js"; @@ -26,7 +27,7 @@ import { decideLegacyContinuation, legacyDispositionEpisode, legacyDispositionFingerprint, LEGACY_DISPOSITION_REPAIR_INSTRUCTION, type LegacyDispositionEpisode, } from "./legacy-continuation.js"; -import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController } from "../legacy-controller-lease.js"; +import { CONTROLLER_LEASE_LOST_FAILURE_CODE, hasLiveLegacyController, liveLegacyControllerExpiresAt } from "../legacy-controller-lease.js"; import { isCompletedOnboardingHandoffWake } from "../chat-completion-delivery.js"; import { instanceSettingsService } from "../instance-settings.js"; import { isWaitingConversation, settleConversationTurnForRun, deliverConversationComments } from "../agent-conversations.js"; @@ -244,6 +245,7 @@ type ResolvedDependencyWakeBackstopOptions = { runId?: string | null; companyId?: string | null; blockerIssueId?: string | null; + issueId?: string; source?: ResolvedDependencyWakeBackstopSource; }; @@ -2201,7 +2203,7 @@ export function recoveryService( return Boolean(budgetBlock); } - async function reconcileUnassignedBlockingIssues() { + async function reconcileUnassignedBlockingIssues(opts?: { companyId?: string }) { const candidates = await db .select({ id: issues.id, @@ -2215,6 +2217,8 @@ export function recoveryService( .where( and( eq(issueRelations.type, "blocks"), + eq(issueRelations.companyId, issues.companyId), + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId), @@ -2716,6 +2720,7 @@ export function recoveryService( const now = new Date(); const retryAt = readProviderQuotaRetryAt(input.latestRun, now); return database.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3316,6 +3321,7 @@ export function recoveryService( created = Boolean(enqueuedRun); } else { scheduledRun = await db.transaction(async (tx) => { + await notifyRunDispatch(tx, { companyId: input.issue.companyId }); const wakeup = await tx .insert(agentWakeupRequests) .values({ @@ -3497,7 +3503,7 @@ export function recoveryService( return Boolean(run || wake); } - async function reconcileActiveRecoveryActions() { + async function reconcileActiveRecoveryActions(opts?: { companyId?: string }) { const rows = await db .select({ action: issueRecoveryActions, @@ -3511,7 +3517,8 @@ export function recoveryService( eq(issues.companyId, issueRecoveryActions.companyId), ), ) - .where(inArray(issueRecoveryActions.status, ["active", "escalated"])); + .where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + opts?.companyId ? eq(issueRecoveryActions.companyId, opts.companyId) : undefined)); const result = { requeued: 0, @@ -4523,14 +4530,20 @@ export function recoveryService( } async function reconcileStrandedAssignedIssues(opts?: { + companyId?: string; + issueId?: string; + onDeferredDeadline?: (at: number) => void; issueCreatedAtGte?: Date | null; skipExternalConversationSettlement?: boolean; + skipRelatedRecovery?: boolean; }) { const candidates = await db .select(recoveryIssueFields) .from(issues) .where( and( + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, + opts?.issueId ? eq(issues.id, opts.issueId) : undefined, isNull(issues.assigneeUserId), inArray(issues.status, ["todo", "in_progress", "in_review"]), or( @@ -5711,6 +5724,7 @@ export function recoveryService( classification.baseBackoffMs * Math.pow(2, Math.max(0, consecutive - 1)); if (elapsed < requiredDelay) { + opts?.onDeferredDeadline?.(latestFinishedAt.getTime() + requiredDelay); result.skipped += 1; continue; } @@ -5770,16 +5784,18 @@ export function recoveryService( } } - const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(); - result.orphanBlockersAssigned = orphanBlockerRecovery.assigned; - result.skipped += orphanBlockerRecovery.skipped; - result.issueIds.push(...orphanBlockerRecovery.issueIds); + if (!opts?.skipRelatedRecovery) { + const orphanBlockerRecovery = await reconcileUnassignedBlockingIssues(opts); + result.orphanBlockersAssigned = orphanBlockerRecovery.assigned; + result.skipped += orphanBlockerRecovery.skipped; + result.issueIds.push(...orphanBlockerRecovery.issueIds); - const activeRecovery = await reconcileActiveRecoveryActions(); - result.continuationRequeued += activeRecovery.requeued; - result.escalated += activeRecovery.escalated; - result.skipped += activeRecovery.skipped; - result.issueIds.push(...activeRecovery.issueIds); + const activeRecovery = await reconcileActiveRecoveryActions(opts); + result.continuationRequeued += activeRecovery.requeued; + result.escalated += activeRecovery.escalated; + result.skipped += activeRecovery.skipped; + result.issueIds.push(...activeRecovery.issueIds); + } result.issueIds = [...new Set(result.issueIds)]; return result; @@ -5812,7 +5828,7 @@ export function recoveryService( source === "workspace.finalize" ? "workspace_finalize_reconciliation" : "issue_graph_liveness_reconciliation"; - const useCursor = !opts?.blockerIssueId; + const useCursor = !opts?.blockerIssueId && !opts?.issueId; const queryCandidates = (afterIssueId: string | null) => { const filters = [ @@ -5822,6 +5838,7 @@ export function recoveryService( sql`${issues.assigneeAgentId} is not null`, ]; if (opts?.companyId) filters.push(eq(issues.companyId, opts.companyId)); + if (opts?.issueId) filters.push(eq(issues.id, opts.issueId)); if (afterIssueId) filters.push(gt(issues.id, afterIssueId)); if (opts?.blockerIssueId) { @@ -6112,6 +6129,7 @@ export function recoveryService( // fallback below. Without it, a terminal issue named in the run context // snapshot would still terminalize the shared run and defeat the guard. runReferencedByActiveIssue?: boolean; + onDeferredDeadline?: (at: number) => void; }, ): Promise<{ terminalized: boolean; status: string }> { // Act only on a run in "running" status. A "queued" run has no process yet, @@ -6127,8 +6145,11 @@ export function recoveryService( // Another controller may own a sandbox run whose PID has no meaning on // this host. Its live lease owns both execution and finalization, even if // the issue is already terminal or this process has no in-memory handle. - if (await hasLiveLegacyController(db, run)) + const controllerDeadline = await liveLegacyControllerExpiresAt(db, run); + if (controllerDeadline) { + options?.onDeferredDeadline?.(controllerDeadline.getTime()); return { terminalized: false, status: run.status }; + } const pid = run.processPid ?? null; const processGroupId = run.processGroupId ?? null; @@ -6349,7 +6370,7 @@ export function recoveryService( // still claims to be live but can no longer reach a terminal status on its // own, so a stuck "running" run can no longer block the sweep. Idempotent and // safe: clears at most one row's worth of lock columns per candidate. - async function sweepStaleIssueLocks() { + async function sweepStaleIssueLocks(opts?: { companyId?: string; issueId?: string; onDeferredDeadline?: (at: number) => void }) { const result = { cleared: 0, issueIds: [] as string[], @@ -6366,7 +6387,9 @@ export function recoveryService( }) .from(issues) .where( - sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`, + and(sql`(${issues.checkoutRunId} is not null or ${issues.executionRunId} is not null)`, + opts?.companyId ? eq(issues.companyId, opts.companyId) : undefined, + opts?.issueId ? eq(issues.id, opts.issueId) : undefined), ); const referencedRunIds = [ @@ -6381,7 +6404,8 @@ export function recoveryService( ? await db .select() .from(heartbeatRuns) - .where(inArray(heartbeatRuns.id, referencedRunIds)) + .where(and(inArray(heartbeatRuns.id, referencedRunIds), + opts?.companyId ? eq(heartbeatRuns.companyId, opts.companyId) : undefined)) : []; const runStatusById = new Map(); for (const row of runRows) runStatusById.set(row.id, row.status); @@ -6392,8 +6416,15 @@ export function recoveryService( // the same run id in a stale lock column. The terminal reference alone must // not terminalize a run that an active issue still owns, so exclude these // runs from the issue-terminal authority below. + const ownershipReferences = opts?.issueId && referencedRunIds.length > 0 + ? await db.select({ companyId: issues.companyId, status: issues.status, + checkoutRunId: issues.checkoutRunId, executionRunId: issues.executionRunId }).from(issues).where(and( + opts.companyId ? eq(issues.companyId, opts.companyId) : undefined, + or(inArray(issues.checkoutRunId, referencedRunIds), inArray(issues.executionRunId, referencedRunIds)), + )) + : candidates; const runIdsReferencedByActiveIssue = new Set(); - for (const issue of candidates) { + for (const issue of ownershipReferences) { if (issue.status === "done" || issue.status === "cancelled") continue; for (const runId of [issue.checkoutRunId, issue.executionRunId]) { if (runId) runIdsReferencedByActiveIssue.add(runId); @@ -6434,6 +6465,7 @@ export function recoveryService( referencingIssueTerminalStatus: issueTerminalStatusByRunId.get(row.id) ?? null, runReferencedByActiveIssue: runIdsReferencedByActiveIssue.has(row.id), + onDeferredDeadline: opts?.onDeferredDeadline, }); runStatusById.set(row.id, outcome.status); if (outcome.terminalized) result.terminalizedRunIds.push(row.id); @@ -6537,6 +6569,8 @@ export function recoveryService( recordWatchdogDecision, scanSilentActiveRuns, reconcileStrandedAssignedIssues, + reconcileUnassignedBlockingIssues, + reconcileActiveRecoveryActions, reconcileLegacyContinuation, legacyRepairDispatchBlock, sweepStaleIssueLocks, diff --git a/server/src/services/recovery/work.ts b/server/src/services/recovery/work.ts new file mode 100644 index 00000000000..407e4f1eb6d --- /dev/null +++ b/server/src/services/recovery/work.ts @@ -0,0 +1,113 @@ +import { registerTaskWatchdogWork } from "../task-watchdog-work.js"; +import { and, asc, eq, gt, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { issueRecoveryActions, issueRelations, issues, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, type DeliveryQueue } from "../delivery-work-notifications.js"; +import type { IssueRecoveryScope } from "../issue-recovery-notifications.js"; +import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; +type Scope = { companyId: string; issueId: string; onDeferredDeadline: (at: number) => void }; +const PAGE_SIZE = 100; + +export function registerIssueRecoveryWork( + db: Db, + coordinator: ReturnType, + owner: { + stranded(scope: Scope): Promise; + orphanBlockers(scope: { companyId: string }): Promise; + recoveryActions(scope: { companyId: string }): Promise; + dependencies(scope: Scope): Promise<{ enqueueFailed: number }>; + locks(scope: Scope): Promise; + watchdogs(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; + schedulingSuppressed(): Promise; + }, + canRun: () => boolean, +) { + function register(queue: DeliveryQueue, statuses: string[] | null, check: (scope: Scope) => Promise) { + const cursors = new Map(); + let enabled = false; + return registerScopedDeadlineWork(coordinator, { + queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || (entry.companyId === scope.companyId && (!scope.issueId || entry.id === scope.issueId)), + load: async scope => { + const key = scope ? JSON.stringify([scope.companyId, scope.issueId]) : "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = await db.select({ id: issues.id, companyId: issues.companyId }).from(issues).where(and( + scope ? eq(issues.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(issues.id, scope.issueId) : undefined, + cursor ? gt(issues.id, cursor) : undefined, + statuses ? inArray(issues.status, statuses) : or(isNotNull(issues.checkoutRunId), isNotNull(issues.executionRunId)), + statuses ? isNull(issues.hiddenAt) : undefined, + statuses ? isNull(issues.assigneeUserId) : undefined, + statuses ? or(isNotNull(issues.assigneeAgentId), eq(issues.status, "in_review")) : undefined, + )).orderBy(asc(issues.id)).limit(PAGE_SIZE); + const complete = rows.length < PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, + check: async entry => { + if (!enabled) return null; + let next: number | null = null; + await check({ companyId: entry.companyId, issueId: entry.id, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); + } + function registerCompany(queue: DeliveryQueue, kind: "blockers" | "actions", check: (scope: { companyId: string }) => Promise) { + const cursors = new Map(); + let enabled = false; + return registerScopedDeadlineWork(coordinator, { + queue, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = !(await owner.schedulingSuppressed()); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + load: async scope => { + const key = scope?.companyId ?? "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = kind === "actions" + ? await db.selectDistinct({ id: issueRecoveryActions.companyId, companyId: issueRecoveryActions.companyId }) + .from(issueRecoveryActions).where(and(inArray(issueRecoveryActions.status, ["active", "escalated"]), + scope ? eq(issueRecoveryActions.companyId, scope.companyId) : undefined, + cursor ? gt(issueRecoveryActions.companyId, cursor) : undefined)) + .orderBy(asc(issueRecoveryActions.companyId)).limit(PAGE_SIZE) + : await db.selectDistinct({ id: issues.companyId, companyId: issues.companyId }).from(issues).where(and( + inArray(issues.status, ["todo", "blocked"]), isNull(issues.assigneeAgentId), isNull(issues.assigneeUserId), + isNotNull(issues.createdByAgentId), + scope ? eq(issues.companyId, scope.companyId) : undefined, + cursor ? gt(issues.companyId, cursor) : undefined, + sql`exists (select 1 from ${issueRelations} r inner join ${issues} dependent + on dependent.id = r.related_issue_id and dependent.company_id = r.company_id + where r.company_id = ${issues.companyId} and r.issue_id = ${issues.id} + and r.type = 'blocks' and dependent.status not in ('done', 'cancelled'))`, + )).orderBy(asc(issues.companyId)).limit(PAGE_SIZE); + const complete = rows.length < PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, + check: async entry => { if (enabled) await check({ companyId: entry.companyId }); return null; }, + }, canRun); + } + const assignments = register(DELIVERY_QUEUES.strandedAssignments, ["todo", "in_progress", "in_review"], owner.stranded); + const orphanBlockers = registerCompany(DELIVERY_QUEUES.orphanBlockers, "blockers", owner.orphanBlockers); + const recoveryActions = registerCompany(DELIVERY_QUEUES.activeRecoveryActions, "actions", owner.recoveryActions); + const dependencies = register(DELIVERY_QUEUES.resolvedDependencies, ["blocked"], async scope => { + const result = await owner.dependencies(scope); + if (result.enqueueFailed > 0) throw new Error("Resolved dependency wake remains undelivered"); + }); + const locks = register(DELIVERY_QUEUES.staleIssueLocks, null, owner.locks); + const watchdogs = registerTaskWatchdogWork(db, coordinator, { + reconcile: owner.watchdogs, enabled: async () => !(await owner.schedulingSuppressed()), + }, canRun); + const workers = [assignments, orphanBlockers, recoveryActions, dependencies, locks, watchdogs]; + return { ready: Promise.all(workers.map(worker => worker.ready)), + wake() { for (const worker of workers) worker.invalidate(); }, + async stop() { await Promise.all(workers.map(worker => worker.stop())); } }; +} diff --git a/server/src/services/retained-native-session-cleanup-notifications.ts b/server/src/services/retained-native-session-cleanup-notifications.ts index 1ff39143e0e..e29665b3171 100644 --- a/server/src/services/retained-native-session-cleanup-notifications.ts +++ b/server/src/services/retained-native-session-cleanup-notifications.ts @@ -1,3 +1,4 @@ +import { notifyNativeHarnessBackupExpiration } from "./native-harness-backup-expiration-notifications.js"; import { notifySkillVersionRetention } from "./skill-version-retention-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -9,14 +10,15 @@ export type RetainedNativeSessionCleanupScope = { }; export async function notifyRetainedNativeSessionCleanup(tx: object, scope: RetainedNativeSessionCleanupScope) { + await notifyNativeHarnessBackupExpiration(tx, scope); await notifySkillVersionRetention(tx, scope.companyId); await notifyDeliveryWork(tx, DELIVERY_QUEUES.retainedNativeSessionCleanup, JSON.stringify({ companyId: scope.companyId, runId: scope.runId, issueId: scope.issueId, agentId: scope.agentId })); } -const listeners = new WeakMap void>>(); -export function onRetainedNativeSessionCleanupRelease(db: object, listener: (scope: RetainedNativeSessionCleanupScope) => void) { +const listeners = new WeakMap void>>(); +export function onRetainedNativeSessionCleanupRelease(db: object, listener: (scope: RetainedNativeSessionCleanupScope, source: RetainedNativeSessionCleanupScope) => void) { let owned = listeners.get(db); if (!owned) { owned = new Set(); listeners.set(db, owned); } owned.add(listener); @@ -25,6 +27,7 @@ export function onRetainedNativeSessionCleanupRelease(db: object, listener: (sco if (!owned.size && listeners.get(db) === owned) listeners.delete(db); }; } -export function releaseRetainedNativeSessionCleanup(db: object, scope: RetainedNativeSessionCleanupScope) { - for (const listener of listeners.get(db) ?? []) listener(scope); +export function releaseRetainedNativeSessionCleanup(db: object, scope: RetainedNativeSessionCleanupScope, + source: RetainedNativeSessionCleanupScope = scope) { + for (const listener of listeners.get(db) ?? []) listener(scope, source); } diff --git a/server/src/services/retention-work-notifications.ts b/server/src/services/retention-work-notifications.ts index ffedb0b1a4d..79cffd624a8 100644 --- a/server/src/services/retention-work-notifications.ts +++ b/server/src/services/retention-work-notifications.ts @@ -1,14 +1,89 @@ import { notifyApprovalWork } from "./approval-work-notifications.js"; +import { RECOVERY_ORIGIN_KINDS, parseIssueGraphLivenessIncidentKey } from "./recovery/origins.js"; +import { sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import type { AttentionSourceKind } from "@paperclipai/shared"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; -export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind) { +export const ATTENTION_RETENTION_MAX_ISSUE_IDS = 100; +export type AttentionRetentionScope = { companyId?: string; sourceKind?: AttentionSourceKind; + issueId?: string; issueIds?: string[]; sourceId?: string }; + +export async function notifyAttentionRetentionWork(transaction: object, scope: AttentionRetentionScope = {}) { + const issueIds = [...new Set([...(scope.issueIds ?? []), ...(scope.issueId ? [scope.issueId] : [])])].sort(); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.attentionRetention, JSON.stringify({ + companyId: scope.companyId, sourceKind: scope.sourceKind, sourceId: scope.sourceId, + issueId: issueIds.length === 1 ? issueIds[0] : undefined, + issueIds: issueIds.length > 1 && issueIds.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? issueIds : undefined, + })); +} + +export async function notifyIssueAttentionRetentionWork(transaction: object, + scope: { companyId: string; issueId?: string; parentId?: string | null; runId?: string; originKind?: string | null; originId?: string | null }) { + const database = transaction as Db; + const seeds = [scope.issueId, scope.parentId].filter((id): id is string => Boolean(id)); + if (!seeds.length && scope.runId) { + const rows = await database.execute(sql`select issue_id::text as id from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and issue_id is not null + union select native_issue_id::text from heartbeat_runs + where company_id = ${scope.companyId}::uuid and id = ${scope.runId}::uuid and native_issue_id is not null + union select issue.id::text from heartbeat_runs run inner join issues issue + on issue.company_id = run.company_id + and issue.id::text = coalesce(run.context_snapshot->>'issueId', run.context_snapshot->>'taskId') + where run.company_id = ${scope.companyId}::uuid and run.id = ${scope.runId}::uuid`); + for (const row of rows) seeds.push(row.id as string); + } + if (!seeds.length) return notifyAttentionRetentionWork(transaction, { companyId: scope.companyId }); + const origins = await database.execute(sql`select id::text as id, origin_kind as "originKind", origin_id as "originId" from issues + where company_id = ${scope.companyId}::uuid + and id in (${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)})`); + const current = origins.find(source => source.id === scope.issueId); + origins.push({ originKind: scope.originKind ?? current?.originKind, originId: scope.originId ?? current?.originId }); + const references: string[] = []; + for (const source of origins) { + if (source.originKind === RECOVERY_ORIGIN_KINDS.strandedIssueRecovery && typeof source.originId === "string") { + references.push(source.originId); + } + const incident = source.originKind === RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation + ? parseIssueGraphLivenessIncidentKey(source.originId as string | null) : null; + if (incident?.companyId === scope.companyId) references.push(incident.issueId, incident.leafIssueId); + } + if (references.length) { + const trusted = await database.execute(sql`select id::text as id from issues + where company_id = ${scope.companyId}::uuid + and id::text in (${sql.join([...new Set(references)].map(id => sql`${id}`), sql`, `)})`); + for (const row of trusted) seeds.push(row.id as string); + } + // UNION deduplicates visited UUIDs (including cycles). Each relation edge is + // unique by schema, so 101 neighbours guarantee overflow of a 100-ID scope. + // Bound that adjacency before materialization and stream at most 101 unique + // IDs to the caller; an inner UNION DISTINCT would consume the full fan-in. + const affected = await database.execute(sql`with recursive affected(id) as ( + select unnest(array[${sql.join([...new Set(seeds)].map(id => sql`${id}::uuid`), sql`, `)}]) + union + select parent.id from affected current cross join lateral ( + (select related_issue_id as id from issue_relations where company_id = ${scope.companyId}::uuid + and issue_id = current.id and type = 'blocks' limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}) + union all select parent_id from issues where company_id = ${scope.companyId}::uuid + and id = current.id and parent_id is not null + ) parent + ) select id::text as "issueId" from affected limit ${ATTENTION_RETENTION_MAX_ISSUE_IDS + 1}`); + // The sentinel means the affected set is incomplete. Reconstruct the whole + // company through the existing paged authority instead of dropping ancestors. + await notifyAttentionRetentionWork(transaction, { companyId: scope.companyId, + issueIds: affected.length <= ATTENTION_RETENTION_MAX_ISSUE_IDS ? affected.map(row => row.issueId as string) : undefined, + }); +} + +export async function notifyRetentionWork(transaction: object, kind: AttentionSourceKind, scope: { companyId?: string; sourceId?: string } = {}) { if (kind === "approval") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.approvalRetention); if (kind === "join_request") await notifyDeliveryWork(transaction, DELIVERY_QUEUES.joinRequestRetention); + if (kind !== "approval" && kind !== "join_request") await notifyAttentionRetentionWork(transaction, { ...scope, sourceKind: kind, + sourceId: kind === "blocker_attention" ? undefined : scope.sourceId }); } export async function notifyCompanyRetentionWork(transaction: object, companyId: string) { + await notifyAttentionRetentionWork(transaction, { companyId }); await notifyApprovalWork(transaction, companyId); await notifyRetentionWork(transaction, "join_request"); } diff --git a/server/src/services/routine-execution-status-notifications.ts b/server/src/services/routine-execution-status-notifications.ts new file mode 100644 index 00000000000..fb9df28dda9 --- /dev/null +++ b/server/src/services/routine-execution-status-notifications.ts @@ -0,0 +1,8 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type RoutineExecutionStatusScope = { issueId?: string; companyId?: string }; + +export function notifyRoutineExecutionStatus(tx: object, issueId?: string, companyId?: string) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.routineExecutionStatus, + issueId || companyId ? JSON.stringify({ issueId, companyId } satisfies RoutineExecutionStatusScope) : "*"); +} diff --git a/server/src/services/routine-execution-status-work.ts b/server/src/services/routine-execution-status-work.ts new file mode 100644 index 00000000000..0a4437f89f2 --- /dev/null +++ b/server/src/services/routine-execution-status-work.ts @@ -0,0 +1,27 @@ +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; +import { ROUTINE_EXECUTION_STATUS_PAGE_SIZE, type routineExecutionStatusService } from "./routine-run-status.js"; + +export function registerRoutineExecutionStatusWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + const cursors = new Map(); + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.routineExecutionStatus, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as RoutineExecutionStatusScope, + matches: (entry, scope?: RoutineExecutionStatusScope) => (!scope?.issueId || entry.issueId === scope.issueId) + && (!scope?.companyId || entry.companyId === scope.companyId), + load: async (scope?: RoutineExecutionStatusScope) => { + const key = JSON.stringify(scope) ?? "*"; + const rows = await service.load(scope, cursors.get(key)); + const complete = rows.length < ROUTINE_EXECUTION_STATUS_PAGE_SIZE; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, + check: async entry => { await service.settle(entry); return null; }, + }, canRun); +} diff --git a/server/src/services/routine-run-status.ts b/server/src/services/routine-run-status.ts index 77341f9cba4..46a2bae4cf5 100644 --- a/server/src/services/routine-run-status.ts +++ b/server/src/services/routine-run-status.ts @@ -1,4 +1,5 @@ -import { and, eq } from "drizzle-orm"; +import { notifyRoutineExecutionStatus, type RoutineExecutionStatusScope } from "./routine-execution-status-notifications.js"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { issues, routineRuns, routines } from "@paperclipai/db"; @@ -37,6 +38,7 @@ function legacyExecutionIssueTransientFailureStatus( } export async function finalizeRoutineRun(executor: Db, runId: string, patch: Partial) { + if (patch.linkedIssueId) await notifyRoutineExecutionStatus(executor, patch.linkedIssueId); return executor .update(routineRuns) .set({ @@ -49,6 +51,10 @@ export async function finalizeRoutineRun(executor: Db, runId: string, patch: Par } export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string) { + return executor.transaction(tx => syncLockedRoutineRunStatusForIssue(tx as unknown as Db, issueId)); +} + +async function syncLockedRoutineRunStatusForIssue(executor: Db, issueId: string) { const issue = await executor .select({ id: issues.id, @@ -58,7 +64,8 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string originRunId: issues.originRunId, }) .from(issues) - .where(eq(issues.id, issueId)) + .where(and(eq(issues.id, issueId), eq(issues.originKind, "routine_execution"))) + .for("update") .then((rows) => rows[0] ?? null); if (!issue || issue.originKind !== "routine_execution" || !issue.originRunId) return null; const run = await executor @@ -67,14 +74,17 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string status: routineRuns.status, failureReason: routineRuns.failureReason, triggerPayload: routineRuns.triggerPayload, + completedAt: routineRuns.completedAt, }) .from(routineRuns) .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, issue.companyId))) .where(and(eq(routineRuns.id, issue.originRunId), eq(routineRuns.companyId, issue.companyId), eq(routineRuns.linkedIssueId, issue.id))) + .for("update", { of: routineRuns }) .then((rows) => rows[0] ?? null); - if (!run) return null; + if (!run || ["received", "skipped", "coalesced"].includes(run.status)) return null; if (issue.status === "done") { + if (run.status === "completed" && run.failureReason === null && run.completedAt) return null; const transientFailureStatus = executionIssueTransientFailureStatusFromPayload(run.triggerPayload) ?? legacyExecutionIssueTransientFailureStatus(run.failureReason); const transientFailureClearedAt = executionIssueTransientFailureClearedAtFromPayload(run.triggerPayload); @@ -99,6 +109,7 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string } if (issue.status === "blocked" || issue.status === "cancelled") { const failureReason = executionIssueTransientFailureReason(issue.status); + if (run.status === "failed" && run.failureReason === failureReason && run.completedAt) return null; return finalizeRoutineRun(executor, issue.originRunId, { status: "failed", failureReason, @@ -135,3 +146,57 @@ export async function syncRoutineRunStatusForIssue(executor: Db, issueId: string return null; } + +export const ROUTINE_EXECUTION_STATUS_PAGE_SIZE = 50; + +export function routineExecutionStatusDebtCondition() { + const transientFailure = sql`((${routineRuns.triggerPayload} -> 'transientFailure' ->> 'code' = ${EXECUTION_ISSUE_TRANSIENT_FAILURE_CODE} + and ${routineRuns.triggerPayload} -> 'transientFailure' ->> 'status' in ('blocked', 'cancelled')) + or ${routineRuns.failureReason} in ('Execution issue moved to blocked', 'Execution issue moved to cancelled'))`; + return or( + and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "blocked", "cancelled"])), + and(eq(routineRuns.status, "completed"), or(inArray(issues.status, ["blocked", "cancelled"]), + and(eq(issues.status, "done"), sql`(${routineRuns.completedAt} is null or ${routineRuns.failureReason} is not null)`))), + and(eq(routineRuns.status, "failed"), transientFailure, + sql`(${issues.status} not in ('blocked', 'cancelled') + or ${routineRuns.completedAt} is null + or ${routineRuns.failureReason} is distinct from 'Execution issue moved to ' || ${issues.status})`), + ); +} + +export function routineExecutionStatusService(db: Db) { + async function load(scope?: RoutineExecutionStatusScope, afterId?: string) { + return db.select({ id: routineRuns.id, companyId: routineRuns.companyId, + issueId: issues.id, routineId: routineRuns.routineId }) + .from(routineRuns) + .innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) + .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), eq(issues.companyId, routineRuns.companyId), + eq(issues.originKind, "routine_execution"), sql`${issues.originRunId} = ${routineRuns.id}::text`)) + .where(and(routineExecutionStatusDebtCondition(), + ...(scope?.issueId ? [eq(issues.id, scope.issueId)] : []), + ...(scope?.companyId ? [eq(routineRuns.companyId, scope.companyId)] : []), + ...(afterId ? [sql`${routineRuns.id} > ${afterId}::uuid`] : []))) + .orderBy(routineRuns.id).limit(ROUTINE_EXECUTION_STATUS_PAGE_SIZE); + } + async function settle(candidate: Awaited>[number]) { + return db.transaction(async tx => { + await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( + eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); + if (!routine) return false; + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), + eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), + eq(issues.originRunId, candidate.id))).for("update"); + if (!issue) return false; + const [run] = await tx.select({ id: routineRuns.id }).from(routineRuns) + .innerJoin(issues, eq(issues.id, routineRuns.linkedIssueId)) + .where(and(eq(routineRuns.id, candidate.id), eq(routineRuns.companyId, candidate.companyId), + eq(routineRuns.routineId, candidate.routineId), eq(routineRuns.linkedIssueId, issue.id), + routineExecutionStatusDebtCondition())).for("update", { of: routineRuns }); + if (!run) return false; + return (await syncRoutineRunStatusForIssue(tx as unknown as Db, issue.id)) !== null; + }); + } + return { load, settle }; +} diff --git a/server/src/services/routines.ts b/server/src/services/routines.ts index 23787b45249..30fefbbabbf 100644 --- a/server/src/services/routines.ts +++ b/server/src/services/routines.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -73,7 +74,7 @@ import { secretService } from "./secrets.js"; import { getSecretProvider } from "../secrets/provider-registry.js"; import { parseCron, validateCron } from "./cron.js"; import { heartbeatService } from "./heartbeat.js"; -import { finalizeRoutineRun, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; +import { finalizeRoutineRun, routineExecutionStatusService, syncRoutineRunStatusForIssue } from "./routine-run-status.js"; import { instanceSettingsService, isTruthyRuntimeEnvValue, @@ -1949,6 +1950,7 @@ export function routineService( if (createdIssue) { await notifyApprovalWork(txDb, createdIssue.companyId); await notifyDecisionTargetIssueRemoval(txDb, createdIssue.id); + await notifyStatusCardIssueChange(txDb, createdIssue.companyId); await txDb.delete(issues).where(eq(issues.id, createdIssue.id)); } const failureReason = error instanceof Error ? error.message : String(error); @@ -2001,7 +2003,8 @@ export function routineService( return run; } - let statusReconcileCursor: string | null = null; + + let statusReconcileCursor: string | undefined; async function syncRunStatusForIssue(issueId: string, executor: Db = db) { return syncRoutineRunStatusForIssue(executor, issueId); @@ -3245,41 +3248,14 @@ export function routineService( return { triggered }; }, - // Repair a missed issue-status hook through the same status transition as - // normal issue writes. Never discard queued input or infer run completion - // from age. Lock both owners and re-read the issue before applying repair. reconcileExecutionRunStatuses: async (): Promise<{ scanned: number; settled: number; deferred: number }> => { - const candidates = await db.select({ id: routineRuns.id, companyId: routineRuns.companyId, - issueId: issues.id, routineId: routineRuns.routineId }) - .from(routineRuns).innerJoin(routines, and(eq(routines.id, routineRuns.routineId), eq(routines.companyId, routineRuns.companyId))) - .innerJoin(issues, and(eq(issues.id, routineRuns.linkedIssueId), - eq(issues.companyId, routineRuns.companyId), eq(issues.originKind, "routine_execution"), - sql`${issues.originRunId} = ${routineRuns.id}::text`, sql`${issues.originId} = ${routineRuns.routineId}::text`)) - .where(and(eq(routineRuns.status, "issue_created"), inArray(issues.status, ["done", "cancelled", "blocked"]), - ...(statusReconcileCursor ? [sql`${routineRuns.id}::text > ${statusReconcileCursor}`] : []))) - .orderBy(asc(routineRuns.id)).limit(50); - statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : null; - let settled = 0; - let deferred = 0; + const service = routineExecutionStatusService(db); + const candidates = await service.load(undefined, statusReconcileCursor); + statusReconcileCursor = candidates.length === 50 ? candidates.at(-1)!.id : undefined; + let settled = 0, deferred = 0; for (const candidate of candidates) { try { - const repaired = await db.transaction(async tx => { - await tx.execute(sql`SET LOCAL lock_timeout = '500ms'`); - await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); - const [routine] = await tx.select({ id: routines.id }).from(routines).where(and( - eq(routines.id, candidate.routineId), eq(routines.companyId, candidate.companyId))).for("share"); - if (!routine) return false; - const [issue] = await tx.select().from(issues).where(and(eq(issues.id, candidate.issueId), - eq(issues.companyId, candidate.companyId), eq(issues.originKind, "routine_execution"), - eq(issues.originRunId, candidate.id), eq(issues.originId, candidate.routineId))).for("update"); - if (!issue || !["done", "cancelled", "blocked"].includes(issue.status)) return false; - const [run] = await tx.select().from(routineRuns).where(and(eq(routineRuns.id, candidate.id), - eq(routineRuns.companyId, candidate.companyId), eq(routineRuns.routineId, candidate.routineId), - eq(routineRuns.linkedIssueId, issue.id), eq(routineRuns.status, "issue_created"))).for("update"); - if (!run) return false; - return (await syncRunStatusForIssue(issue.id, tx as unknown as Db)) !== null; - }); - if (repaired) settled += 1; + if (await service.settle(candidate)) settled++; } catch (error) { let current: unknown = error; let retryable = false; @@ -3289,7 +3265,7 @@ export function routineService( current = cause.cause; } if (!retryable) throw error; - deferred += 1; + deferred++; } } return { scanned: candidates.length, settled, deferred }; diff --git a/server/src/services/run-dispatch-notifications.ts b/server/src/services/run-dispatch-notifications.ts new file mode 100644 index 00000000000..cba58357cf0 --- /dev/null +++ b/server/src/services/run-dispatch-notifications.ts @@ -0,0 +1,11 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +export type RunDispatchScope = { companyId: string }; + +export async function notifyRunDispatch(transaction: object, scope?: RunDispatchScope) { + await notifyIssueRecovery(transaction, scope ? { companyId: scope.companyId } : undefined); + const hint = scope ? JSON.stringify({ companyId: scope.companyId }) : "*"; + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryDispatch, hint); + await notifyDeliveryWork(transaction, DELIVERY_QUEUES.queuedRunDispatch, hint); +} diff --git a/server/src/services/sandbox-absence-proof.test.ts b/server/src/services/sandbox-absence-proof.test.ts index d145425137e..82a7a3cfbba 100644 --- a/server/src/services/sandbox-absence-proof.test.ts +++ b/server/src/services/sandbox-absence-proof.test.ts @@ -1,6 +1,7 @@ import { createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"; import { describe, expect, it, vi } from "vitest"; -import { requestSandboxAbsenceProof, verifySandboxAbsenceReceipt } from "./sandbox-absence-proof.js"; +import { inspectCloudSandboxCleanupAbsence, requestSandboxAbsenceProof, verifySandboxAbsenceReceipt, + verifySandboxCleanupAbsenceReceipt } from "./sandbox-absence-proof.js"; const { privateKey, publicKey } = generateKeyPairSync("ed25519"); const binding = { companyId: randomUUID(), issueId: randomUUID(), runId: randomUUID(), leaseId: randomUUID(), @@ -9,10 +10,10 @@ const env = { PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", PAPERCLIP_CLOUD_API_ORI PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-token", PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret", PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) }; const now = new Date("2099-01-01T00:00:00Z"), seconds = now.getTime() / 1000; -function receipt(overrides: Record = {}) { - const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-absence+jwt" })).toString("base64url"); - const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-absence/v1", sub: "fixture-stack", - ...binding, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); +function receipt(overrides: Record = {}, purpose = "sandbox-absence", source: Record = binding) { + const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: `paperclip-${purpose}+jwt` })).toString("base64url"); + const payload = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: `paperclip-${purpose}/v1`, sub: "fixture-stack", + ...source, absent: true, iat: seconds, exp: seconds + 60, ...overrides })).toString("base64url"); return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), privateKey).toString("base64url")}`; } describe("sandbox absence proof", () => { @@ -45,3 +46,44 @@ describe("sandbox absence proof", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); }); + +describe("cleanup-only sandbox absence", () => { + const cleanup = { purpose: "cleanup" as const, companyId: binding.companyId, environmentId: randomUUID(), + leaseId: binding.leaseId, heartbeatRunId: null as string | null, provider: binding.provider, providerLeaseId: binding.providerLeaseId }; + const lease = { id: cleanup.leaseId, ...cleanup }; + const signed = (overrides: Record = {}) => receipt(overrides, "sandbox-cleanup-absence", cleanup); + it.each([null, binding.runId])("accepts exact absence with nullable run %s without issuing a run receipt", async heartbeatRunId => { + const source = { ...cleanup, heartbeatRunId }; + const fetchImpl = vi.fn().mockResolvedValue(Response.json({ receipt: receipt({}, "sandbox-cleanup-absence", source) })); + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, heartbeatRunId }, { env, now, fetchImpl })) + .toEqual({ providerLeaseId: cleanup.providerLeaseId, state: "destroyed" }); + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://cloud.example.test/v1/tenant/sandbox-absence"); + expect(JSON.parse(init.body)).toEqual(source); + expect(init.headers.authorization).not.toContain(env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN); + expect(init.redirect).toBe("error"); + }); + it("requires every binding, current stack, signature and freshness, and rejects recovery receipts", () => { + expect(verifySandboxCleanupAbsenceReceipt(signed(), cleanup, env, now)).toMatchObject(cleanup); + for (const overrides of [ + ...Object.keys(cleanup).map(key => ({ [key]: randomUUID() })), + { sub: "foreign-stack" }, { absent: false }, { aud: "paperclip-sandbox-absence/v1" }, { iss: "other" }, + { iat: seconds + 31 }, { exp: seconds }, { exp: seconds + 61 }, + ]) expect(verifySandboxCleanupAbsenceReceipt(signed(overrides), cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(signed().slice(0, -8) + "tampered", cleanup, env, now)).toBeNull(); + expect(verifySandboxCleanupAbsenceReceipt(receipt(), cleanup, env, now)).toBeNull(); + expect(verifySandboxAbsenceReceipt(signed(), binding, env, now)).toBeNull(); + }); + it("keeps old Clouds, outages, invalid replies and unconfigured instances inconclusive", async () => { + for (const answer of [{ absent: false }, { absent: true }, { receipt: "forged" }, { receipt: signed({ heartbeatRunId: binding.runId }) }]) + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockResolvedValue(Response.json(answer)) })).toBeUndefined(); + for (const status of [400, 401, 403, 503]) expect(await inspectCloudSandboxCleanupAbsence(lease, + { env, now, fetchImpl: vi.fn().mockResolvedValue(new Response(null, { status })) })).toBeUndefined(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env, now, fetchImpl: vi.fn().mockRejectedValue(new Error("offline")) })).toBeUndefined(); + const fetchImpl = vi.fn(); + expect(await inspectCloudSandboxCleanupAbsence(lease, { env: {}, now, fetchImpl })).toBeUndefined(); + for (const change of [{ provider: "other" }, { environmentId: null }, { providerLeaseId: "invalid" }]) + expect(await inspectCloudSandboxCleanupAbsence({ ...lease, ...change }, { env, now, fetchImpl })).toBeUndefined(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/sandbox-absence-proof.ts b/server/src/services/sandbox-absence-proof.ts index 8ae852688ed..ae6044dd0af 100644 --- a/server/src/services/sandbox-absence-proof.ts +++ b/server/src/services/sandbox-absence-proof.ts @@ -7,8 +7,18 @@ const bindingSchema = z.object({ companyId: z.string().uuid(), issueId: z.string().uuid(), runId: z.string().uuid(), leaseId: z.string().uuid(), ownerToken: z.string().uuid(), provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), -}); +}).strict(); +const cleanupBindingSchema = z.object({ + purpose: z.literal("cleanup"), companyId: z.string().uuid(), environmentId: z.string().uuid(), + leaseId: z.string().uuid(), heartbeatRunId: z.string().uuid().nullable(), + provider: z.literal("daytona"), providerLeaseId: z.string().uuid(), +}).strict(); export type SandboxAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceBinding = z.infer; +export type SandboxCleanupAbsenceProof = SandboxCleanupAbsenceBinding & { + v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-cleanup-absence/v1"; sub: string; + absent: true; iat: number; exp: number; +}; export type SandboxAbsenceProof = SandboxAbsenceBinding & { v: 1; iss: "paperclip-cloud"; aud: "paperclip-sandbox-absence/v1"; sub: string; absent: true; iat: number; exp: number; @@ -16,19 +26,30 @@ export type SandboxAbsenceProof = SandboxAbsenceBinding & { export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbsenceBinding, env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxAbsenceProof | null { + const parsed = bindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-absence", env, now) as SandboxAbsenceProof | null : null; +} + +export function verifySandboxCleanupAbsenceReceipt(receipt: string, binding: SandboxCleanupAbsenceBinding, + env: NodeJS.ProcessEnv = process.env, now = new Date()): SandboxCleanupAbsenceProof | null { + const parsed = cleanupBindingSchema.safeParse(binding); + return parsed.success ? verifyReceipt(receipt, parsed.data, "sandbox-cleanup-absence", env, now) as SandboxCleanupAbsenceProof | null : null; +} + +function verifyReceipt(receipt: string, binding: Record, purpose: string, + env: NodeJS.ProcessEnv, now: Date): Record | null { try { - const parsed = bindingSchema.safeParse(binding); - if (receipt.length > 16_000 || !parsed.success) return null; + if (receipt.length > 16_000) return null; const parts = receipt.split("."); if (parts.length !== 3 || parts.some(part => !/^[A-Za-z0-9_-]+$/.test(part))) return null; const header = JSON.parse(Buffer.from(parts[0], "base64url").toString("utf8")); - if (header.alg !== "EdDSA" || header.typ !== "paperclip-sandbox-absence+jwt" || typeof header.kid !== "string") return null; + if (header.alg !== "EdDSA" || header.typ !== `paperclip-${purpose}+jwt` || typeof header.kid !== "string") return null; if (!verify(null, Buffer.from(`${parts[0]}.${parts[1]}`, "ascii"), cloudRuntimePublicKeyForKid(env, header.kid), Buffer.from(parts[2], "base64url"))) return null; - const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as SandboxAbsenceProof; + const proof = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")); const seconds = Math.floor(now.getTime() / 1000); - if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== "paperclip-sandbox-absence/v1" + if (proof.v !== 1 || proof.iss !== "paperclip-cloud" || proof.aud !== `paperclip-${purpose}/v1` || !env.PAPERCLIP_CLOUD_STACK_ID || proof.sub !== env.PAPERCLIP_CLOUD_STACK_ID || proof.absent !== true - || Object.keys(parsed.data).some(key => proof[key as keyof SandboxAbsenceBinding] !== parsed.data[key as keyof SandboxAbsenceBinding]) + || Object.keys(binding).some(key => proof[key] !== binding[key]) || !Number.isInteger(proof.iat) || !Number.isInteger(proof.exp) || proof.iat > seconds + 30 || proof.exp <= seconds || proof.exp <= proof.iat || proof.exp - proof.iat > 60) return null; return proof; @@ -38,11 +59,35 @@ export function verifySandboxAbsenceReceipt(receipt: string, binding: SandboxAbs export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date; } = {}): Promise<{ proof: SandboxAbsenceProof; receipt: string } | null> { + const parsed = bindingSchema.safeParse(binding); + if (!parsed.success) return null; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt) return null; + const proof = verifySandboxAbsenceReceipt(receipt, parsed.data, options.env, options.now); + return proof ? { proof, receipt } : null; +} + +/** Cleanup-only observation: never grants continuation authority to a run. */ +export async function inspectCloudSandboxCleanupAbsence(lease: { + id: string; companyId: string; environmentId: string | null; heartbeatRunId: string | null; + provider: string | null; providerLeaseId: string | null; +}, options: { env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; now?: Date } = {}) { + const parsed = cleanupBindingSchema.safeParse({ purpose: "cleanup", companyId: lease.companyId, + environmentId: lease.environmentId, leaseId: lease.id, heartbeatRunId: lease.heartbeatRunId, + provider: lease.provider, providerLeaseId: lease.providerLeaseId }); + if (!parsed.success) return undefined; + const receipt = await requestReceipt(parsed.data, options); + if (!receipt || !verifySandboxCleanupAbsenceReceipt(receipt, parsed.data, options.env, options.now)) return undefined; + return { providerLeaseId: parsed.data.providerLeaseId, state: "destroyed" as const }; +} + +async function requestReceipt(binding: Record, options: { + env?: NodeJS.ProcessEnv; fetchImpl?: typeof fetch; +}): Promise { const env = options.env ?? process.env; const context = getCloudStackContext(env); const secret = env.PAPERCLIP_AUTH_SECRET ?? env.BETTER_AUTH_SECRET; - const parsed = bindingSchema.safeParse(binding); - if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim() || !parsed.success) return null; + if (!context?.stackId || !context.primaryHost || !context.cloudOrigin || !secret?.trim()) return null; try { const origin = new URL(context.cloudOrigin); const host = new URL(`https://${context.primaryHost}`); @@ -51,12 +96,11 @@ export async function requestSandboxAbsenceProof(binding: SandboxAbsenceBinding, const response = await (options.fetchImpl ?? fetch)(`${origin.origin}/v1/tenant/sandbox-absence`, { method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), headers: { authorization: `Bearer ${capability}`, "x-paperclip-cloud-tenant-host": host.hostname, "content-type": "application/json" }, - body: JSON.stringify(parsed.data), + body: JSON.stringify(binding), }); if (!response.ok) return null; const payload = await response.json() as { receipt?: unknown }; if (typeof payload.receipt !== "string") return null; - const proof = verifySandboxAbsenceReceipt(payload.receipt, binding, env, options.now); - return proof ? { proof, receipt: payload.receipt } : null; + return payload.receipt; } catch { return null; } } diff --git a/server/src/services/scheduled-retry-cancellation-notifications.ts b/server/src/services/scheduled-retry-cancellation-notifications.ts index 8a8b4fe1a2f..24e57de66cb 100644 --- a/server/src/services/scheduled-retry-cancellation-notifications.ts +++ b/server/src/services/scheduled-retry-cancellation-notifications.ts @@ -1,7 +1,9 @@ +import { notifyRunDispatch } from "./run-dispatch-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; export type ScheduledRetryCancellationScope = { companyId: string; runId?: string }; export async function notifyScheduledRetryCancellation(transaction: object, scope?: ScheduledRetryCancellationScope) { + await notifyRunDispatch(transaction, scope); await notifyDeliveryWork(transaction, DELIVERY_QUEUES.scheduledRetryCancellation, scope ? JSON.stringify(scope) : "*"); } diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index b46841e2a91..dbb9fe3df95 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -12,6 +12,8 @@ export function registerScopedDeadlineWork( queue: DeliveryQueue; batchSize?: number; minimumPassIntervalMs?: number; + reconstructionIntervalMs?: number; + beforePass?(): Promise; decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; load(scope?: Scope, signal?: AbortSignal): Promise; @@ -25,6 +27,7 @@ export function registerScopedDeadlineWork( const entries = new Map(); const failures = new Map(); const scopeFailures = new Map(); + const reconstructAfter = new Map(); let earliest = 0; let lastCompany: string | null = null; function failed(map: typeof failures, id: string) { @@ -34,13 +37,14 @@ export function registerScopedDeadlineWork( function dirtyScopes() { return [...dirty].filter(hint => hint !== "*" || !scopeFailures.has(hint)) .filter(hint => !scans.has(hint) || scopeFailures.has(hint)) + .filter(hint => (reconstructAfter.get(hint) ?? 0) <= Date.now() || scopeFailures.has(hint)) .map(hint => { try { return options.decodeScope(hint); } catch { return undefined; } }); } function next() { if (!canRun()) return dirty.size > 0 || entries.size > 0 ? Date.now() + 30_000 : null; let nextAt: number | null = null; const take = (at: number) => { nextAt = nextAt === null ? at : Math.min(nextAt, at); }; - for (const hint of dirty) take(Math.max(earliest, scopeFailures.get(hint)?.retryAt ?? 0)); + for (const hint of dirty) take(Math.max(earliest, reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0)); const unavailable = dirtyScopes(); for (const entry of entries.values()) if (!unavailable.some(scope => options.matches(entry, scope))) take(Math.max(earliest, entry.dueAt, failures.get(entry.id)?.retryAt ?? 0)); @@ -54,9 +58,18 @@ export function registerScopedDeadlineWork( } }, run: async signal => { if (!canRun() || Date.now() < earliest) return; + if ((options.reconstructionIntervalMs ?? 0) > 0) for (const [hint, until] of reconstructAfter) { + if (until <= Date.now()) reconstructAfter.delete(hint); + } + const ready = [...dirty].filter(hint => Math.max(reconstructAfter.get(hint) ?? 0, scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); + if (ready.length === 0 && ![...entries.values()].some(entry => Math.max(entry.dueAt, failures.get(entry.id)?.retryAt ?? 0) <= Date.now())) return; earliest = Date.now() + (options.minimumPassIntervalMs ?? 1000); + try { await options.beforePass?.(); } catch (error) { + for (const hint of ready) failed(scopeFailures, hint); + for (const entry of entries.values()) if (entry.dueAt <= Date.now()) failed(failures, entry.id); + throw error; + } const errors: unknown[] = []; - const ready = [...dirty].filter(hint => (scopeFailures.get(hint)?.retryAt ?? 0) <= Date.now()); const requested = ready.includes("*") ? ["*", ...ready.filter(hint => hint !== "*")].slice(0, 25) : ready.slice(0, 25); @@ -81,6 +94,9 @@ export function registerScopedDeadlineWork( entries.delete(entry.id); failures.delete(entry.id); } scans.delete(hint); + if ((options.reconstructionIntervalMs ?? 0) > 0) { + reconstructAfter.set(hint, Date.now() + options.reconstructionIntervalMs!); + } if ((versions.get(hint) ?? 0) !== scan.version) dirty.add(hint); else versions.delete(hint); if (!scope) scopeFailures.clear(); else scopeFailures.delete(hint); } catch (error) { diff --git a/server/src/services/slack-conversation-lifecycle.ts b/server/src/services/slack-conversation-lifecycle.ts index 0fd213dc084..b7754387c00 100644 --- a/server/src/services/slack-conversation-lifecycle.ts +++ b/server/src/services/slack-conversation-lifecycle.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js"; import { and, asc, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm"; @@ -126,6 +128,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu const ready = await readSettlement(tx, companyId, issueId, true); if (!ready) return false; const { binding, issue, run } = ready; + await notifyAttentionRetentionWork(tx, { companyId }); const changed = await tx.update(chatConversations).set({ state: "waiting", updatedAt: new Date() }) .where(and(eq(chatConversations.id, binding.conversation.id), eq(chatConversations.companyId, companyId), eq(chatConversations.state, "active"))) .returning({ id: chatConversations.id }); @@ -133,6 +136,7 @@ export async function settleExternalConversation(db: Db, companyId: string, issu await notifyUnsafeWorkspaceExportRecovery(tx, { companyId, issueId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "in_review", updatedAt: new Date() }) .where(and(eq(issues.id, issueId), eq(issues.companyId, companyId))); activity = (await persistActivity(tx as unknown as Db, { diff --git a/server/src/services/slack-conversation-state.ts b/server/src/services/slack-conversation-state.ts index dedaa8d6676..622149badeb 100644 --- a/server/src/services/slack-conversation-state.ts +++ b/server/src/services/slack-conversation-state.ts @@ -1,3 +1,5 @@ +import { notifyAttentionRetentionWork } from "./retention-work-notifications.js"; +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js"; import { chatConversations, issues, type Db } from "@paperclipai/db"; @@ -45,10 +47,12 @@ export function nonIdleSlackIssueCondition() { /** Called inside message admission while its transaction owns the issue lock. */ export async function resumeExternalConversation(tx: Db, companyId: string, issueId: string) { + await notifyAttentionRetentionWork(tx, { companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers); await notifyExternalConversationSettlement(tx, { companyId, issueId }); // Only the verified idle convention may leave In Review automatically. // A real decision or delivery problem keeps the existing review semantics. + await notifyStatusCardIssueChange(tx, companyId); await tx.update(issues).set({ status: "todo", updatedAt: new Date() }) .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId), sql`${externalConversationStateSql()} = 'waiting'`)); diff --git a/server/src/services/status-card-evaluation.ts b/server/src/services/status-card-evaluation.ts new file mode 100644 index 00000000000..e6086535fc9 --- /dev/null +++ b/server/src/services/status-card-evaluation.ts @@ -0,0 +1,98 @@ +import { and, asc, eq, isNotNull, isNull } from "drizzle-orm"; +import { companies, statusCards, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { issueService } from "./issues.js"; +import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "./issue-assignment-wakeup.js"; +import { statusCardService } from "./status-cards.js"; + +/** Feature-owned generation and assignment. Existing issue services retain + * execution authority, durable wake dedupe and stalled-generation cleanup. */ +export function statusCardEvaluationService(db: Db, heartbeat: IssueAssignmentWakeupDeps) { + const cards = statusCardService(db); + const issues = issueService(db); + const settings = instanceSettingsService(db); + let reconstructReactive = true; + let allChanges = false; + const changes = new Set(); + let enabled: boolean | null = null; + let retryAt: number | null = null; + + return { + changed(hints?: ReadonlySet) { + for (const hint of hints ?? ["*"]) { + if (hint === "*" || hint === "changes:*") { allChanges = true; if (hint === "*") enabled = null; } + else if (hint.startsWith("changes:")) changes.add(hint.slice("changes:".length)); + } + }, + async run(signal: AbortSignal) { + if (enabled === null) enabled = (await settings.getExperimental()).enableStatusCards === true; + if (!enabled || signal.aborted) return; + const reconstruct = reconstructReactive; + const evaluateAllChanges = allChanges; + const companiesChanged = new Set(changes); + changes.clear(); + reconstructReactive = false; + allChanges = false; + try { + const result = await cards.tickDueStatusCards(new Date(), evaluateAllChanges ? "changed" : companiesChanged, reconstruct); + retryAt = result.deferredCompanyIds.size ? Date.now() + 30_000 : null; + for (const companyId of result.deferredCompanyIds) changes.add(companyId); + const assignments = await Promise.allSettled(result.enqueued.map(async ({ cardId, generatingIssue }) => { + // Once generated, dispatch or cancel even if shutdown begins: leaving + // an unassigned claimed card is not a completed sweep. + try { + await queueIssueAssignmentWakeup({ + heartbeat, issue: generatingIssue, + reason: "status_card_update_assigned", + mutation: "status_card.scheduler_update_requested", + contextSource: "status_card_scheduler", + requestedByActorType: "system", taskKey: `status-card:${cardId}`, + rethrowOnError: true, + }); + } catch (error) { + await issues.update(generatingIssue.id, { status: "cancelled" }); + throw error; + } + })); + const failures = assignments.filter(result => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Status-card assignment failed"); + if (result.evaluated || result.enqueued.length) { + logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card evaluation complete"); + } + } catch (error) { + reconstructReactive ||= reconstruct; + allChanges ||= evaluateAllChanges; + for (const companyId of companiesChanged) changes.add(companyId); + throw error; + } + }, + async next() { + if (!enabled) return null; + const [next] = await db.select({ at: statusCards.nextEvalAt }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), + isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt))) + .orderBy(asc(statusCards.nextEvalAt)).limit(1); + const deadline = next?.at?.getTime() ?? null; + return retryAt === null ? deadline : Math.min(deadline ?? Infinity, retryAt); + }, + }; +} + +export function registerStatusCardEvaluationWork( + coordinator: ReturnType, + service: ReturnType, + canRun: () => boolean, +) { + return registerDeadlineWork(coordinator, DELIVERY_QUEUES.statusCardEvaluation, { + run: service.run, next: service.next, onSettled: service.changed, canRun, retryMs: 30_000, + // Batch source hints for at most one second after reconciliation. Do not + // use minimumSpacingMs: known policy/retry deadlines must preempt batching. + // This window alone arms no timer when the feature has no pending work. + notificationCoalesceMs: 1_000, + }); +} diff --git a/server/src/services/status-card-finalization.ts b/server/src/services/status-card-finalization.ts index 7c7d260992e..702509be8b4 100644 --- a/server/src/services/status-card-finalization.ts +++ b/server/src/services/status-card-finalization.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { and, eq, isNull } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { statusCards, statusCardUpdates } from "@paperclipai/db"; @@ -36,6 +37,7 @@ export async function finalizeStatusCardsForStalledGeneration( ) { if (!STALLED_GENERATION_STATUSES.has(issue.status)) return []; + await notifyStatusCardEvaluation(dbOrTx, issue.companyId); const now = new Date(); const failureReason = failureReasonForIssue(issue); const cards = await dbOrTx @@ -45,6 +47,7 @@ export async function finalizeStatusCardsForStalledGeneration( failureReason, generatingIssueId: null, nextEvalAt: null, + nextEvalReason: null, updatedAt: now, }) .where( diff --git a/server/src/services/status-card-notifications.ts b/server/src/services/status-card-notifications.ts new file mode 100644 index 00000000000..6fc12985847 --- /dev/null +++ b/server/src/services/status-card-notifications.ts @@ -0,0 +1,10 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** Card/settings changes reconstruct deadlines; issue changes also evaluate reactive scope. */ +export function notifyStatusCardEvaluation(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `cards:${companyId}` : "*"); +} + +export function notifyStatusCardIssueChange(transaction: object, companyId?: string) { + return notifyDeliveryWork(transaction, DELIVERY_QUEUES.statusCardEvaluation, companyId ? `changes:${companyId}` : "changes:*"); +} diff --git a/server/src/services/status-card-update-engine.ts b/server/src/services/status-card-update-engine.ts index ff14d35c640..0b1e5325446 100644 --- a/server/src/services/status-card-update-engine.ts +++ b/server/src/services/status-card-update-engine.ts @@ -109,29 +109,42 @@ export function statusCardFingerprintHash(fingerprint: StatusCardFingerprint) { return createHash("sha256").update(JSON.stringify(stable)).digest("hex"); } +// These caches contain only timezone/policy calculations, never card or company +// data. Bound both maps because timezone aliases and hour pairs are user input. +const activeHoursFormatters = new Map(); +const upcomingActiveHours = new Map(); + +function remember(cache: Map, key: K, value: V, limit: number): V { + cache.delete(key); + cache.set(key, value); + if (cache.size > limit) cache.delete(cache.keys().next().value!); + return value; +} + +function activeHoursFormatter(timezone: string) { + const cached = activeHoursFormatters.get(timezone); + return remember(activeHoursFormatters, timezone, cached ?? new Intl.DateTimeFormat("en-GB", { + timeZone: timezone, hour: "2-digit", minute: "2-digit", hourCycle: "h23", + }), 32); +} + export function isWithinStatusCardActiveHours(policy: StatusCardRefreshPolicy, now: Date) { if (!policy.activeHours) return true; - const parts = new Intl.DateTimeFormat("en-GB", { - timeZone: policy.activeHours.timezone, - hour: "2-digit", - minute: "2-digit", - hourCycle: "h23", - }).formatToParts(now); - const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); - const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); - const current = hour * 60 + minute; const [startHour, startMinute] = policy.activeHours.start.split(":").map(Number); const [endHour, endMinute] = policy.activeHours.end.split(":").map(Number); const start = startHour! * 60 + startMinute!; const end = endHour! * 60 + endMinute!; + if (start === end) return false; + const parts = activeHoursFormatter(policy.activeHours.timezone).formatToParts(now); + const hour = Number(parts.find((part) => part.type === "hour")?.value ?? 0); + const minute = Number(parts.find((part) => part.type === "minute")?.value ?? 0); + const current = hour * 60 + minute; return start <= end ? current >= start && current < end : current >= start || current < end; } export function nextStatusCardEvaluationAt(policy: StatusCardRefreshPolicy, now: Date) { - if (policy.mode === "manual") return null; - const seconds = policy.mode === "interval" - ? (policy.intervalMinutes ?? 15) * 60 - : Math.min(policy.debounceSeconds ?? 60, 60); + if (policy.mode !== "interval") return null; + const seconds = (policy.intervalMinutes ?? 15) * 60; return new Date(now.getTime() + seconds * 1000); } @@ -172,3 +185,51 @@ export function evaluateStatusCardPolicy(input: { } return { action: "run" as const }; } + +/** A retry exists only while filtered changes remain. Use the owning policy's + * actual eligibility boundary instead of repeatedly rediscovering a closed gate. */ +export function nextStatusCardPolicyEligibility(input: { + policy: StatusCardRefreshPolicy; + now: Date; + lastChangeAt: Date; + completedUpdateTimes: Date[]; + tokensToday: number; +}): Date | null { + if (input.policy.mode === "manual") return null; + const hours = input.policy.activeHours; + if (hours && hours.start === hours.end) return null; + let at = input.now.getTime(); + if (input.tokensToday >= (input.policy.dailyTokenCap ?? 100_000)) { + const day = new Date(input.now); + day.setUTCHours(24, 0, 0, 0); + at = Math.max(at, day.getTime()); + } + if (input.policy.mode === "reactive") { + at = Math.max(at, input.lastChangeAt.getTime() + (input.policy.debounceSeconds ?? 60) * 1000); + const cap = input.policy.maxUpdatesPerHour ?? 6; + const recent = input.completedUpdateTimes.map(time => time.getTime()) + .filter(time => time > at - 3_600_000).sort((a, b) => a - b); + if (recent.length >= cap) at = Math.max(at, recent[recent.length - cap]! + 3_600_000 + 1); + } + // Check the precise instant before reusing a minute-aligned opening. Some + // historical timezone offsets include seconds, so a window may be open now. + if (!hours || isWithinStatusCardActiveHours(input.policy, new Date(at))) return new Date(at); + const key = JSON.stringify([hours.timezone, hours.start, hours.end]); + const cached = upcomingActiveHours.get(key); + if (cached && at >= cached.closedAt && at < cached.opensAt) { + remember(upcomingActiveHours, key, cached, 256); + return new Date(cached.opensAt); + } + const closedAt = at; + // Active-hour precision is one local minute. Walking UTC minutes also honors + // skipped/repeated local times at daylight-saving transitions. + const end = at + 48 * 3_600_000; + for (at = Math.floor(at / 60_000) * 60_000 + 60_000; at <= end; at += 60_000) { + if (isWithinStatusCardActiveHours(input.policy, new Date(at))) { + remember(upcomingActiveHours, key, { closedAt, opensAt: at }, 256); + return new Date(at); + } + } + // No opening inside the existing bounded search horizon. Retain changes. + return null; +} diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index 4a7439b6aa5..d4660227c28 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,7 +1,9 @@ +import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; import { createHash } from "node:crypto"; -import { and, desc, eq, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; +import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { agents, + companies, costEvents, documentRevisions, documents, @@ -9,6 +11,7 @@ import { issueComments, statusCards, statusCardUpdates, + withDedicatedDbConnection, type Db, } from "@paperclipai/db"; import type { @@ -34,6 +37,7 @@ import { extractIssueMentions, filterStatusCardChanges, nextStatusCardEvaluationAt, + nextStatusCardPolicyEligibility, STATUS_CARD_MAX_MENTIONED_ISSUES, statusCardChangesHash, statusCardFingerprintHash, @@ -151,9 +155,15 @@ export function statusCardService( db: Db, deps: { issuesSvc?: ReturnType } = {}, ) { - const builtIns = builtInAgentService(db); - const issuesSvc = deps.issuesSvc ?? issueService(db); + const issuesSvc = () => deps.issuesSvc ?? issueService(db); const searchSvc = companySearchService(db); + type Transaction = Parameters[0]>[0]; + function mutate(companyId: string, write: (tx: Transaction) => Promise) { + return db.transaction(async tx => { + await notifyStatusCardEvaluation(tx, companyId); + return write(tx); + }); + } async function readWatchedIssueVisibility(card: StatusCardRow, options: StatusCardReadOptions = {}) { if (card.queries.length === 0 && (card.mentionedIssueIds?.length ?? 0) === 0) { @@ -240,10 +250,10 @@ export function statusCardService( }; const agentId = actor.agentId; if (!agentId) { - return db.insert(statusCards).values(values).returning().then((rows) => rows[0]!); + return mutate(companyId, tx => tx.insert(statusCards).values(values).returning().then((rows) => rows[0]!)); } - return db.transaction(async (tx) => { + return mutate(companyId, async (tx) => { const author = await tx .select({ id: agents.id }) .from(agents) @@ -292,11 +302,12 @@ export function statusCardService( ...(input.refreshPolicy !== undefined ? { refreshPolicy: input.refreshPolicy, - nextEvalAt: card.archivedAt ? null : nextStatusCardEvaluationAt(input.refreshPolicy, now), + nextEvalReason: "refresh", + nextEvalAt: card.archivedAt ? null : input.refreshPolicy.mode === "reactive" && card.queries.length > 0 ? now : nextStatusCardEvaluationAt(input.refreshPolicy, now), } : {}), ...(archiveChanged && input.archived - ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null } + ? { archivedAt: now, archivedByAgentId: actor.agentId, archivedByUserId: actor.userId, nextEvalAt: null, nextEvalReason: null } : {}), ...(archiveChanged && !input.archived ? { @@ -306,24 +317,27 @@ export function statusCardService( lastChangeAt: now, lastUpdateRunKind: null, nextEvalAt: card.queries.length > 0 ? now : null, + nextEvalReason: "refresh", } : {}), }; - const next = await db.update(statusCards).set({ + const next = await mutate(card.companyId, tx => tx.update(statusCards).set({ ...values, ...(archiveChanged && input.archived ? { generatingIssueId: null, pendingChangeHash: null } : {}), - }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!); + }).where(eq(statusCards.id, card.id)).returning().then((rows) => rows[0]!)); if (archiveChanged && input.archived && card.generatingIssueId) { const generationIssue = await db.select().from(issues).where(eq(issues.id, card.generatingIssueId)).then((rows) => rows[0] ?? null); if (generationIssue && !TERMINAL_ISSUE_STATUSES.has(generationIssue.status)) { - await issuesSvc.update(generationIssue.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue.id, { status: "cancelled" }); } } return next; } async function remove(id: string) { - return db.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null); + const card = await getById(id); + if (!card) return null; + return mutate(card.companyId, tx => tx.delete(statusCards).where(eq(statusCards.id, id)).returning().then((rows) => rows[0] ?? null)); } async function listUpdates(cardId: string) { @@ -360,7 +374,7 @@ export function statusCardService( .then((rows) => rows[0] ?? null); if (override) return override.id; } - const builtIn = await builtIns.get(card.companyId, SUMMARIZER_BUILT_IN_KEY); + const builtIn = await builtInAgentService(db).get(card.companyId, SUMMARIZER_BUILT_IN_KEY); if (builtIn.status !== "ready" || !builtIn.agentId) { throw unprocessable("Summarizer built-in agent is not configured", { code: "summarizer_not_configured", @@ -391,7 +405,7 @@ export function statusCardService( let deduplicated = false; const createdAt = new Date(); - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `Compile status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: compileDescription(card, null, hash), status: "todo", @@ -409,16 +423,16 @@ export function statusCardService( // covers idempotency-key hits that resolve to a terminal task (done/cancelled) // as well as a `blocked` one that a manual re-kick is reviving. const reopened = deduplicated && (TERMINAL_ISSUE_STATUSES.has(created.status) || created.status === "blocked") - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: compileDescription(card, reopened!.id, hash), }); - const [nextCard] = await db + const [nextCard] = await mutate(card.companyId, tx => tx .update(statusCards) .set({ generatingIssueId: generationIssue!.id, state: "compiling", failureReason: null, updatedAt: createdAt }) .where(eq(statusCards.id, card.id)) - .returning(); + .returning()); return { card: nextCard!, generatingIssue: generationIssue!, @@ -464,7 +478,7 @@ export function statusCardService( if (card.archivedAt) throw unprocessable("Archived status cards cannot accept generation writes"); await assertSummarizerWriter(card, input.generationIssueId, actor); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card compilation was superseded by a newer task"); @@ -528,9 +542,10 @@ export function statusCardService( companyId: string, issueIds: string[], options: StatusCardReadOptions = {}, + queryDb: Db = db, ): Promise { if (issueIds.length === 0) return []; - const rows = await db + const rows = await queryDb .select({ id: issues.id, identifier: issues.identifier, @@ -580,12 +595,12 @@ export function statusCardService( return loadIssueSummaries(card.companyId, card.mentionedIssueIds ?? [], options); } - async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}) { + async function executeQueries(card: StatusCardRow, options: StatusCardReadOptions = {}, onNextMembershipChange?: (at: Date | null) => void, queryDb: Db = db) { const issueMap = new Map(); for (const storedQuery of card.queries) { const query = companySearchQuerySchema.parse(storedQuery); - const response = await searchSvc.search(card.companyId, query, { - issueReadCondition: options.issueReadCondition, + const response = await companySearchService(queryDb).search(card.companyId, query, { + issueReadCondition: options.issueReadCondition, onNextMembershipChange, }); for (const result of response.results) { if (result.type === "issue" && result.issue) issueMap.set(result.issue.id, result.issue); @@ -596,12 +611,12 @@ export function statusCardService( const mentioned = await loadIssueSummaries( card.companyId, (card.mentionedIssueIds ?? []).filter((issueId) => !issueMap.has(issueId)), - options, + options, queryDb, ); for (const issue of mentioned) issueMap.set(issue.id, issue); const snapshot = [...issueMap.values()]; if (snapshot.length === 0) return snapshot; - const latestHumanComments = await db + const latestHumanComments = await queryDb .select({ issueId: issueComments.issueId, // The postgres-js driver returns the `max()` aggregate over a timestamp @@ -622,6 +637,12 @@ export function statusCardService( return snapshot.map((issue) => ({ ...issue, latestHumanCommentAt: commentByIssueId.get(issue.id) ?? null })); } + function nextEvaluationAt(card: StatusCardRow, membershipAt: Date | null, now: Date) { + const interval = nextStatusCardEvaluationAt(card.refreshPolicy, now); + return card.refreshPolicy.mode === "reactive" && (card.refreshPolicy.triggers.membershipChanges || card.refreshPolicy.triggers.anyUpdate) + ? membershipAt : interval; + } + async function requestRefresh(cardId: string, input: { full?: boolean; trigger?: "manual" | "interval" | "reactive" | "restore"; @@ -642,21 +663,23 @@ export function statusCardService( } const now = input.now ?? new Date(); - const snapshot = await executeQueries(card); + let membershipAt: Date | null = null; + const snapshot = await executeQueries(card, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }); const fingerprint = buildStatusCardFingerprint(snapshot); const allChanges = diffStatusCardFingerprint(card.fingerprint as StatusCardFingerprint | null, fingerprint); const changes = filterStatusCardChanges(allChanges, card.refreshPolicy); const trigger = input.trigger ?? "manual"; const forceRun = trigger === "manual" || trigger === "restore"; - const nextEvalAt = nextStatusCardEvaluationAt(card.refreshPolicy, now); + const nextEvalAt = nextEvaluationAt(card, membershipAt, now); if (!forceRun && changes.length === 0) { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, state: "active", nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -678,13 +701,18 @@ export function statusCardService( manual: forceRun, }); if (decision.action !== "run") { - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ pendingChangeCount: changes.length, pendingChangeHash, lastChangeAt, state: decision.action === "pause_budget" ? "paused_budget" : decision.action === "pause_hours" ? "paused_hours" : "active", - nextEvalAt: decision.action === "wait" && "dueAt" in decision ? decision.dueAt : nextEvalAt, - }).where(eq(statusCards.id, card.id)).returning(); + nextEvalReason: "policy", + nextEvalAt: nextStatusCardPolicyEligibility({ + policy: card.refreshPolicy, now, lastChangeAt, + completedUpdateTimes: recent.filter(row => row.kind !== "compile" && row.finishedAt).map(row => row.startedAt), + tokensToday: Number(daily[0]?.tokens ?? 0), + }), + }).where(eq(statusCards.id, card.id)).returning()); return { card: next!, generatingIssue: null, alreadyGenerating: false, enqueued: false }; } @@ -707,7 +735,7 @@ export function statusCardService( : null; const fingerprintHash = statusCardFingerprintHash(fingerprint); let deduplicated = false; - const created = await issuesSvc.create(card.companyId, { + const created = await issuesSvc().create(card.companyId, { title: `${kind === "full" ? "Rebuild" : "Update"} status card: ${card.title ?? card.interestPrompt.slice(0, 80)}`, description: updateDescription({ card, generationIssueId: null, fingerprint, changes, kind, trigger, previousSummary, snapshot }), status: "todo", @@ -720,33 +748,34 @@ export function statusCardService( onDeduplicated: (reason) => { deduplicated = reason === "idempotency_key"; }, }); const reopened = deduplicated && TERMINAL_ISSUE_STATUSES.has(created.status) - ? await issuesSvc.update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) + ? await issuesSvc().update(created.id, { status: "todo", assigneeAgentId: summarizerAgentId }) : created; - const generationIssue = await issuesSvc.update(reopened!.id, { + const generationIssue = await issuesSvc().update(reopened!.id, { description: updateDescription({ card, generationIssueId: reopened!.id, fingerprint, changes, kind, trigger, previousSummary, snapshot }), }); const priorGenerationPredicate = card.generatingIssueId ? eq(statusCards.generatingIssueId, card.generatingIssueId) : isNull(statusCards.generatingIssueId); - const [next] = await db.update(statusCards).set({ + const [next] = await mutate(card.companyId, tx => tx.update(statusCards).set({ generatingIssueId: generationIssue!.id, pendingChangeCount: changes.length, pendingChangeHash, lastChangeAt, state: "active", nextEvalAt, + nextEvalReason: nextEvalAt ? card.refreshPolicy.mode === "reactive" ? "membership" : "refresh" : null, failureReason: null, - }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning(); + }).where(and(eq(statusCards.id, card.id), isNull(statusCards.archivedAt), or(isNull(statusCards.generatingIssueId), priorGenerationPredicate))).returning()); if (!next) { const winner = await getById(card.id); if (!winner?.generatingIssueId) { if (!TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } throw conflict("Status-card refresh claim was lost"); } if (generationIssue!.id !== winner.generatingIssueId && !TERMINAL_ISSUE_STATUSES.has(generationIssue!.status)) { - await issuesSvc.update(generationIssue!.id, { status: "cancelled" }); + await issuesSvc().update(generationIssue!.id, { status: "cancelled" }); } const winnerIssue = await db.select().from(issues).where(eq(issues.id, winner.generatingIssueId)).then((rows) => rows[0] ?? null); return { card: winner, generatingIssue: winnerIssue, alreadyGenerating: true, enqueued: false, kind, changes }; @@ -765,28 +794,57 @@ export function statusCardService( return { card: next, generatingIssue: generationIssue!, alreadyGenerating: deduplicated, enqueued: true, kind, changes }; } - async function tickDueStatusCards(now = new Date()) { - const due = await db.select().from(statusCards).where(and(isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now))); + async function tickDueStatusCards(now = new Date(), reactiveCompanies?: ReadonlySet | "changed", reconstruct = false) { + const changedScope = reactiveCompanies === "changed" ? sql`true` : reactiveCompanies?.size + ? inArray(statusCards.companyId, [...reactiveCompanies]) : sql`false`; + // Startup recovers lost source hints for membership deadlines as well as + // disarmed cards. Saved policy gates and retry leases keep their owner. + // Legacy untagged leases are at most five minutes from their claim time: + // preserve that bounded ambiguity, never a days-away membership deadline. + const startupScope = reconstruct ? or( + isNull(statusCards.nextEvalAt), eq(statusCards.nextEvalReason, "membership"), + and(isNull(statusCards.nextEvalReason), isNull(statusCards.pendingChangeHash), + eq(statusCards.state, "active"), gt(statusCards.nextEvalAt, new Date(now.getTime() + 5 * 60_000))), + ) : undefined; + const reactiveScope = or(startupScope, changedScope); + const due = await db.select({ card: statusCards }).from(statusCards) + .innerJoin(companies, eq(companies.id, statusCards.companyId)) + .where(and(eq(companies.status, "active"), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + or(and(isNotNull(statusCards.nextEvalAt), lte(statusCards.nextEvalAt, now)), + and(reactiveScope, ne(statusCards.state, "error"), ne(statusCards.state, "compiling"), sql`${statusCards.refreshPolicy}->>'mode' = 'reactive'`, sql`jsonb_array_length(${statusCards.queries}) > 0`)))) + .then(rows => rows.map(row => row.card)); const enqueued: Array<{ cardId: string; generatingIssue: typeof issues.$inferSelect }> = []; let evaluated = 0; + const deferredCompanyIds = new Set(); for (const candidate of due) { - const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); - const [claimed] = await db.update(statusCards).set({ nextEvalAt: claimUntil }) - .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.generatingIssueId), lte(statusCards.nextEvalAt, now))) - .returning(); - if (!claimed) continue; - evaluated += 1; - try { - const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); - if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); - } catch (err) { - logger.warn( - { err, cardId: claimed.id, companyId: claimed.companyId }, - "status card scheduled refresh failed", + // nextEvalAt is both a retry lease and an evaluation deadline. A source + // event may preempt a genuine future deadline, so serialize scheduler + // evaluation across connections without holding a row lock while the + // existing issue/card services commit through their own transactions. + await withDedicatedDbConnection(db, connection => connection.transaction(async lockTx => { + const [lock] = await lockTx.execute<{ acquired: boolean }>( + sql`select pg_try_advisory_xact_lock(hashtextextended(${candidate.id}, 0)) as acquired`, ); - } + if (!lock?.acquired) { deferredCompanyIds.add(candidate.companyId); return; } + const claimUntil = new Date(now.getTime() + 5 * 60 * 1000); + const [claimed] = await mutate(candidate.companyId, tx => tx.update(statusCards).set({ nextEvalAt: claimUntil, nextEvalReason: "retry" }) + .where(and(eq(statusCards.id, candidate.id), isNull(statusCards.archivedAt), isNull(statusCards.generatingIssueId), + candidate.nextEvalAt ? eq(statusCards.nextEvalAt, candidate.nextEvalAt) : isNull(statusCards.nextEvalAt))) + .returning()); + if (!claimed) return; + evaluated += 1; + try { + const result = await requestRefresh(claimed.id, { trigger: claimed.refreshPolicy.mode === "reactive" ? "reactive" : "interval", now }); + if (result.enqueued && result.generatingIssue) enqueued.push({ cardId: claimed.id, generatingIssue: result.generatingIssue }); + } catch (err) { + logger.warn( + { err, cardId: claimed.id, companyId: claimed.companyId }, + "status card scheduled refresh failed", + ); + } + })); } - return { evaluated, enqueued }; + return { evaluated, enqueued, deferredCompanyIds }; } async function writeSummary(cardId: string, input: WriteStatusCardSummary, actor: StatusCardWriter) { @@ -796,7 +854,7 @@ export function statusCardService( await assertSummarizerWriter(card, input.generationIssueId, actor); if (card.queries.length === 0) throw conflict("Compile the status-card query before writing its summary"); const now = new Date(); - return db.transaction(async (tx) => { + return mutate(card.companyId, async (tx) => { const current = await tx.select().from(statusCards).where(eq(statusCards.id, card.id)).then((rows) => rows[0] ?? null); if (!current || current.archivedAt || current.generatingIssueId !== input.generationIssueId) { throw conflict("Status-card generation was superseded by a newer task"); @@ -816,9 +874,11 @@ export function statusCardService( const mentionedIssueIds = await resolveMentionedIssueIds(current.companyId, input.markdown); // Current watched membership: compiled-query matches plus the issues this // summary mentions. - const watchedNow = buildStatusCardFingerprint( - await executeQueries({ ...current, mentionedIssueIds }), - ); + let membershipAt: Date | null = null; + // All watched-source reads use this transaction. Borrowing the root + // pool here would deadlock a one-connection installation. + const watchedSnapshot = await executeQueries({ ...current, mentionedIssueIds }, {}, at => { if (at && (!membershipAt || at < membershipAt)) membershipAt = at; }, tx as unknown as Db); + const watchedNow = buildStatusCardFingerprint(watchedSnapshot); let snapshot: StatusCardFingerprint; if (payloadFingerprint) { // Keep the generation-time fingerprint as the change baseline: issues @@ -889,7 +949,12 @@ export function statusCardService( pendingChangeCount: 0, pendingChangeHash: null, lastChangeAt: null, - nextEvalAt: nextStatusCardEvaluationAt(current.refreshPolicy, now), + nextEvalReason: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? "refresh" : membershipAt ? "membership" : null, + nextEvalAt: current.refreshPolicy.mode === "reactive" && + filterStatusCardChanges(diffStatusCardFingerprint(snapshot, watchedNow), current.refreshPolicy).length > 0 + ? now : nextEvaluationAt(current, membershipAt, now), updatedAt: now, }).where(and(eq(statusCards.id, current.id), eq(statusCards.generatingIssueId, input.generationIssueId))).returning(); if (!next) throw conflict("Status-card generation was superseded by a newer task"); diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index 8dba366e1af..347593dafed 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -1,3 +1,4 @@ +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { and, eq, sql } from "drizzle-orm"; import { heartbeatRuns, issueComments, issueRecoveryActions, type Db } from "@paperclipai/db"; import { writeExecutionControl } from "./execution-control-notifications.js"; @@ -11,6 +12,7 @@ export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourc .where(and(eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, source.actionId), eq(issueRecoveryActions.sourceIssueId, source.issueId))).for("update"); if (!existing || existing.evidence.systemRecovery) return; + await notifyIssueRecovery(tx, { companyId: source.companyId }); await notifyDeliveryWork(tx, DELIVERY_QUEUES.workspaceRecovery); const state = createSystemRecovery(source.runId); // Atomic, once per source. An exhausted previous execution budget is history; @@ -71,7 +73,7 @@ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySour sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,revision}' = ${String(previous.revision)}`)) .returning({ id: issueRecoveryActions.id }); return Boolean(updated); - }); + }, { companyId: source.companyId }); } export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index e9f0bd4edbf..ba5959b3eb6 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -334,13 +334,15 @@ export async function requestTaskRetry(input: { details: { requestId: active?.id ?? id, sourceRunId: latest.id, attachedToExistingWork: Boolean(active || busy) } }); return { requestId: active?.id ?? id, accepted: true, stale: false, observedRunId: latest.id, chatActionId: intent.chatActionId }; - }); + }, { companyId: input.companyId }); } /** Chat owns admission and authorization. The task receipt follows that * durable outcome; it never launches a parallel generic board execution. */ -export async function reconcileTaskChatRetryReceipts(db: Db) { +export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId?: string; requestId?: string; settleOnly?: boolean } = {}) { const pending = await db.select().from(agentWakeupRequests).where(and( + scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, + scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, )).orderBy(agentWakeupRequests.updatedAt).limit(50); @@ -356,7 +358,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db) { // A deferred chat receipt is admitted input, not a completed dispatch. // Re-stage that exact action through chat's owner; it rechecks authority // and normal queue policy before updating the existing receipt. - if (chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && + if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" && !receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) { await db.transaction(async tx => { diff --git a/server/src/services/task-watchdog-work.ts b/server/src/services/task-watchdog-work.ts new file mode 100644 index 00000000000..98794472da2 --- /dev/null +++ b/server/src/services/task-watchdog-work.ts @@ -0,0 +1,49 @@ +import { and, asc, eq, gt } from "drizzle-orm"; +import { issueWatchdogs, type Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import type { IssueRecoveryScope } from "./issue-recovery-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; + +export function registerTaskWatchdogWork( + db: Db, + coordinator: ReturnType, + owner: { + enabled(): Promise; + reconcile(scope: { companyId: string; watchdogId: string; onDeferredDeadline: (at: number) => void }): Promise<{ incomplete: number }>; + }, + canRun: () => boolean, +) { + const cursors = new Map(); + let enabled = false; + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.taskWatchdogs, + reconstructionIntervalMs: 30_000, + beforePass: async () => { enabled = await owner.enabled(); }, + decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as IssueRecoveryScope, + matches: (entry, scope) => !scope || entry.companyId === scope.companyId, + load: async scope => { + const key = scope?.companyId ?? "*"; + if (!enabled) { cursors.delete(key); return []; } + const cursor = cursors.get(key); + const rows = await db.select({ id: issueWatchdogs.id, companyId: issueWatchdogs.companyId }) + .from(issueWatchdogs).where(and(eq(issueWatchdogs.status, "active"), + scope ? eq(issueWatchdogs.companyId, scope.companyId) : undefined, + cursor ? gt(issueWatchdogs.id, cursor) : undefined)) + .orderBy(asc(issueWatchdogs.id)).limit(100); + const complete = rows.length < 100; + if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); + return { rows: rows.map(row => ({ ...row, dueAt: Date.now() })), complete }; + }, + check: async entry => { + if (!enabled) return null; + let next: number | null = null; + const result = await owner.reconcile({ companyId: entry.companyId, watchdogId: entry.id, + onDeferredDeadline: at => { next = next === null ? at : Math.min(next, at); } }); + if (result.incomplete > 0) throw new Error(`Task watchdog ${entry.id} in company ${entry.companyId} has an incomplete subtree`); + return next === null ? null : { ...entry, dueAt: next }; + }, + }, canRun); +} diff --git a/server/src/services/task-watchdogs.ts b/server/src/services/task-watchdogs.ts index cf8d92d39df..ec0f13b7ac1 100644 --- a/server/src/services/task-watchdogs.ts +++ b/server/src/services/task-watchdogs.ts @@ -1,3 +1,5 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; +import { notifyIssueRecovery } from "./issue-recovery-notifications.js"; import { createHash } from "node:crypto"; import { and, asc, eq, gte, inArray, isNull, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -831,7 +833,7 @@ async function updateIssueWatchdogRow( return updated; } -export async function upsertIssueWatchdogForIssue( +async function upsertIssueWatchdogInTransaction( dbOrTx: any, companyId: string, issueId: string, @@ -847,12 +849,13 @@ export async function upsertIssueWatchdogForIssue( .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows: IssueWatchdogRow[]) => rows[0] ?? null); + await notifyIssueRecovery(dbOrTx, { companyId }); if (existing) { const updated = await updateIssueWatchdogRow(dbOrTx, existing, input, now); return { watchdog: toIssueWatchdog(updated), created: false }; } - const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx + const insertResult: { row: IssueWatchdogRow; created: boolean } = await dbOrTx.transaction(async (tx: Db) => tx .insert(issueWatchdogs) .values({ companyId, @@ -870,7 +873,7 @@ export async function upsertIssueWatchdogForIssue( updatedAt: now, }) .returning() - .then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true })) + .then((rows: IssueWatchdogRow[]) => ({ row: rows[0], created: true }))) .catch(async (error: unknown) => { if (!isIssueWatchdogUniqueConflict(error)) throw error; const winner = await dbOrTx @@ -885,6 +888,15 @@ export async function upsertIssueWatchdogForIssue( return { watchdog: toIssueWatchdog(insertResult.row), created: insertResult.created }; } +export function upsertIssueWatchdogForIssue( + dbOrTx: Db | Parameters[0]>[0], + companyId: string, + issueId: string, + input: IssueWatchdogUpsertInput, +): Promise<{ watchdog: IssueWatchdog; created: boolean }> { + return dbOrTx.transaction(tx => upsertIssueWatchdogInTransaction(tx, companyId, issueId, input)); +} + export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) { const issuesSvc = issueService(db); @@ -1398,10 +1410,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }) ?? fallback : fallback; if (!shouldReopen && watchdogIssue.originFingerprint !== input.classification.stopFingerprint) { - await db + await db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.watchdog.companyId); + await tx .update(issues) .set({ originFingerprint: input.classification.stopFingerprint, updatedAt: new Date() }) .where(and(eq(issues.companyId, input.watchdog.companyId), eq(issues.id, watchdogIssue.id))); + }); watchdogIssue.originFingerprint = input.classification.stopFingerprint; } await issuesSvc.addComment( @@ -1478,7 +1493,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) return created; } - async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null } = {}) { + async function evaluateWatchdog(row: IssueWatchdogRow, opts: { runId?: string | null; onDeferredDeadline?: (at: number) => void } = {}) { const watchdog = await markTerminalWatchdogIssueReviewed(row, opts); const sourceIssue = await db .select() @@ -1491,6 +1506,13 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) const input = await collectClassifierInput(watchdog.companyId, watchdog); const classification = classifyTaskWatchdogSubtree(input); + if (classification.state === "pending_first_run") { + const pending = new Set(classification.pendingIssueIds); + for (const issue of input.issues) if (pending.has(issue.id)) { + const createdAt = toEpochMs(issue.createdAt); + if (createdAt !== null) opts.onDeferredDeadline?.(createdAt + TASK_WATCHDOG_FIRST_RUN_GRACE_MS); + } + } if (classification.state !== "stopped") { return { state: classification.state, reason: classification.reason, classification }; } @@ -1610,13 +1632,14 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }; } - async function listActiveWatchdogsForCompany(companyId?: string | null) { + async function listActiveWatchdogsForCompany(companyId?: string | null, watchdogId?: string) { return db .select() .from(issueWatchdogs) .where(and( eq(issueWatchdogs.status, "active"), ...(companyId ? [eq(issueWatchdogs.companyId, companyId)] : []), + watchdogId ? eq(issueWatchdogs.id, watchdogId) : undefined, )); } @@ -1746,14 +1769,16 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) issueId: string, actor: ActorFields = {}, ): Promise => { - await assertWatchedIssue(db, companyId, issueId); - const existing = await db + return db.transaction(async tx => { + await assertWatchedIssue(tx, companyId, issueId); + const existing = await tx .select() .from(issueWatchdogs) .where(and(eq(issueWatchdogs.companyId, companyId), eq(issueWatchdogs.issueId, issueId))) .then((rows) => rows[0] ?? null); if (!existing || existing.status === "disabled") return null; - const [updated] = await db + await notifyIssueRecovery(tx, { companyId }); + const [updated] = await tx .update(issueWatchdogs) .set({ status: "disabled", @@ -1765,14 +1790,17 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) .where(eq(issueWatchdogs.id, existing.id)) .returning(); return toIssueWatchdog(updated); + }); }, reconcileTaskWatchdogs: async (opts: { companyId?: string | null; runId?: string | null; + watchdogId?: string; + onDeferredDeadline?: (at: number) => void; issueCreatedAtGte?: Date | null; } = {}) => { - let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null); + let rows = await listActiveWatchdogsForCompany(opts.companyId ?? null, opts.watchdogId); if (opts.issueCreatedAtGte) { const watchdogIssueIds = [...new Set(rows.map((row) => row.issueId))]; const eligibleIssueIds = new Set( @@ -1802,7 +1830,7 @@ export function taskWatchdogService(db: Db, deps: TaskWatchdogServiceDeps = {}) }; for (const row of rows) { result.checked += 1; - const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null }); + const evaluated = await evaluateWatchdog(row, { runId: opts.runId ?? null, onDeferredDeadline: opts.onDeferredDeadline }); if (evaluated.state === "triggered") { result.triggered += 1; result.watchdogIssueIds.push(evaluated.watchdogIssueId); diff --git a/server/src/services/workspace-runtime.ts b/server/src/services/workspace-runtime.ts index 600d5e32a42..2bb9239eaf3 100644 --- a/server/src/services/workspace-runtime.ts +++ b/server/src/services/workspace-runtime.ts @@ -1,3 +1,4 @@ +import { notifyStatusCardIssueChange } from "./status-card-notifications.js"; import { insertIssueComment } from "./issue-comment-writes.js"; import { restoreGitWorkspaceContents } from "./git-workspace-contents.js"; import { savedGitWorkspaceRestoration, ensureSavedGitCommit, GitCacheRestorationFailure } from "./git-workspace-restoration.js"; @@ -1636,10 +1637,11 @@ async function writeDirtyQuarantineAuditComments(input: { let sourceAuditCommentId: string | null = null; let claimantAuditCommentId: string | null = null; const now = new Date(); - if (input.evidence.sourceIssueId) { + const sourceIssueId = input.evidence.sourceIssueId; + if (sourceIssueId) { const [sourceComment] = await insertIssueComment(input.db, { companyId: input.companyId, - issueId: input.evidence.sourceIssueId, + issueId: sourceIssueId, authorAgentId: null, authorUserId: null, authorType: "system", @@ -1647,10 +1649,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); sourceAuditCommentId = sourceComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, input.evidence.sourceIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, sourceIssueId)); + }); } const claimantIssueId = input.evidence.contention?.claimedByIssueId ?? null; @@ -1665,10 +1667,10 @@ async function writeDirtyQuarantineAuditComments(input: { body, }); claimantAuditCommentId = claimantComment?.id ?? null; - await input.db - .update(issues) - .set({ updatedAt: now }) - .where(eq(issues.id, claimantIssueId)); + await input.db.transaction(async tx => { + await notifyStatusCardIssueChange(tx, input.companyId); + await tx.update(issues).set({ updatedAt: now }).where(eq(issues.id, claimantIssueId)); + }); } return { sourceAuditCommentId, claimantAuditCommentId }; diff --git a/tests/e2e/disconnect-write-recovery.spec.ts b/tests/e2e/disconnect-write-recovery.spec.ts index 8f81ee178dc..780f6f27a04 100644 --- a/tests/e2e/disconnect-write-recovery.spec.ts +++ b/tests/e2e/disconnect-write-recovery.spec.ts @@ -178,10 +178,19 @@ test("replaying an accepted comment does not reopen completed work", async ({ pa } }); -test("startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { +for (const delayWorkspacePreference of [false, true]) test(delayWorkspacePreference + ? "startup title survives session recovery while workspace preferences remount the editor" + : "startup work retries a failed session lookup while other APIs stay healthy", async ({ page, request }) => { const company = await json(await request.post("/api/companies", { data: { name: `Session recovery ${Date.now()}` } })); const session = await json(await request.get("/api/auth/get-session")); let healthy = false, saving = false, failedAfterSave = 0; + let releasePreference!: () => void; + const preferenceGate = new Promise(resolve => { releasePreference = resolve; }); + let sawPreference!: () => void; + const preferenceRequested = new Promise(resolve => { sawPreference = resolve; }); + if (delayWorkspacePreference) await page.route("**/api/workspace-preferences/me", async route => { + sawPreference(); await preferenceGate; await route.continue(); + }); try { const issue = await json(await request.post(`/api/companies/${company.id}/issues`, { data: { title: "Before session recovery", status: "backlog" }, @@ -199,12 +208,24 @@ test("startup work retries a failed session lookup while other APIs stay healthy saving = true; await header.locator("textarea").press("Enter"); await expect.poll(() => failedAfterSave).toBeGreaterThan(0); + if (delayWorkspacePreference) { + await expect(page.getByText("Couldn't update the task", { exact: true })).toBeVisible(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => resolve(null)))); + } healthy = true; + if (delayWorkspacePreference) { + // Returning to the tab retries session lookup while the saved edit waits. + await page.evaluate(() => window.dispatchEvent(new Event("visibilitychange"))); + await preferenceRequested; + await expect(page.getByTestId("task-chat-history-content")).toBeHidden(); + releasePreference(); + } await expect.poll(async () => (await json(await request.get(`/api/issues/${issue.id}`))).title, { timeout: 15_000 }).toBe("Saved after session recovery"); await page.reload(); await expect(header.getByRole("heading", { name: "Saved after session recovery", exact: true })).toBeVisible(); } finally { + releasePreference(); await request.patch(`/api/companies/${company.id}`, { data: { status: "archived" } }); } }); diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 895005cd15a..98864325189 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -34,6 +34,7 @@ never start a server or depend on a tenant filesystem. | `protected-actions` | Native agent; four work runs; authenticated reachable review-service fixture | Approve one action and decline another on separate owned tasks; verify zero calls before either decision, one after approval, zero after decline and a truthful saved explanation | | `create-skill-studio` | Native agent with skill-write permission; one work run | Create a uniquely named company skill, open its activity card, edit in Studio, reload and verify task-card readback | | `stop-redirect` | Native agent; three work runs; qualified Docker oracle | Download a verified checkpoint, Stop an active revision, require native cancellation acknowledgement, submit a new direction once and verify completion without original-run activity | +| `followup-while-running` | Native agent; a maximum of two work runs | Save a follow-up while an independently observed native shell command is still active, then require one attributed final reply with the late requirement after reload; in-turn steering and one normal successor are accepted | | `cloud-onboarding` | Signed-in admitted Cloud account; fresh workspace; qualified bounded provider route | Create workspace in the portal, connect API key and native Codex agent in the wizard, complete a task, download and independently check its artifact | The question-resumption journey uses the same natural-language scenario, browser @@ -228,6 +229,37 @@ its disposable agent, makes owned tasks terminal, verifies no active runs, retai the records for decision-history integrity, and releases both QA leases. This is reported as retained data, rather than confirmed deletion. +## Active follow-up + +The `followup-while-running` story uses the same ordinary task composer and saved +reply observer on managed self-hosted and attached Cloud targets. The initial +request includes one bounded `sleep 90` command. It supplies no changed launch +day or final marker. The observer requires a valid running native turn and a +pending named builtin command or process execution before sending the follow-up. +Paperclip semantic tools, including completion/block/progress calls, cannot +satisfy this boundary even if a provider labels them builtin execute or wraps +their names with a native prefix. Unnamed or ambiguous builtin calls fail closed. +The observed older PRP v1 builtin `bash` shape is also supported when its +operation is `unknown` and `readOnly` is explicitly false; other unknown +operations cannot satisfy the boundary. If all owned work settles before an +active boundary is observed, the journey fails immediately rather than polling +until the campaign deadline. +The same command must remain active after exactly one user message is saved. +The task page and composer are ready before observing the boundary, so navigation is outside the +submission window. The longer bounded command leaves room for remote persistence. +Merely observing a running heartbeat or waiting for a fixed interval cannot satisfy the boundary. + +The late message changes the launch day and supplies a unique final marker. +The independent oracle requires the expected reply from one bound native run, +one or two successful work runs in total, an empty saved-message queue, Done, +and visible readback after reload. A provider can steer the original turn or +queue one normal successor; retries, duplicate replies and unrelated work fail. +Source/model/environment checks and full usage collection remain required. +Only the closed boundary receipt and attributed final reply are retained in the +private story directory. No local workspace path or provider-side fixture hook +is required. The registered source and unit calibration do not qualify a live +Cloud target; admission to scheduled testing requires its own fresh pass. + ## Evidence and lifecycle Results use the existing campaign/result schemas and dashboard. Each result diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1e98adc848b..cac4fcc784a 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "followup-while-running", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); diff --git a/tests/runner-e2e/clarification-readback.spec.ts b/tests/runner-e2e/clarification-readback.spec.ts new file mode 100644 index 00000000000..087457d5707 --- /dev/null +++ b/tests/runner-e2e/clarification-readback.spec.ts @@ -0,0 +1,34 @@ +import { expect, test } from "@playwright/test"; +import { waitForClarificationDocument } from "./product-journeys/clarification-flow.js"; + +const answer = "Use AMBERreadback."; +const selector = '[id="document-welcome-note"]:visible, [id="side-panel-content-document:welcome-note"]:visible'; + +for (const stage of ["mount", "text"] as const) { + test(`clarification readback waits beyond the default timeout for ${stage}`, async ({ page }) => { + const body = `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(stage === "mount" ? "
" : '
Loading
'); + await page.evaluate(({ stage, body }) => { + setTimeout(() => { + if (stage === "mount") document.querySelector("main")!.innerHTML = `
${body}
`; + else document.querySelector("section")!.textContent = body; + }, 5_200); + }, { stage, body }); + await waitForClarificationDocument(page.locator(selector), answer, Date.now() + 8_000); + await expect(page.locator(selector)).toHaveText(body); + }); +} + +test("clarification readback rejects an exhausted campaign before waiting", async ({ page }) => { + await page.setContent(`
Welcome to the afternoon meetup. ${answer}
`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() - 1)) + .rejects.toMatchObject({ code: "clarification_document_not_ready" }); +}); + +for (const invalid of ["duplicate", "wrong-reference", "wrong-choice"] as const) { + test(`clarification readback rejects ${invalid} evidence within the campaign deadline`, async ({ page }) => { + const body = invalid === "wrong-reference" ? "Welcome to the afternoon meetup. Use OLD." : invalid === "wrong-choice" ? answer : `Welcome to the afternoon meetup. ${answer}`; + await page.setContent(`
${body}
${invalid === "duplicate" ? `
${body}
` : ""}`); + await expect(waitForClarificationDocument(page.locator(selector), answer, Date.now() + 250)).rejects.toThrow(); + }); +} diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 683f170c1e7..b0259b504d3 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,8 @@ function associate(suites: readonly string[], cases: Record; +export interface ActiveFollowupBoundary { + runId: string; + executionId: string; + startedSeq: number; + sourceEventId: string; + sourceInstanceId: string; + sourceSeq: number; + startedRowSha256: string; +} + +/** A claimed run or elapsed delay is not proof that native work is active. */ +export function activeFollowupBoundary(run: Row, events: Row[]): ActiveFollowupBoundary | null { + if (!activeNativeTurnStarted(run, events) || !events.length + || events.some((event, index) => event.seq !== index + 1)) return null; + const bound = events.filter(event => { + const envelope = event.payload?.prpEvent; + return envelope && isValidNativePrpEnvelope(envelope, event.protocolSchemaVersion) + && event.runId === run.id && event.companyId === run.companyId && event.agentId === run.agentId + && envelope.runId === run.id && envelope.eventType === event.eventType && envelope.sourceKind === "runner" + && typeof envelope.sourceEventId === "string" && Boolean(envelope.sourceEventId) && envelope.sourceEventId === event.sourceEventId + && typeof envelope.sourceInstanceId === "string" && Boolean(envelope.sourceInstanceId) && envelope.sourceInstanceId === event.sourceInstanceId + && Number.isSafeInteger(envelope.sourceSeq) && envelope.sourceSeq > 0 && envelope.sourceSeq === event.sourceSeq; + }); + const latestTurn = bound.filter(event => /^turn\.(started|completed|failed|cancelled|interrupted)$/.test(event.eventType)).at(-1); + if (latestTurn?.eventType !== "turn.started") return null; + const starts = bound.filter(event => { + const payload = event.payload.prpEvent.payload; + return event.seq > latestTurn.seq && event.eventType === "tool.execution.started" && payload?.schema === "paperclip.tool.execution.v1" + && typeof payload.executionId === "string" && Boolean(payload.executionId) + && ["builtin", "process"].includes(payload.transport) + && (payload.operation === "execute" + // Older native packs classify builtin bash as unknown. Recognize only + // that named v1 shell shape; unknown semantic/custom tools still fail. + || event.protocolSchemaVersion === 1 && payload.operation === "unknown" + && payload.transport === "builtin" && payload.name === "bash" && payload.readOnly === false) + // Provider display titles vary, but Paperclip semantic calls never prove + // native command work, even when a provider maps them to builtin execute. + && payload.namespace == null + && (payload.transport === "process" || typeof payload.name === "string" && Boolean(payload.name.trim()) + && !/^(?:unknown|tool(?:[ _-]+call)?)$/i.test(payload.name.trim()) + && !/(?:^|[._:/-])paperclip(?:[._:/-]|$)/i.test(payload.name.trim())) + && payload.readOnly !== true && payload.status === "running"; + }); + const pending = starts.filter(start => { + const id = start.payload.prpEvent.payload.executionId; + return starts.filter(other => other.payload.prpEvent.payload.executionId === id).length === 1 + && !events.some(event => event.eventType === "tool.execution.completed" + && event.payload?.prpEvent?.payload?.executionId === id); + }); + const started = pending.at(-1); + if (!started) return null; + const envelope = started.payload.prpEvent; + return { runId: run.id, executionId: envelope.payload.executionId, startedSeq: started.seq, + sourceEventId: envelope.sourceEventId, sourceInstanceId: envelope.sourceInstanceId, sourceSeq: envelope.sourceSeq, + startedRowSha256: createHash("sha256").update(JSON.stringify(started)).digest("hex") }; +} + +export function sameActiveFollowupBoundary(before: ActiveFollowupBoundary, after: ActiveFollowupBoundary | null) { + return Boolean(after && Object.keys(before).every(key => before[key as keyof ActiveFollowupBoundary] === after[key as keyof ActiveFollowupBoundary])); +} + +/** Accept in-turn steering or one normal successor; reject duplicate delivery. */ +export function activeFollowupOutcome(input: { + issueId: string; agentId: string; before: ActiveFollowupBoundary; after: ActiveFollowupBoundary | null; + runs: Row[]; expectedReply: string; marker: string; replies: Array<{ runId: string; body: string }>; + storedFollowupCount: number; queuedCommentCount: number; finalIssueStatus: string; +}) { + const ids = input.runs.map(run => run.id); + const matching = input.replies.filter(reply => Boolean(input.marker) && reply.body.includes(input.marker)); + return sameActiveFollowupBoundary(input.before, input.after) + && input.storedFollowupCount === 1 && input.queuedCommentCount === 0 && input.finalIssueStatus === "done" + && input.runs.length >= 1 && input.runs.length <= 2 && new Set(ids).size === ids.length && ids.includes(input.before.runId) + && input.runs.every(run => run.agentId === input.agentId && run.status === "succeeded" && run.runtimeMode === "native" + && (run.nativeIssueId ?? run.issueId ?? run.contextSnapshot?.issueId ?? run.contextSnapshot?.taskId) === input.issueId) + && matching.length === 1 && matching[0]!.body === input.expectedReply && ids.includes(matching[0]!.runId) + && (input.runs.length === 1 || matching[0]!.runId !== input.before.runId); +} diff --git a/tests/runner-e2e/product-journeys/active-followup-flow.ts b/tests/runner-e2e/product-journeys/active-followup-flow.ts new file mode 100644 index 00000000000..2746b5ba89e --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup-flow.ts @@ -0,0 +1,110 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { createTaskThroughUi, submitTaskReply } from "../user-actions.js"; +import { collectRunEvents } from "../run-observations.js"; +import { nativeFinalReplyEvents, readBoundTaskFinalReply } from "../native-task-reply.js"; +import { fixtureReplyStoredOnce, validateAgentTaskState, waitFor } from "./agent-flows.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +type Row = Record; +const id = "followup-while-running"; + +export function activeFollowupStory(campaignId: string) { + const nonce = campaignId.replace(/[^a-z0-9]/gi, ""); + const reference = `BRIEF${nonce}`, marker = `UPDATED${nonce}`; + const expectedReply = `Launch Friday. Reference ${reference}. Update ${marker}.`; + return { + first: `Our launch is planned for Monday. Run the bounded command sleep 90 exactly once before answering, so I can send additional context while you work. After the command finishes, briefly acknowledge the launch day and reference ${reference}, then finish this task. No files, projects or child tasks are needed.`, + followup: `Change the launch day to Friday. When ready, reply with exactly this one line and finish: ${expectedReply}`, + reference, marker, expectedReply, + }; +} + +/** Once owned work has settled, an unobserved live boundary cannot arrive. */ +export async function observeActiveFollowupCommand(taskRuns: Row[], loadEvents: (runId: string) => Promise) { + if (taskRuns.length > 0 && taskRuns.every(run => run.status === "succeeded")) + throw new JourneyError("failed", "followup_native_command_not_active"); + const run = taskRuns.find(run => run.status === "running"); + return { run, boundary: run ? activeFollowupBoundary(run, await loadEvents(run.id)) : null }; +} + +/** The same browser/API/native story runs against managed and attached targets. */ +export async function runActiveFollowup(context: JourneyContext, session: TargetSession, campaignId: string, + evidence: JourneyEvidence, options: { deadlineAt: number; directory: string }) { + const agent = context.agent; + if (!agent || agent.runtimeMode !== "native" || !context.environmentId) throw new JourneyError("unsupported", "native_agent_and_environment_required"); + const story = activeFollowupStory(campaignId), title = `Journey QA active followup ${campaignId}`; + const check = (name: string, passed: boolean) => { + evidence.checks.push({ id: name, passed }); + if (!passed) throw new JourneyError("failed", name.replaceAll("-", "_")); + }; + await mkdir(options.directory, { recursive: true, mode: 0o700 }); + const retain = (name: string, value: unknown) => writeFile(path.join(options.directory, name), JSON.stringify(value, null, 2), { mode: 0o600 }); + const created = await createTaskThroughUi({ page: context.page, origin: context.api.origin, issuePrefix: context.company.issuePrefix, + agentName: agent.name, title, prompt: story.first, workMode: "standard", requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: title, journeyId: id }) }); + await session.own({ kind: "issue", name: title, id: created.issueId, journeyId: id }); + const route = `/api/issues/${created.issueId}`; + const read = async () => { + const [currentIssue, summaries, comments, queue] = await Promise.all([ + context.api.get(route), context.api.get(`${route}/runs`), context.api.get(`${route}/comments?order=asc`), + context.api.get(`${route}/queued-comments`), + ]); + if (!Array.isArray(summaries) || summaries.length > 2) throw new JourneyError("failed", "followup_run_limit_exceeded"); + const taskRuns = await Promise.all(summaries.map(summary => context.api.get(`/api/heartbeat-runs/${summary.runId}`))); + validateAgentTaskState(context, created.issueId, { currentIssue, taskRuns, comments, interactions: [], attachments: [] }, 2); + if (currentIssue.description !== story.first) throw new JourneyError("failed", "followup_initial_prompt_changed"); + return { currentIssue, taskRuns, comments, queue }; + }; + const events = (runId: string) => collectRunEvents((afterSeq, limit) => + context.api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=${limit}`)); + const issue = (await read()).currentIssue; + await context.page.goto(`${context.api.origin}/${context.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`, { waitUntil: "domcontentloaded" }); + await expect(context.page.getByTestId("task-chat-composer-input").last()).toBeVisible({ timeout: 30_000 }); + const beforeComments = (await read()).comments; + const active = await waitFor(options.deadlineAt, async () => { + const state = await read(); + return observeActiveFollowupCommand(state.taskRuns, events); + }, value => Boolean(value.boundary), "followup_native_command_not_active"); + const before = active.boundary!; + await submitTaskReply(context.page, story.followup); + const persisted = await waitFor(Math.min(options.deadlineAt, Date.now() + 30_000), read, state => state.comments.some(comment => comment.authorUserId === context.user.id + && !beforeComments.some(old => old.id === comment.id)), "followup_not_persisted"); + const afterRun = persisted.taskRuns.find(run => run.id === before.runId)!; + const after = activeFollowupBoundary(afterRun, await events(before.runId)); + await retain("followup-boundary.json", { before, after }); + check("followup-native-command-active", sameActiveFollowupBoundary(before, after)); + check("followup-stored-once", fixtureReplyStoredOnce(beforeComments, persisted.comments, context.user.id, story.followup)); + + const final = await waitFor(options.deadlineAt, async () => { + const state = await read(); + const replies = [] as Array<{ runId: string; body: string }>; + for (const run of state.taskRuns.filter(run => run.status === "succeeded")) { + const stream = await events(run.id); + const decision = run.resultJson?.presentationDecision?.commentId; + const posted = state.comments.filter(comment => comment.id === decision && comment.authorAgentId === agent.id && comment.createdByRunId === run.id); + if (posted.length) replies.push(...posted.map(comment => ({ runId: run.id, body: comment.body }))); + else replies.push(...nativeFinalReplyEvents(stream, run).map(reply => ({ runId: run.id, body: reply.body }))); + } + return { state, replies }; + }, value => activeFollowupOutcome({ issueId: created.issueId, agentId: agent.id, before, after, + runs: value.state.taskRuns, expectedReply: story.expectedReply, marker: story.marker, replies: value.replies, + storedFollowupCount: value.state.comments.filter(comment => comment.authorUserId === context.user.id + && comment.body?.replaceAll("\\_", "_") === story.followup).length, + queuedCommentCount: Array.isArray(value.state.queue.entries) ? value.state.queue.entries.length : -1, + finalIssueStatus: value.state.currentIssue.status }), "followup_final_outcome_missing"); + check("followup-consumed-once", true); + const finalRunId = final.replies.find(reply => reply.body === story.expectedReply)!.runId; + const companies = await context.api.get>("/api/companies?scope=accessible"); + if (!Array.isArray(companies) || !companies.some(company => company.id === context.company.id) || companies.some(company => !company.issuePrefix)) throw new JourneyError("failed", "reply_company_context_missing"); + const reply = await readBoundTaskFinalReply({ page: context.page, runId: finalRunId, agentId: agent.id, + expectedText: story.expectedReply, deadlineAt: options.deadlineAt, knownCompanyPrefixes: companies.map(company => company.issuePrefix), + load: async () => { const latest = await read(); return { run: latest.taskRuns.find(run => run.id === finalRunId)!, comments: latest.comments, events: await events(finalRunId) }; } }); + await retain("followup-final.json", { before, after, runIds: final.state.taskRuns.map(run => run.id), reply }); + check("followup-final-answer-persisted", reply.observedAfterReload === true); + check("followup-one-or-two-native-runs", final.state.taskRuns.length >= 1 && final.state.taskRuns.length <= 2); + await expect(context.page.getByText(story.expectedReply, { exact: true }).last()).toBeVisible(); + check("configured-model-and-environment", true); +} diff --git a/tests/runner-e2e/product-journeys/active-followup.test.ts b/tests/runner-e2e/product-journeys/active-followup.test.ts new file mode 100644 index 00000000000..728160c610c --- /dev/null +++ b/tests/runner-e2e/product-journeys/active-followup.test.ts @@ -0,0 +1,174 @@ +import { describe, expect, it } from "vitest"; +import { canonicalProviderEventsFromCodex, canonicalProviderEventsFromAcpxRuntimeEvent, canonicalProviderEventsFromOpenCodePart } from "../../../packages/paperclip-runner/src/provider-events.js"; +import { activeFollowupBoundary, activeFollowupOutcome, sameActiveFollowupBoundary } from "./active-followup-evidence.js"; +import { activeFollowupStory, observeActiveFollowupCommand } from "./active-followup-flow.js"; +import { journeyRequiredChecks } from "./success-contract.js"; +import { parseTargetConfig, preparationPreview, agentJourneyRunCountValid } from "./catalog.js"; +import { validateRetainedRunnerResult } from "../result-validation.js"; + +const run = { id: "run", companyId: "company", agentId: "agent", issueId: "issue", runtimeMode: "native", status: "running" }; +function event(seq: number, type: string, payload = {}): Record { + return { seq, eventType: type, runId: run.id, companyId: run.companyId, agentId: run.agentId, + protocolSchemaVersion: 2, sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, + payload: { prpEvent: { schema: "paperclip.prp.event.v2", schemaVersion: 2, eventType: type, runId: run.id, + sourceKind: "runner", sourceEventId: `source-${seq}`, sourceInstanceId: "source", sourceSeq: seq, payload } } }; +} +const command = { schema: "paperclip.tool.execution.v1", executionId: "wait", transport: "builtin", operation: "execute", name: "bash", status: "running" }; +const events = () => [event(1, "turn.started"), event(2, "tool.execution.started", command)]; +const boundary = () => activeFollowupBoundary(run, events())!; +// Synthetic identity and filler rows; the v1 operation/transport/name/readOnly +// shape and native/source sequence positions match observed deployment metadata. +function legacyEvents(change: Record = {}): Record[] { + const rows: Record[] = Array.from({ length: 81 }, (_, index) => ({ seq: index + 1, eventType: "adapter.message" })); + const bind = (seq: number, sourceSeq: number, type: string, payload: Record) => { + const row = event(seq, type, payload); + row.protocolSchemaVersion = 1; row.sourceSeq = sourceSeq; + Object.assign(row.payload.prpEvent, { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceSeq }); + return row; + }; + rows[70] = bind(71, 9, "turn.started", { status: "inProgress" }); + rows[80] = bind(81, 12, "tool.execution.started", { ...command, namespace: null, operation: "unknown", readOnly: false, ...change }); + return rows; +} +const outcome = () => ({ issueId: "issue", agentId: "agent", before: boundary(), after: boundary(), + runs: [{ ...run, status: "succeeded" }], expectedReply: "Friday UPDATED", marker: "UPDATED", replies: [{ runId: "run", body: "Friday UPDATED" }], + storedFollowupCount: 1, queuedCommentCount: 0, finalIssueStatus: "done" }); + +describe("deployment-independent active follow-up", () => { + it("withholds the changed requirement and result marker until the follow-up", () => { + const story = activeFollowupStory("campaign-123"); + expect(story.first).toContain("sleep 90"); expect(story.first).toContain(story.reference); + expect(story.first).not.toContain(story.marker); expect(story.first).not.toContain("Friday"); + expect(story.followup).toContain(story.expectedReply); + }); + it("requires a bound running native turn and an actual pending shell execution", () => { + expect(boundary()).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 2 }); + for (const rows of [[], events().slice(0, 1), [...events(), event(3, "tool.execution.completed", { ...command, status: "completed" })], + [...events(), event(3, "turn.completed")], [...events(), event(3, "turn.started")], [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "read" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, readOnly: true })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_finish" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: "paperclip_block" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, namespace: "paperclip" })], + [events()[0]!, event(2, "tool.execution.started", { ...command, name: null })], + [events()[0]!, event(2, "tool.execution.started", { ...command, transport: "mcp" })], + [events()[0]!, event(3, "tool.execution.started", command)], + [...events(), event(3, "tool.execution.started", command)]]) expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = events(); rows[1]![field] = "wrong"; expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + const rows = events(); rows[0]!.payload.prpEvent.sourceKind = "control_plane"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + for (const changed of [{ status: "succeeded" }, { runtimeMode: "legacy" }]) expect(activeFollowupBoundary({ ...run, ...changed }, events())).toBeNull(); + }); + it("accepts canonical Codex command text and ACP display titles without provider-name coupling", () => { + const codex = canonicalProviderEventsFromCodex("item/started", { + item: { id: "wait", type: "commandExecution", command: "sleep 90", status: "inProgress" }, + })[0]!; + const acp = canonicalProviderEventsFromAcpxRuntimeEvent({ + type: "tool_call", tag: "tool_call", toolCallId: "wait", title: "Bounded settlement fixture", kind: "execute", status: "pending", + } as never, "fallback")[0]!; + expect(codex.payload).toMatchObject({ transport: "process", operation: "execute", name: "sleep 90", status: "running" }); + expect(acp.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "Bounded settlement fixture", readOnly: false, status: "running" }); + for (const mapped of [codex, acp]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, mapped.eventType, mapped.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + } + }); + it("accepts the real OpenCode shell fixture and rejects semantic calls normalized as builtin execute", () => { + const mapped = (tool: string, callID = "") => canonicalProviderEventsFromOpenCodePart({ + id: "wait", type: "tool", tool, callID, state: { status: "pending", input: { command: "sleep 90" } }, + })[0]!; + const shell = mapped("bash"); + expect(shell.payload).toMatchObject({ transport: "builtin", operation: "execute", name: "bash", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, shell.payload)])).toMatchObject({ runId: "run", startedSeq: 2 }); + for (const semantic of [mapped("paperclip_finish"), mapped("paperclip_block"), mapped("paperclip_report_progress"), + mapped("paperclip_paperclip_finish"), mapped("unknown", "functions.paperclip_block:0"), + mapped("functions.paperclip_finish"), mapped("mcp__paperclip__paperclip_block"), mapped("paperclip.paperclip_report_progress")]) { + expect(semantic.payload).toMatchObject({ transport: "builtin", operation: "execute", readOnly: false, status: "running" }); + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, semantic.eventType, semantic.payload)])).toBeNull(); + } + for (const name of [null, "", " ", "unknown", "tool", "tool call", "tool_call", "tool-call"]) { + expect(activeFollowupBoundary(run, [event(1, "turn.started"), event(2, shell.eventType, { ...shell.payload, name })])).toBeNull(); + } + }); + it("recognizes the observed legacy v1 builtin bash shape without admitting unknown operations generally", () => { + const before = activeFollowupBoundary(run, legacyEvents()); + expect(before).toMatchObject({ runId: "run", executionId: "wait", startedSeq: 81, sourceSeq: 12 }); + for (const change of [{ name: "unknown" }, { name: "custom-command" }, { name: "paperclip_finish" }, { name: "paperclip_block" }, + { name: "paperclip_report_progress" }, { name: "functions.paperclip_finish" }, { readOnly: true }, { readOnly: undefined }, + { transport: "process" }, { namespace: "paperclip" }]) expect(activeFollowupBoundary(run, legacyEvents(change))).toBeNull(); + for (const field of ["companyId", "agentId", "runId", "sourceEventId", "sourceInstanceId", "sourceSeq", "protocolSchemaVersion"]) { + const rows = legacyEvents(); rows[80]![field] = "foreign"; + expect(activeFollowupBoundary(run, rows)).toBeNull(); + } + expect(activeFollowupBoundary(run, [events()[0]!, event(2, "tool.execution.started", { ...command, operation: "unknown", readOnly: false })])).toBeNull(); + const after = legacyEvents(); + after.push(event(82, "tool.execution.progressed", { ...after[80]!.payload.prpEvent.payload })); + expect(sameActiveFollowupBoundary(before!, activeFollowupBoundary(run, after))).toBe(true); + after.push(event(83, "tool.execution.completed", { ...after[80]!.payload.prpEvent.payload, status: "completed" })); + expect(activeFollowupBoundary(run, after)).toBeNull(); + }); + it("fails promptly after owned work settles without an observed active boundary", async () => { + let eventReads = 0; + const load = async () => { eventReads++; return legacyEvents(); }; + for (const taskRuns of [[{ ...run, status: "succeeded" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "succeeded" }]]) + await expect(observeActiveFollowupCommand(taskRuns, load)).rejects.toMatchObject({ outcome: "failed", code: "followup_native_command_not_active" }); + expect(eventReads).toBe(0); + for (const taskRuns of [[], [{ ...run, status: "queued" }], [{ ...run, status: "succeeded" }, { ...run, id: "second", status: "queued" }]]) + expect((await observeActiveFollowupCommand(taskRuns, load)).boundary).toBeNull(); + expect(eventReads).toBe(0); + expect((await observeActiveFollowupCommand([run], load)).boundary).toMatchObject({ runId: "run", startedSeq: 81 }); + expect(eventReads).toBe(1); + }); + it("proves the same execution remains active after the user message is stored", () => { + expect(sameActiveFollowupBoundary(boundary(), boundary())).toBe(true); + for (const changed of [null, { ...boundary(), executionId: "other" }, { ...boundary(), startedRowSha256: "changed" }]) + expect(sameActiveFollowupBoundary(boundary(), changed)).toBe(false); + }); + it("accepts one steered run or one normal successor with the unique late answer", () => { + expect(activeFollowupOutcome(outcome())).toBe(true); + const value = outcome(); value.runs.push({ ...value.runs[0]!, id: "successor" }); value.replies[0]!.runId = "successor"; + expect(activeFollowupOutcome(value)).toBe(true); + }); + it("rejects lost input, duplicate delivery/work, stale answers and unfinished queues", () => { + const changes = [ + (v: ReturnType) => { v.storedFollowupCount = 0; }, + (v: ReturnType) => { v.storedFollowupCount = 2; }, + (v: ReturnType) => { v.queuedCommentCount = 1; }, + (v: ReturnType) => { v.finalIssueStatus = "in_progress"; }, + (v: ReturnType) => { v.replies = []; }, + (v: ReturnType) => { v.replies.push(v.replies[0]!); }, + (v: ReturnType) => { v.replies[0]!.body = "Monday"; }, + (v: ReturnType) => { v.replies.push({ runId: "run", body: "UPDATED again" }); }, + (v: ReturnType) => { v.replies[0]!.runId = "other"; }, + (v: ReturnType) => { v.runs[0]!.status = "failed"; }, + (v: ReturnType) => { v.runs[0]!.agentId = "other"; }, + (v: ReturnType) => { v.runs[0]!.issueId = "other"; }, + (v: ReturnType) => { v.runs = []; }, + (v: ReturnType) => { v.runs.push(v.runs[0]!); }, + (v: ReturnType) => { v.runs.push({ ...v.runs[0]!, id: "successor" }); }, + ]; + for (const change of changes) { const value = outcome(); change(value); expect(activeFollowupOutcome(value)).toBe(false); } + }); + it("allows a maximum of two declared runs while preserving every other exact bound", () => { + for (const count of [1, 2]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(true); + for (const count of [undefined, 0, 3, NaN, 1.5]) expect(agentJourneyRunCountValid("followup-while-running", count)).toBe(false); + expect(agentJourneyRunCountValid("build-revise", 1)).toBe(false); + const config = { version: 1, id: "qa", mode: "existing-cloud", adapterModule: "/private/provider.mjs", journeys: ["followup-while-running"], + executionEnvironment: "daytona", baseline: { features: {}, connections: [] }, limits: { durationMs: 600_000, modelBudgetCents: 100, maxRuns: 2 } }; + expect(preparationPreview(parseTargetConfig(config)).journeys[0]).toMatchObject({ id: "followup-while-running", paid: true }); + expect(() => parseTargetConfig({ ...config, limits: { ...config.limits, maxRuns: 1 } })).toThrow("agent_journeys_require_bounded_runs"); + }); + it("retains one-run and two-run passes only with the whole success contract", () => { + const result = { schema: "paperclip.runner-e2e.result/v2", executionId: "product-journeys.qa.configured-agent.daytona.followup-while-running", + suiteId: "product-journeys", suiteDefinitionHash: "test", profileId: "configured-agent", environmentId: "daytona", caseId: "followup-while-running", + provider: "opencode", model: "configured-model", runtimeMode: "native", status: "passed", cleanup: "passed", attempt: 1, + startedAt: "2026-01-01T00:00:00Z", finishedAt: "2026-01-01T00:00:01Z", durationMs: 1000, runIds: ["run"], + productJourney: { outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks["followup-while-running"]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://example.test", commit: "a".repeat(40), cloudRelease: "b".repeat(40), runtimeImage: "pinned", configurationHash: "test" } } }; + for (const runIds of [["run"], ["run", "successor"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).not.toThrow(); + for (const runIds of [[], ["a", "b", "c"]]) expect(() => validateRetainedRunnerResult({ ...result, runIds })).toThrow("result.agentJourneyRun"); + for (const check of result.productJourney.checks) expect(() => validateRetainedRunnerResult({ ...result, productJourney: { ...result.productJourney, + checks: result.productJourney.checks.filter(row => row.id !== check.id) } })).toThrow("result.productJourney"); + }); +}); diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index 37ccb8995bf..0cc797c0ae5 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -18,7 +18,7 @@ import { agentJourneyRuns } from "./catalog.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; type Row = Record; -type AgentJourney = Exclude; +type AgentJourney = Exclude; interface TaskState { currentIssue: Row & { id: string; status: string }; taskRuns: Array; comments: Row[]; interactions: Array; attachments: StoryDelivery[] } interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } const active = (run: Row) => ["queued", "running"].includes(run.status); @@ -46,7 +46,7 @@ export function validateAgentTaskState(context: JourneyContext, issueId: string, } } -async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { +export async function waitFor(deadlineAt: number, load: () => Promise, accept: (value: T) => boolean, code: string): Promise { while (Date.now() < deadlineAt) { const value = await load(); if (accept(value)) return value; diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 3d40c2cdc66..3013851a360 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -1,5 +1,5 @@ import { agentJourneyRuns } from "./success-contract.js"; -export { agentJourneyRuns } from "./success-contract.js"; +export { agentJourneyRuns, agentJourneyRunCountValid } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, @@ -13,6 +13,7 @@ export const productJourneys = [ { id: "protected-actions", label: "Approve and decline protected service actions", requires: ["company", "board-user", "task-write", "agent", "review-service"], capabilities: ["questions-and-approvals/tool-review"], paid: true }, { id: "create-skill-studio", label: "Create a company skill and persist a Skill Studio edit", requires: ["company", "board-user", "task-write", "agent", "skill-write"], capabilities: ["skills/authoring"], paid: true }, { id: "stop-redirect", label: "Stop active work and send a new direction once", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/stop-and-pause"], paid: true }, + { id: "followup-while-running", label: "Send a follow-up during active work and verify its saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["steering/queue"], paid: true }, { id: "cloud-onboarding", label: "Create a Cloud workspace and complete a first task", requires: ["cloud-account", "fresh-workspace", "provider-key"], capabilities: ["onboarding/first-task"], paid: true }, ] as const; diff --git a/tests/runner-e2e/product-journeys/clarification-flow.ts b/tests/runner-e2e/product-journeys/clarification-flow.ts index 932efbebf92..800a747284d 100644 --- a/tests/runner-e2e/product-journeys/clarification-flow.ts +++ b/tests/runner-e2e/product-journeys/clarification-flow.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { expect } from "@playwright/test"; +import { expect, type Locator } from "@playwright/test"; import { continuationScenario } from "../continuation-cases.js"; import { continuationAnswerCommitted, continuationCheckpointReady } from "../continuation-readiness.js"; import type { ContinuationCheckpoint } from "../continuation-scoring.js"; @@ -13,6 +13,19 @@ import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSes type Row = Record; interface Options { deadlineAt: number; directory: string; capture(id: string, label: string, file: string): Promise } +export async function waitForClarificationDocument(visible: Locator, answer: string, deadlineAt: number) { + const remaining = () => { + const timeout = deadlineAt - Date.now(); + if (!Number.isFinite(timeout) || timeout <= 0) throw new JourneyError("failed", "clarification_document_not_ready"); + return timeout; + }; + await expect(visible).toHaveCount(1, { timeout: remaining() }); + await expect(visible).toBeVisible({ timeout: remaining() }); + await expect(visible).toContainText(answer, { timeout: remaining() }); + await expect(visible).toContainText(/afternoon/i, { timeout: remaining() }); + remaining(); +} + export async function readClarificationState(context: JourneyContext, issueId: string) { const route = `/api/issues/${issueId}`; const [currentIssue, summaries, interactions, documents, attachments, children] = await Promise.all([ @@ -96,8 +109,7 @@ export async function runClarificationJourney(context: JourneyContext, session: await context.page.goto(new URL(href, context.api.origin).href, { waitUntil: "domcontentloaded" }); await context.page.reload({ waitUntil: "domcontentloaded" }); const visible = context.page.locator(`[id=${JSON.stringify(`document-${document.key}`)}]:visible, [id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`); - await expect(visible).toHaveCount(1); await expect(visible).toBeVisible(); - await expect(visible).toContainText(scenario.answer); await expect(visible).toContainText(/afternoon/i); + await waitForClarificationDocument(visible, scenario.answer, options.deadlineAt); const persisted = await context.api.get(`/api/issues/${issue.id}/documents/${encodeURIComponent(document.key)}`); readback = { issueId: persisted.issueId, key: persisted.key, revisionId: persisted.latestRevisionId, body: persisted.body, visibleBody: await visible.innerText() }; await options.capture("reloaded", "Saved answers after reload", "clarification-reloaded.png"); diff --git a/tests/runner-e2e/product-journeys/clarification-readback.test.ts b/tests/runner-e2e/product-journeys/clarification-readback.test.ts new file mode 100644 index 00000000000..5ec51654630 --- /dev/null +++ b/tests/runner-e2e/product-journeys/clarification-readback.test.ts @@ -0,0 +1,49 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { Locator } from "@playwright/test"; + +const clock = vi.hoisted(() => ({ now: 0, locator: {} })); +const answer = "Use AMBERreadback."; +const body = `Welcome to the afternoon meetup. ${answer}`; + +function stateAt(now: number) { + return { count: now >= 100 ? 1 : 0, visible: now >= 100, + text: now >= 1_300 ? body : now >= 600 ? answer : "Loading" }; +} + +vi.mock("@playwright/test", async importOriginal => { + const actual = await importOriginal(); + const wait = (accept: (state: ReturnType) => boolean, options: { timeout: number }) => { + if (!Number.isFinite(options.timeout) || options.timeout <= 0) throw Error("Invalid wait budget"); + if (accept(stateAt(clock.now))) return; + const deadline = clock.now + options.timeout; + const readyAt = [100, 600, 1_300].find(time => time > clock.now && time <= deadline && accept(stateAt(time))); + clock.now = readyAt ?? deadline; + if (readyAt === undefined) throw Error("Document not ready within wait budget"); + }; + return { ...actual, expect: (locator: Locator) => { + if (locator !== clock.locator) throw Error("Unexpected document locator"); + return { + async toHaveCount(count: number, options: { timeout: number }) { wait(state => state.count === count, options); }, + async toBeVisible(options: { timeout: number }) { wait(state => state.visible, options); }, + async toContainText(text: string | RegExp, options: { timeout: number }) { + wait(state => typeof text === "string" ? state.text.includes(text) : text.test(state.text), options); + }, + }; + } }; +}); + +import { waitForClarificationDocument } from "./clarification-flow.js"; + +afterEach(() => vi.restoreAllMocks()); + +it("shares one document deadline and retains rejection evidence through a later scheduler stall", async () => { + clock.now = 0; + vi.spyOn(Date, "now").mockImplementation(() => clock.now); + const observed = await waitForClarificationDocument(clock.locator as Locator, answer, 1_000).then( + () => ({ outcome: "resolved", at: clock.now, text: stateAt(clock.now).text }), + (error: Error) => ({ outcome: "rejected", error: error.message, at: clock.now, text: stateAt(clock.now).text }), + ); + clock.now += 10_000; + expect(stateAt(clock.now).text).toBe(body); + expect(observed).toEqual({ outcome: "rejected", error: "Document not ready within wait budget", at: 1_000, text: answer }); +}); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 4be973f1e81..72706d53ba9 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -14,6 +14,7 @@ import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; +import { runActiveFollowup } from "./active-followup-flow.js"; import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; import type { RunnerE2EResult } from "../types.js"; @@ -106,7 +107,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "active-followup-flow.ts", "active-followup-evidence.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -203,6 +204,8 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri result.model = ready.agent.model; result.runtimeMode = ready.agent.runtimeMode; if (id === "ask-question") await askQuestion(ready, session!, campaignId, evidence); + else if (id === "followup-while-running") await runActiveFollowup(ready, session!, campaignId, evidence, + { deadlineAt, directory: path.join(outputDir, "private-stories", id) }); else if (id === "question-answer-resume") await runClarificationJourney(ready, session!, campaignId, evidence, { deadlineAt, directory: path.join(outputDir, "private-stories", id), capture: async (_id, _label, file) => { diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index f49db32a938..67d37578e3e 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -1,11 +1,11 @@ /** Closed, target-independent success requirements used by dispatch and retained evidence. */ export const agentJourneyRuns = { "ask-question": 1, "plan-revise-accept": 3, "build-revise": 2, - "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; + "followup-while-running": 2, "protected-actions": 4, "create-skill-studio": 1, "stop-redirect": 3, "question-answer-resume": 3 } as const; export function agentJourneyRunCountValid(id: string, count: unknown): boolean { if (!Object.hasOwn(agentJourneyRuns, id) || !Number.isSafeInteger(count)) return false; const maximum = agentJourneyRuns[id as keyof typeof agentJourneyRuns]; - const minimum = id === "question-answer-resume" ? 1 : maximum; + const minimum = id === "question-answer-resume" || id === "followup-while-running" ? 1 : maximum; return (count as number) >= minimum && (count as number) <= maximum; } export const journeyRequiredChecks: Record = { @@ -24,5 +24,6 @@ export const journeyRequiredChecks: Record = { "create-skill-studio": ["created-skill-persisted", "skill-creation-attributed", "skill-studio-edit-persisted", "configured-model-and-environment"], "stop-redirect": ["checkpoint-artifact-verified", "active-work-stopped", "stop-native-terminal-retained", "new-direction-completed", "stopped-run-remained-terminal", "stopped-run-no-later-events", "stop-preserved-download", "redirect-consumed-once", "user-reply-stored-once", "configured-model-and-environment"], + "followup-while-running": ["followup-native-command-active", "followup-stored-once", "followup-consumed-once", "followup-final-answer-persisted", "followup-one-or-two-native-runs", "configured-model-and-environment"], "cloud-onboarding": ["fresh-workspace", "ui-connection-and-native-agent", "first-task-complete", "downloadable-artifact-verified", "native-trace-verified"], }; diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 0f899efca5e..24f57c2cf7f 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -3,7 +3,7 @@ import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" - | "question-answer-resume" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; + | "question-answer-resume" | "task-search-filter" | "followup-while-running" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { kind: "self-hosted" | "cloud"; diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 28929783eb1..09d6bd6d6d7 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -11,6 +11,7 @@ import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; +import { journeyRequiredChecks } from "./product-journeys/success-contract.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -91,6 +92,30 @@ describe("trusted report catalog discovery", () => { expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]); }); + it.each([ + { caseId: "question-answer-resume", feature: "questions-and-approvals", capabilities: ["questions-and-approvals/questions", "questions-and-approvals/continuation"] }, + { caseId: "followup-while-running", feature: "steering", capabilities: ["steering/queue"] }, + ])("reports deployment journey $caseId in its feature coverage", ({ caseId, feature, capabilities }) => { + const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const execution = { ...known, id: `product-journeys.${known.profile.id}.${known.environment.id}.${caseId}`, + suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: caseId } }; + expect(journeyCapabilities(execution).map(capability => capability.id)).toEqual(capabilities); + expect(journeyFeatureIds(execution)).toEqual([feature]); + expect(journeyCapabilities({ ...execution, suite: { ...execution.suite, id: "unrecognized-suite" } })).toEqual([]); + const page = renderRunnerE2EDashboard({ title: "Deployment coverage", generatedAt: "2026-10-20T12:00:00Z", + expected: [execution.id], catalog: [execution], entries: [{ valid: true, errors: [], + result: { ...result(), executionId: execution.id, suiteId: execution.suite.id, profileId: execution.profile.id, + environmentId: execution.environment.id, caseId, provider: "opencode", runIds: ["run-1"], + productJourney: { targetId: "qa", outcome: "passed", startingState: "recipe", privacy: "private", resourceDisposition: "retained", + phases: [{ phase: "journey", outcome: "passed" }], checks: journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })), + deployment: { kind: "cloud", targetId: "qa", origin: "https://qa.example.test", commit: "a".repeat(40), + cloudRelease: "b".repeat(40), runtimeImage: "immutable-image", configurationHash: "config" } } } }] }); + expect(page).toContain(`data-execution-id="${execution.id}"`); + expect(page).toContain(`data-report-features="${feature}"`); + expect(page).toContain(`title="${capabilities.join(", ")}"`); + expect(page).toContain("1/1 passed"); + }); + it("counts the project/task deployment journey toward projects and task creation coverage", () => { const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } }; diff --git a/ui/src/hooks/useAccountBoundMutation.ts b/ui/src/hooks/useAccountBoundMutation.ts index b011b9ec8ad..5cbf156b703 100644 --- a/ui/src/hooks/useAccountBoundMutation.ts +++ b/ui/src/hooks/useAccountBoundMutation.ts @@ -5,68 +5,12 @@ import { QueryClientContext, type DefaultError, type MutateOptions, - type QueryClient, type UseMutationOptions, type UseMutationResult, type MutationFunctionContext, } from "@tanstack/react-query"; -import { queryKeys } from "@/lib/queryKeys"; -import { authApi } from "@/api/auth"; -import { classifySendFailure, PendingSendAccountChangedError } from "@/lib/pending-send"; - -type AccountGeneration = { owner: string | undefined; generation: number }; -const accounts = new WeakMap(); - -function currentOwner(client: QueryClient): string | undefined { - const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); - return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; -} - -function bindAccount(client: QueryClient) { - let account = accounts.get(client); - if (!account) { - account = { owner: currentOwner(client), generation: 0 }; - accounts.set(client, account); - const tracked = account; - client.getQueryCache().subscribe(event => { - const owner = currentOwner(client); - const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); - if (owner !== tracked.owner || removed) { - if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; - tracked.owner = owner; - } - }); - } - const { owner, generation } = account; - const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); - const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); - const whenAccountReady = async () => { - if (account.owner !== undefined || !isCurrentScope()) return; - let stopped = false; - let unsubscribe = () => {}; - const ready = new Promise(resolve => { - unsubscribe = client.getQueryCache().subscribe(() => { - if (account.owner !== undefined || !isCurrentScope()) resolve(); - }); - }); - const lookup = async () => { - for (let attempt = 0; attempt < 3; attempt += 1) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - try { - await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); - return; - } catch (error) { - if (stopped || account.owner !== undefined || !isCurrentScope()) return; - if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; - await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); - } - } - }; - try { await Promise.race([ready, lookup()]); } - finally { stopped = true; unsubscribe(); } - }; - return { isCurrentAccount, isCurrentScope, whenAccountReady }; -} +import { PendingSendAccountChangedError } from "@/lib/pending-send"; +import { bindAccount } from "@/lib/account-scope"; export function useCurrentAccountGuard(): () => boolean { const client = useContext(QueryClientContext); diff --git a/ui/src/hooks/useDurableAutosave.test.tsx b/ui/src/hooks/useDurableAutosave.test.tsx index edad11f04a0..af5860e0a85 100644 --- a/ui/src/hooks/useDurableAutosave.test.tsx +++ b/ui/src/hooks/useDurableAutosave.test.tsx @@ -173,6 +173,51 @@ describe("useDurableAutosave", () => { client.clear(); }); + it.each([false, true])("keeps an unowned startup edit when identity and unmount race (identityFirst=%s)", async identityFirst => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Retained startup title"); }); + expect(current.status).toBe("waiting"); + act(() => { + if (identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + root.unmount(); root = createRoot(container); + if (!identityFirst) client.setQueryData(queryKeys.auth.session, { user: { id: "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).toHaveBeenCalledExactlyOnceWith("Retained startup title"); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }); + + it.each(["different-account", "round-trip", "removed-session", "signed-out"])( + "does not adopt a later identity for a detached startup edit (%s)", async transition => { + store.setBrowserOnline(false); + const client = new QueryClient({ defaultOptions: { queries: { staleTime: Infinity, retry: false, enabled: false } } }); + const save = vi.fn().mockResolvedValue(undefined); + act(() => root.render( + + )); + await act(async () => { await current.run("Private startup title"); }); + act(() => { root.unmount(); root = createRoot(container); }); + act(() => { + if (transition === "removed-session") client.removeQueries({ queryKey: queryKeys.auth.session }); + else client.setQueryData(queryKeys.auth.session, transition === "signed-out" ? null : { user: { id: "user-1" } }); + if (transition === "round-trip") client.setQueryData(queryKeys.auth.session, { user: { id: "user-2" } }); + client.setQueryData(queryKeys.auth.session, { user: { id: transition === "different-account" ? "user-2" : "user-1" } }); + }); + await act(async () => { store.setBrowserOnline(true); store.reportSuccess(); }); + await advance(2_000); + expect(save).not.toHaveBeenCalled(); + expect(store.getSnapshot().pendingWrites).toBe(0); + client.clear(); + }, + ); + it("bounds retries after an editor unmounts", async () => { const save = vi.fn().mockRejectedValue(outage()); render({ save }); diff --git a/ui/src/hooks/useDurableAutosave.ts b/ui/src/hooks/useDurableAutosave.ts index 658d68cd149..0293975cc8b 100644 --- a/ui/src/hooks/useDurableAutosave.ts +++ b/ui/src/hooks/useDurableAutosave.ts @@ -1,4 +1,6 @@ -import { useCallback, useEffect, useId, useRef, useState } from "react"; +import { useCallback, useContext, useEffect, useId, useRef, useState } from "react"; +import { QueryClientContext } from "@tanstack/react-query"; +import { bindAccount } from "@/lib/account-scope"; import { classifyError, describeError, type ErrorDescription } from "@/api/errors"; import { loadStructuredDraft, saveStructuredDraft } from "@/lib/composer-draft"; import { useConnectivityStore } from "@/lib/connectivity"; @@ -108,6 +110,7 @@ const SAVED_LINGER_MS = 1600; * - Errors become readable copy for "Couldn't save (reason)". */ export function useDurableAutosave(options: DurableAutosaveOptions): DurableAutosave { + const client = useContext(QueryClientContext); const owner = useDraftOwner(); const readOwner = useDraftOwnerReader(); const instanceId = useId(); @@ -120,7 +123,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const targetKey = options.targetKey ?? options.sourceId; const targetRef = useRef(targetKey); targetRef.current = targetKey; - type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; draft: T; + type Job = { token: symbol; registryKey: string; resends: number; saving?: boolean; id: number; target: string; owner: string | undefined; account: ReturnType | null; draft: T; save: (draft: T) => Promise; matchesServer?: DurableAutosaveOptions["matchesServer"]; onRetryFailed?: DurableAutosaveJob["onRetryFailed"]; storageKey: string | null; action: string }; const jobs = useRef(new Map()); @@ -142,6 +145,22 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab entry?.active.delete(job.token); if (entry && !entry.active.size) registry.delete(job.registryKey); }; + // Resolve first identity from the same generation guard used by mutations. + // This also runs after unmount, when React cannot migrate the job for us. + function owned(job: Job) { + if (job.account && !job.account.isCurrentScope()) return false; + const current = readOwner(); + if (job.owner === undefined && current !== undefined) { + const nextKey = JSON.stringify([current, job.target]); + if (registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey)) return false; + releaseJob(job); + job.owner = current; + job.registryKey = nextKey; + registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); + } + return current === job.owner && generations.current.get(job.target) === job.id + && registry.get(job.registryKey)?.latest === job.token; + } const mounted = useRef(true); const savingTimer = useRef | null>(null); const savedTimer = useRef | null>(null); @@ -160,17 +179,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab if (previous === undefined && owner !== undefined) { for (const job of jobs.current.values()) { if (job.owner !== undefined) continue; - const nextKey = JSON.stringify([owner, job.target]); - const superseded = registry.get(job.registryKey)?.latest !== job.token || registry.has(nextKey); - releaseJob(job); - if (superseded) { + if (!owned(job)) { + releaseJob(job); jobs.current.delete(job.target); if (generations.current.get(job.target) === job.id) generations.current.delete(job.target); - continue; } - job.owner = owner; - job.registryKey = nextKey; - registry.set(nextKey, { latest: job.token, active: new Set([job.token]) }); } refresh(value => value + 1); return; @@ -194,12 +207,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab return stored?.version === 1 && stored.draft !== undefined ? stored.draft : null; }, [storageKey]); const execute = useCallback(async (job: Job, retry = false): Promise => { - const owned = () => readOwner() === job.owner && generations.current.get(job.target) === job.id && registry.get(job.registryKey)?.latest === job.token; - const visible = () => mounted.current && owned() && targetRef.current === job.target; + const stillOwned = () => owned(job); + const visible = () => mounted.current && stillOwned() && targetRef.current === job.target; const publish = (status: DurableAutosaveStatus, error: ErrorDescription | null = null) => { if (visible()) setView({ target: job.target, status, error }); }; - if (!owned()) { + if (!stillOwned()) { if (jobs.current.get(job.target) === job) jobs.current.delete(job.target); releaseJob(job); if (mounted.current) refresh(value => value + 1); @@ -234,7 +247,12 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab remove(); return { kind: "saved" }; }; - try { await job.save(job.draft); return done(); } + try { + if (!mounted.current && job.account && !job.account.isCurrentAccount()) await job.account.whenAccountReady(); + if (!stillOwned()) { remove(); return { kind: "waiting" }; } + await job.save(job.draft); + return done(); + } catch (initialCause) { let cause = initialCause; if (classifyError(cause) === "conflict" && job.matchesServer) { @@ -255,7 +273,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab const current = optionsRef.current; const target = current.targetKey ?? current.sourceId; const admittedOwner = ownerRef.current ?? readOwner(); - const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, draft, + const job: Job = { token: Symbol(), registryKey: JSON.stringify([admittedOwner, target]), resends: 0, id: ++sequence.current, target, owner: admittedOwner, account: client ? bindAccount(client) : null, draft, save: overrides.save ?? current.save, matchesServer: overrides.matchesServer ?? current.matchesServer, onRetryFailed: overrides.onRetryFailed, storageKey: current.storageKey ?? null, action: current.action ?? "save" }; const previous = jobs.current.get(target); @@ -266,7 +284,7 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab generations.current.set(target, job.id); jobs.current.set(target, job); refresh(value => value + 1); return execute(job); - }, [execute, readOwner]); + }, [client, execute, readOwner]); const waitingJobs = [...jobs.current.values()].filter(job => !job.saving).length; const auto = useAutoResend({ sourceId: options.sourceId, active: waitingJobs > 0, pendingCount: waitingJobs, resend: async (): Promise => { @@ -281,7 +299,6 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab detachedWork.current = () => { if (!jobs.current.size) return; if (auto.stalled) { for (const job of jobs.current.values()) releaseJob(job); jobs.current.clear(); return; } - const admittedOwner = ownerRef.current; const source = `${optionsRef.current.sourceId}:detached:${instanceId}`; let timer: ReturnType | null = null; let disposed = false, running = false; @@ -298,7 +315,11 @@ export function useDurableAutosave(options: DurableAutosaveOptions): Durab }; const wake = async () => { if (disposed || running) return; - if (mounted.current || readOwner() !== admittedOwner || admittedOwner === undefined) { release(); return; } + if (mounted.current) { release(); return; } + for (const job of jobs.current.values()) if (!owned(job)) { + releaseJob(job); + jobs.current.delete(job.target); + } if (!jobs.current.size) { release(); return; } store.setPendingWrites(source, jobs.current.size); if (store.getSnapshot().status !== "online") return; diff --git a/ui/src/lib/account-scope.ts b/ui/src/lib/account-scope.ts new file mode 100644 index 00000000000..0e7d050e502 --- /dev/null +++ b/ui/src/lib/account-scope.ts @@ -0,0 +1,59 @@ +import type { QueryClient } from "@tanstack/react-query"; +import { queryKeys } from "./queryKeys"; +import { authApi } from "@/api/auth"; +import { classifySendFailure } from "./pending-send"; + +type AccountGeneration = { owner: string | undefined; generation: number }; +const accounts = new WeakMap(); + +function currentOwner(client: QueryClient): string | undefined { + const session = client.getQueryData<{ user?: { id?: string }; session?: { userId?: string } } | null>(queryKeys.auth.session); + return session === undefined ? undefined : session?.user?.id ?? session?.session?.userId ?? "signed-out"; +} + +export function bindAccount(client: QueryClient) { + let account = accounts.get(client); + if (!account) { + account = { owner: currentOwner(client), generation: 0 }; + accounts.set(client, account); + const tracked = account; + client.getQueryCache().subscribe(event => { + const owner = currentOwner(client); + const removed = event.type === "removed" && JSON.stringify(event.query.queryKey) === JSON.stringify(queryKeys.auth.session); + if (owner !== tracked.owner || removed) { + if (removed || tracked.owner !== undefined || tracked.generation !== 0) tracked.generation += 1; + tracked.owner = owner; + } + }); + } + const { owner, generation } = account; + const isCurrentScope = () => account.generation === generation && (owner === undefined ? generation === 0 : account.owner === owner); + const isCurrentAccount = () => account.owner !== undefined && isCurrentScope(); + const whenAccountReady = async () => { + if (account.owner !== undefined || !isCurrentScope()) return; + let stopped = false; + let unsubscribe = () => {}; + const ready = new Promise(resolve => { + unsubscribe = client.getQueryCache().subscribe(() => { + if (account.owner !== undefined || !isCurrentScope()) resolve(); + }); + }); + const lookup = async () => { + for (let attempt = 0; attempt < 3; attempt += 1) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + try { + await client.fetchQuery({ queryKey: queryKeys.auth.session, queryFn: () => authApi.getSession(), retry: false }); + return; + } catch (error) { + if (stopped || account.owner !== undefined || !isCurrentScope()) return; + if (attempt === 2 || classifySendFailure(error) !== "pending") throw error; + await new Promise(resolve => setTimeout(resolve, 250 * 2 ** attempt)); + } + } + }; + try { await Promise.race([ready, lookup()]); } + finally { stopped = true; unsubscribe(); } + }; + return { isCurrentAccount, isCurrentScope, whenAccountReady }; +} +