diff --git a/doc/plans/2026-10-24-recurring-server-sleep-stack.md b/doc/plans/2026-10-24-recurring-server-sleep-stack.md index 4dc61f402f6..6d3ea3beff1 100644 --- a/doc/plans/2026-10-24-recurring-server-sleep-stack.md +++ b/doc/plans/2026-10-24-recurring-server-sleep-stack.md @@ -1853,3 +1853,320 @@ children pass 30 report/startup tests after propagation. Fresh remote gates stil apply. The next new reason is chat run-milestone projection, separate from wake notices, terminal confirmations, interaction continuations and GitHub recovery. The entire original inventory and older unresolved findings remain open. + + +Slice 28 native finalization checkpoint: existing native finalizer transactions +now signal milestone projection; governed review metadata restoration avoids +unchanged writes and signals only actual changes. Both missing wake and repeated +rewrite were reproduced before repair. All 16 milestone worker cases, nine +selected native chat recovery integrations, and server production types pass. +See the slice audit for failure logs and coverage limits. The production scan +remains until status-decision/effect, admission, lineage, publication, and remaining +producer coverage is complete. This is local progress, not a published migration +or completion of the full stack. + + +Slice 28 now also covers new native status-decision commits and pending-effect +receipts, after their no-op exits. Two regressions failed first; all 18 milestone +worker cases, nine native chat recovery integrations, and server production types +pass after repair. Real late continuation receipts now reactivate projection. +Next is heartbeat queue/admission/recovery producer coverage. Periodic projection +and all remaining inventory/remote acceptance duties remain outstanding. + + +Slice 28 queue checkpoint: issue admission, ordinary/comment claims, drain +release, coalesced lineage and native review claims now notify from their owning +transactions. Real missing-wake regressions were reproduced for admission, +coalescence and all ordinary/comment claim/release paths. The 24 milestone worker +cases and 92 existing queue/policy cases pass, as does server production typecheck. +The audit records the initial fixture FK failure, exact logs, remaining raw +cancellation/promotion/receipt writers, and native reviewer acceptance gap. +Periodic projection remains until producer coverage is complete; no new PR or +full-goal completion is claimed. + + +Slice 28 retry/recovery checkpoint: due retry promotion, suppressed retry and +stale-run cancellation, deferred wake promotion and immediate recovery admission +now signal from their existing transactions. Five real missing-notice regressions +were reproduced before repair; all 166 selected worker/module/policy cases and +server production types pass. The slice audit preserves the initial invalid +staleness fixture and corrected causal evidence. Periodic projection remains +until all supported producers are covered. Remaining original inventory, open +PR gates/findings and final master reconciliation are unchanged. + + +Slice 28 native wake receipt checkpoint: interrupted recovery, same-scope +coalescence and fresh admission receipt writes now notify from narrow database +transactions. Both missed-recovery and admission-before-receipt races were +reproduced. All 56 milestone/queue cases, 13 existing dependency cases and server +production types pass; four native cases were rerun after fixing external-test +fixture cleanup. The audit preserves setup failures and provider-acceptance +limits. No periodic projector has been removed yet. Fresh remote inspection +still finds #17590 Paperclip Review queued, #17592 neutral, and #17490 E2E shard +3 plus aggregate failing. Those gates and all remaining original scope stay open. + + +Slice 28 raw queue cancellation checkpoint: general chat-control recovery, +superseded case handoffs, stale scheduled retries and reassigned execution owners +now signal before terminal writes. Missing notices were reproduced for the first +two; replacement-admission tests cover the latter two and their early intent +placement is audited. All 60 milestone/queue tests and server production types +pass. Queued-comment/lineage and remaining dependency producers still prevent +removing periodic projection. No new PR, merge, or full-goal completion is claimed. + + +Slice 28 lineage checkpoint: queued-comment run-context rewrites, deletion and +cancellation signal through their existing adapter transaction. Real queue +reconstruction and caller-owned discard regressions failed before repair. The +native reviewer claim now has real database integration and causal signal-removal +coverage. All 74 selected worker/adapter/policy cases and server production types +pass. Remaining publication, interaction, configuration, question and monitor +producer coverage still gates periodic-projector removal. Original inventory, +remote gates/findings and final master reconciliation remain open. + +## Slice 28 draft and publication/interaction coverage + +[Draft #17606](https://github.com/paperclipai/paperclip/pull/17606) is stacked on +#17592. It adds the lifecycle projector worker and covered producer signals; +production registration and polling removal remain intentionally unfinished until +the remaining producer audit is complete. Native progress stays a separate slice. + +The latest local checkpoint covers real publication success/failure/operator +resolution and question answer/cancel/withdraw/skip/terminal-expiry paths, +including historical answers that enqueue no continuation. All 160 milestone and +interaction-service cases pass on fresh isolated PostgreSQL databases. Four +existing chat acceptance cases, publication policy cases, and server production +typecheck passed at their recorded checkpoints. See the detailed milestone audit +for retained regression/setup failures, controlled ports and remaining coverage. +The initial draft CI was running; no final green-check or readiness claim is made. + + +Slice 28 now also wakes lifecycle projection from confirmation accept/reject and +live/historical comment invalidation of questions and confirmations. Six real +producer regressions failed before the added intent signals. All 169 cases across +the worker and interaction-service suites pass, as does production server +typechecking. The draft remains incomplete: other producers and production +registration/polling removal are still pending. See the milestone audit for logs. + + +Slice 28 creation/sibling replacement now registers intent before the first +interaction write. Both question and confirmation replacement failed before the +fix; all 184 milestone/integration/unit interaction cases and production server +typechecking pass. Remaining terminal-interaction writers are document-target +expiry, deleted-addressee cancellation and startup supersession; other producer +audits and production migration remain pending. + + +Slice 28 covers the remaining direct terminal-interaction writers in the +interaction service: document-target expiry, deleted addressees and startup +supersession. Seven real-producer regressions failed before the fix; all 191 +worker/integration/unit cases and production server typechecking pass. Database +fixture setup/cleanup moved to a cohesive helper, keeping the worker test file +below 1,000 lines. The broader raw-writer audit still includes external services +(secret proposals, tool reviews, conversation reset and native review retirement), +plus the previously listed publication/configuration/recovery dependencies. + + +Slice 28 now covers binding-proposal terminal reflection and tool-review human +verdict/expiry reflection. Nine governed-producer cases (including outer rollback +and dependent rejection) pass with the lifecycle/interaction suites, 200 total; +all 109 existing tool-gateway cases and server production typechecking pass. +The audit retains the initial invalid tool-publication fixture assertion and +corrected before-fix failures. Native external-review supersession is already +covered by its status-decision outer transaction. Other external interaction and +lineage producers remain under audit; the draft is not production-wired yet. + + +Slice 28 covers native automatic/obsolete review retirement and conversation +reset with seven additional governed cases, including retained-history and +explicit-request controls. Three missing wakes were reproduced before repair; +207 lifecycle/governed/interaction cases pass. The draft CI chat-shard failure +was reproduced as a test subscribing to unrelated unresolved transaction intent; +isolating that test's DB owner preserves the production protocol and passes three +native review acceptance cases. Server production types pass. The slice audit +retains the invalid fixture and missed test-file selection. Polling removal, +remaining producers, fresh CI/review, older findings and the full inventory remain +unfinished; no stack merge or whole-goal completion is claimed. + + +Slice 28 also covers interactive callback recovery of an unknown prompt receipt. +Its existing Slack acceptance test failed before the answer-write barrier without +the receipt signal, then five Slack/Discord/Telegram cases and server production +types passed after the one-line integration. File transport and ordinary tool +receipt ordering were audited without adding unnecessary notifications. Remaining +configuration/recovery producers and production wiring still gate polling removal. + + +## Drain review repair and fresh stack gates + +The first broad stack refresh found #17576 Greptile finding 4370544173: dirty +in-memory hints kept the coordinator's settled-write activity hold alive during +drain. SQL silence alone had missed it. It failed a new real-PostgreSQL counter +assertion before repair. The owning parent is fixed at 1b43332c74; all checks now +pass and that review thread is resolved. Slice 22 carries the repair into its +shared dispatcher at b2af095de2. Open/uncertain writes remain fenced centrally; +settled hints and their next deadlines survive resume without holding idle active. + +The fix is propagated through #17580 (0aab051900), #17583 (c46bcbf9ff), #17586 +(6d99318a8d), #17590 (0bb6de2343), #17592 (6e40ac2570) and #17606 (71a84ee0c0). +Report and retention resume tests initially returned early because their flush +helper checked only the active counter. Those parent-owned tests now advance the +actual retained nextWakeAt and prove the durable result, plus unchanged idle +snapshots while paused. See 2026-10-25-voice-drain-repair.md for all retained +failure logs, including one invalid intermediate report-field assertion. + +Verified selection at the combined tip: +- 99 chat lifecycle/governed/voice recovery cases pass. +- 24 Teams consent/abandoned-attempt scheduler cases pass. +- 38 inbound cleanup, retention and call-report cases pass after test repairs. +- Production server typecheck passes. The 108-case full voice integration suite + passed at the shared-dispatch parent; strengthened reply drain passed separately. + +Fresh current-head rollup is saved at +/tmp/sleep-stack-checks-after-drain-20261025.json. #17490 still fails E2E shard 3 +and aggregate at 31bcd19888. #17518 Paperclip Review remains pending. #17578 and +newer propagated heads have pending checks, with no failures in that snapshot. +The inbound cleanup/company-deletion review and security finding is NOT repaired +by this drain change; current pending checks do not discharge that finding. +No blind reruns or merges into master occurred. + +Next actionable parent audit: tool health (#17554), Discord command registration +(#17557), and receiver discovery (#17572) still contain the same unconditional +hasPending dirty-scope predicate. Reproduce their idle-hold behavior, repair at +their owning branches, and propagate. Their currently passing CI is not evidence +against that missing idle-counter assertion. Then continue milestone configuration +and alternate-receipt coverage, production registration/poll removal, the entire +original inventory, older E2E/cleanup findings and final master reconciliation. + +## Configuration drain propagation and resource milestone producers + +Discord #17557 at 954a0faa75 and receiver #17572 at 8815dfad38 now carry the +confirmed drain-hold repairs, following tool health #17554 at e685a0e7ef. +The prerequisite #17561 inherited health at 05ac7835fc. Propagation reached +#17576 a4bc3ea50e, #17578 05797e2b9f, #17580 82d023a3a0, #17583 f4f7e14f7a, +#17586 920a951505, #17590 d626ff4b13, #17592 33ffff8a12, and #17606 9775b2bc4a. +All 109 configuration lifecycle/protocol cases pass together after applying the +tip schema to their disposable databases. See the configuration-worker drain +repair document for individual causal failures and parent evidence. + +Slice 28 adds real inventory/lifecycle resource reactivation wakes. The two +regressions fail before repair; all 90 milestone lifecycle/governed/resource cases +and six selected provider acceptance cases pass after it. The owning resource +module only wakes reopened issue scopes, never unchanged inventory or completed +history. See the milestone audit for full commands/results and remaining producers. + +Fresh gate snapshot /tmp/sleep-stack-gates-configuration-drain-20261025.json has +no failures for slices 1–12 and 14–16, plus #17505; #17490 still fails E2E shard 3 +and its aggregate. #17518/#17554 have pending review and later updated heads have +pending checks. New #17576 security finding 4371283330 concerns repeated +connection writes loading every live voice session in that connection. Inspect +and address it without dropping future deadlines or configuration revocation. +The existing #17580 cleanup/company-deletion denial finding remains open. No +review blocker is resolved merely by pushing inherited changes. No PRs merged. + +Origin/master refreshed to e335265568 (task-run UI polling removal #17608). +Full base reconciliation, remaining recurring-work slices, milestone startup +wiring, fresh reviews, and all latest-head CI gates remain required. + + +Slice 28 Telegram migration now registers affected issue intents in its owning +transaction before reactivating/rebinding conversations. The missing wake was +reproduced first. All 92 milestone/resource/governed tests, eight existing +Telegram migration cases, server typecheck and server build pass. Duplicate +migration and unchanged inventory remain quiet. See the milestone audit for +failure and verification logs. Inbound activation and remaining producer audits +still gate production wiring; no recurring projector has been removed yet. + +Fresh pre-push gates: #17606 at b634353059 has passing checks; #17572 at +8815dfad38 and #17490 at 31bcd19888 still fail E2E shard 3 and aggregate. +Three local cancellation diagnostic runs passed, including full projects coverage +and a database-only delay, without proving the CI deadlock's competing owner. +Those failures, parent security/deletion findings, all other original slices, +fresh remote review and reconciliation with master 7ebf98eafd remain open. + + +Slice 28 ordinary ingress now wakes existing milestone debt before reactivating +conversation bindings and writing inbound links. A real Slack ingress regression +failed before repair and passes without depending on a new run being created. +All 12 selected ingress/revocation/Telegram migration cases and server types/build +pass. The milestone audit records evidence, isolated DB, and remaining authority/ +lineage producers. This is another producer checkpoint in draft #17606, not +production polling removal or completion of the full requested stack. + + +Slice 28 now signals successful inbound action settlement and repaired delivery +acceptance for already-admitted runs. Both post-close missing notices reproduced +before repair. All 41 selected post-close/ingress/revocation/migration cases and +server types/build pass; recovery proves it does not admit a second wake. The +audit retains the initial contention-sensitive assertion and its correction to +honor the existing retry. Identity activation and remaining authority/lineage +producers still gate production polling removal. All original slices and older +parent CI/review findings remain required. + + +Slice 28 identity activation now signals the bound issue before changing the +responsible user. Two before-fix failures now pass with outer commit/rollback, +pending steering, replay and settled quiet checks. All 43 existing identity and +GitHub credential tests pass after repairing their external-DB fixture URL; +server types/build pass. Recovery terminalization/classification writers are the +next audit. Master 34409a92c4 was fetched without disturbing this draft. The +worker is still not production-registered and all original stack duties remain. + + +Slice 28 now covers orphan-run terminalization in stale-lock recovery. Both +real-producer missing-notice cases failed before repair and pass afterward; +all 40 existing stale-lock ownership/race cases and server types/build pass. +The conditional update and provider/process authority retain their existing +owners. Adapter-failure classification and remaining producer audits still gate +production registration; this does not migrate the stale-lock polling duty itself. + + +Slice 28 now covers the late adapter-failure classification write, which can +return an old failed run to the existing publication window after the issue's +earlier wake. The missing notice reproduced before repair; all three recovery +milestone cases, 89 recovery-action cases and server types/build pass. Four +classification callers reuse their issue identity, with no new queue table or +provider policy in shared code. Authority/configuration audit and production +wiring remain next; the original full stack and remote gates remain incomplete. + + +Slice 28 now covers nine real authority-restoration owners: connection/resource +reach, unlinked policy, identity confirmation, four membership paths and task +visibility. Seven initial causal failures plus connection/resource reproductions +now pass; the invalid Discord group-reach fixture is explicitly retained in the +failure record. Final local selection: 50 chat integrations, 18 access tests, +six action-worker tests, server typecheck/build. No provider authorization was +relaxed. Cloud trusted-header membership sync and board claim remain alternate +writers to cover before production registration/poll removal. Older parent +findings and the complete original stack remain open. + + +Slice 28 now covers Cloud trusted-header sync and board claim. The missing +Cloud notice and absent board-claim intents were reproduced before repair. +All 39 post-close cases plus 30 provisioning/auth/claim cases pass; server types +and build pass. Unchanged Cloud profile synchronization and active board membership +stay silent. See the milestone audit for transaction boundaries and retained logs. +The previous pushed head 0f38d1079d has the recurring E2E shard-3 cancellation +failure; process-identity event append deadlock evidence is retained for diagnosis. +Production registration/poll removal and the entire remaining stack are still open. + + +Slice 28 production milestone migration: the app now registers the scoped +lifecycle worker and the periodic coordinator invokes only native progress. +The final GitHub guest/sponsor settings owner signals actual authority changes +inside its existing transaction; its missing intent reproduced before repair. +159 worker/startup/coordinator cases, four GitHub acceptance cases and all 16 +assembled-app project-chat E2Es pass; server typecheck passes. The milestone +audit has a consolidated alternate-writer disposition table. Native progress +remains the next separate reason. Remote gates, parent findings and the rest of +the original stack are not complete; local cancellation success does not repair +the recorded CI deadlock. + + +Slice 28 review repair: cover GitHub observed-identity linking, suppress +unchanged native-finalization recovery wakes, and repair the voice callback and +Cloud-auth test contracts. Restore shared user profile after the Cloud authority +test so downstream Slack publication tests see their original identity. Local +40 stale-lock, 122 voice, 38 auth/session and 28 combined chat cases pass; server +typecheck/build pass. See the milestone audit for failed-run dispositions. +Latest-head remote gates and parent failures remain open. diff --git a/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md b/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md new file mode 100644 index 00000000000..045ba245b36 --- /dev/null +++ b/doc/plans/2026-10-25-chat-run-milestone-sleep-audit.md @@ -0,0 +1,1675 @@ +# Chat run-milestone projection sleep audit + +Baseline: Teams attempt recovery #17592 at f53ae3d6d2. +Branch: codex/sleep-28-chat-run-milestones. +Status: lifecycle worker registered in production; recurring lifecycle projection +removed. Selected local worker, authorization and assembled-app acceptance passes. +Final full-stack reconciliation, CI/review gates and older parent findings remain open. + +## Current production disposition + +The app registers `chatRunMilestoneWork` with its existing delivery coordinator. +The remaining periodic chat lane calls only `enqueueSafeNativeChatProgress` and +is named native progress in the coordinator and tests. The explicit/manual +combined reconciliation API remains available, with its original shared budget. +Publications created by the lifecycle worker notify the existing delivery owner. +The coordinator owns subscriptions, retained dirty scopes, retries and shutdown; +no independent timer or extra table is introduced. Cloud sleep proof is unchanged. + +The audit's remaining alternatives were classified against actual reader fields: + +| Alternate writer family | Disposition | +| --- | --- | +| Monitor semantic receipts | The real tool writes receipt plus monitor atomically before future admission; covered queue admission establishes the successor. | +| Retry scheduling in heartbeat/recovery/native replacement/capacity owners | Creates `scheduled_retry`, outside lifecycle selection. Covered promotion/claim introduces eligibility; coalescence counts and retry-strategy metadata do not supply lineage. | +| Native status dispatch and effect receipts | Existing issue-scoped signals cover original coalesced intent, dispatched wake/run binding and delivered effect. | +| Generic wake settlement | Ordinary queued→claimed/completed retains existing admission predicates; cancellation/skipping removes authority. Run status/queue producers already signal their corresponding run transition. No new general wake-count loop is needed. | +| Execution-wait, human-directed assignment and tree cancellation | Diagnostic skipped receipts have no run; human assignment has its distinct source; tree cancellation selects unclaimed null-run receipts. None creates an admitted monitor/interaction/native-status successor. | +| Task-retry, preserved-result and fresh-workspace receipts | Distinct task_retry/strategy metadata; actual successor run admission/finalization is covered. | +| Dot/Muse assignment follow-up receipts | Board conversation/issue-comment paths; they do not establish external lifecycle lineage merely by sharing an issue. | +| Slack Board reply bindings | Source issue.comment uses its own response path; it is excluded from lifecycle candidate sources. | +| GitHub configuration | The only configuration writer upserts; deletion is endpoint/company teardown rather than restoration. Upgrading disables automatic queued/working notices. Guest/sponsor changes can restore post-close authority and now signal before mutation in the existing locked transaction. | +| Initial local-board membership / agent compatibility backfill | Startup precedes worker reconstruction; agent-only memberships do not restore human authority. Live Cloud and board-claim human restoration now has signals. | + +This table complements the real producer cases and chronological audit below; +it does not claim independent SQL writers or another replica are covered. +The source of truth remains existing rows and the original 24-hour run window. +Company hints are used only for broader configuration/authority operations; +issue hints cover lineage and its descendants. Settled projection does not retain +periodic discovery merely because history or configuration exists. + +Latest local evidence: +- GitHub guest-setting transaction regression failed before its signal in + /tmp/sleep28-github-config-before.log; the production owner now emits intent + before mutation, settles after the outer commit, and stays silent for an + unchanged settings save. Four real GitHub settings/guest/authorization cases + pass in /tmp/sleep28-github-config-acceptance.log. +- All 159 cases in seven worker/lifecycle/identity/recovery/governed/resource, + app-coordinator and startup suites pass in + /tmp/sleep28-production-worker-acceptance.log. These include configured idle, + retained history, first producer, outer rollback/commit, retry, restart, + drain/resume and shutdown tests; not every case runs the full HTTP app. +- All 16 project-chat Playwright cases pass against a fresh assembled app at + port 32658, database paperclip_sleep28_production_e2e_20261025_test. + Log /tmp/sleep28-production-e2e.log; traces /tmp/sleep28-production-e2e-results. + This includes split-handoff cancellation and experiment restoration, but local + success does not resolve the observed remote cancellation deadlock. +- Server production typecheck passes in /tmp/sleep28-production-worker-typecheck.log. + The compiler excludes tests. Server build passes in + /tmp/sleep28-production-worker-build.log, and the four GitHub owner cases pass + again after the endpoint-deletion race guard in /tmp/sleep28-github-config-final.log. + No fleet suspension/cost result is claimed. Final remote gates remain required. + +## Historical implementation checkpoints + +The entries below preserve earlier incomplete states, failures and subsequent +repairs. Their statements that production still polls describe those checkpoints; +the current production disposition above supersedes them. + +## Separate obligations discovered + +The app's recurring chat coordinator invokes enqueueChatRunMilestones. The same +method also invokes enqueueSafeNativeChatProgress when the milestone batch does +not fill its limit. These are different reasons and must have separate migrations: + +1. Coarse run lifecycle projection: queued, working, waiting for input, completed, + failed. Existing heartbeat run state plus conversation/continuation lineage + determines whether a safe milestone is owed. This is slice 28. +2. Native progress-event projection: admitted heartbeat_run_events and current + run/presentation authority determine which safe progress update is publishable. + Keep this as the next separate reason; do not silently remove its only sweep. + +Both produce existing chat_publications and already notify publication delivery +before insert. No queue table should be introduced merely to migrate projection. +Existing publication identity is run/milestone/endpoint. Respect the existing +24-hour selection window until its semantics are explicitly audited. + +## Existing owner and correctness constraints + +server/src/services/chat-run-publications.ts owns lineage, source-authorized safe +text, final-comment suppression and GitHub owned-provider-interaction suppression. +Keep this policy there or in cohesive chat publication modules. Shared scheduler +and app wiring must not interpret run/provider payloads. This is projection, not +execution recovery; never dispatch another run to produce a missing notice. + +A succeeded run is not eligible until its durable presentation decision exists. +Explicit authored final publications suppress generic completion. Answered +questions, direct interactions, issue monitors and task continuations each supply +lineage. Run state alone is therefore not a complete producer inventory. Audit +writes to those associations, presentation decisions, final publications, +conversation bindings and policy/authority changes before removing the backstop. + +The current live-event fast path calls notifyPublications only while not draining +or in standby. It is not a durable commit protocol and can drop a first wake during +drain. Use supported transaction boundaries and the shared coordinator, preserve +outer commit/rollback and uncertain outcomes, and reconstruct existing debt on +startup. A zero inserted count can mean contention or suppression; it is not by +itself proof that no projection remains. Distinguish passive/suppressed history +from recoverable lock contention and dependency changes. + +Current pagination uses JavaScript Date for run updated_at and native event +created_at cursors. Audit precision when preserving or replacing it; PostgreSQL +submillisecond values must not duplicate/skip candidates or pin a page. Preserve +company fairness, bounded work and existing publication deduplication. + +## Required next investigation + +- Enumerate every supported lifecycle/presentation/lineage producer and its + existing notification/transaction owner. Inspect heartbeat boundary rules. +- Separate the combined function without changing the explicit/manual contract + or losing native progress while its own migration is pending. +- Define an observable unfinished-work predicate and a scoped candidate loader; + ordinary settled history must disarm. Avoid global scans for every run hint. +- Prove real normal completion, authored-final suppression, question/direct/monitor + continuations, configuration changes, backlog fairness, contention retry, + rollback, startup, drain/resume and shutdown. Keep external source authority. +- Keep current Cloud blockers and external alarm limits. Single active process + through the owning DB client remains the supported signal model. + +All earlier stack remote gates, #17490 E2E cancellation, #17580 unresolved cleanup +policy, and final master reconciliation remain required. No merge authority or +full-goal completion is implied by this audit. + + +## Projection boundary checkpoint + +`enqueueChatRunLifecycleMilestones` now owns only coarse lifecycle projection. +`enqueueSafeNativeChatProgress` exposes the existing native progress owner. +The existing `enqueueChatRunMilestones` entry point still combines them with +lifecycle priority, the same 24-hour default window, and one capped insertion +budget. Production still calls that combined entry point. No periodic owner has +been removed prematurely; this checkpoint is not completion of slice 28. + +The lifecycle cursor now carries PostgreSQL's timestamp text through a typed +`(updated_at, run_id, conversation_id)` tuple comparison. With 30 unauthorized +candidates at a submillisecond timestamp, the old Date cursor repeated a page +indefinitely, never reaching valid notices. The test caps candidate page reads +so reproducing the defect cannot leave an asynchronous SQL loop after a timeout. +Native progress has its own timestamp cursor; its precision audit belongs to +that next migration and is still required. + +### Producer coverage still to implement + +The current readers establish these dependencies. This table identifies signal +owners to audit; it does not claim those signals are already installed. + +| Dependency | Owning paths and required wake | +| --- | --- | +| Run admission/status/issue binding/error code | `heartbeat/queue.ts`, `heartbeat/run-lifecycle.ts`, and native/legacy recovery writers. Existing `notifyExecutionRunWork` covers only some paths. Audit raw writes before adding milestone coverage. | +| Presentation decision and native result | `heartbeat/run-completion.ts`, native finalizers, and semantic receipt writers. Succeeded alone is insufficient: presentation may settle later. Source monitor receipts can authorize a successor without changing the successor row. | +| Question response claim and terminal receipt | `question-response-delivery.ts`: claim moves to delivering; `recordTerminal` owns a transaction that records target run/status. A late receipt can redirect an existing run's publication destination. Lease timestamp renewal is not a new presentation obligation. | +| Interaction state and source-run ownership | `issue-thread-interactions.ts`, plus recovery writers. Pending agent-authored questions suppress generic finals even before publication; resolved prompts can retain that suppression. | +| Published prompts, final comments, and close controls | Publication settlement and `issues.ts`. Published prompt receipts enable lineage; authored finals suppress generic completion; a published close requires fresh independently admitted sources before later presentation. Publication notifications must avoid a global scan feedback loop. | +| Inbound comment links and conversation binding | Ingress and conversation owners. The run's source string alone grants no authority. Endpoint assignment, publication mode, conversation state/issue and verified comment links matter. | +| Continuation wake/decision/effect receipts | `native-continuation-delivery.ts` reads applied status decisions, delivered continuation effects, the original wake and dispatched wake, plus responsible-user continuity. Audit their writers; a run notification before these commits is too early. | +| Monitor wake and source semantic receipt | `native-monitor-delivery.ts` requires an admitted monitor wake plus the source run's exact saved `set_task_monitor` receipt. Audit both writers, including late completion. | +| Board reply identity and provider configuration | `slackBoardReplyBindings`, endpoint configuration, and upgraded GitHub configuration affect selection. Scope configuration invalidation to its company/endpoints; preserve each provider's existing policy. | + +`resolveChatOriginPublicationBindings` also follows arbitrarily long verified +lineage with cycle rejection. Changing an ancestor can affect a successor; a +strict run-ID-only invalidation is insufficient for all producer families. +Issue-scoped reconstruction is appropriate for lineage changes. Run-only hints +can be used only where the writer cannot affect descendants. Shared scheduling +must not decode provider context or implement this authorization graph. + +The coarse projector currently waits on its writes and does not use skip-locked +selection. A zero insertion count may be an authoritative suppression or a +concurrent idempotent insert; database failures must retain retry. Native progress +has different lock/contention behavior and must not inherit that conclusion. + +### Verification and failure record + +- Initial owner split: nine selected PostgreSQL integration cases passed, + including separate-owner/combined-budget checks and six provider/surface cases. +- First cursor reproduction incorrectly guarded every `db.select`, including + legitimate authorization reads, and ran against retained earlier fixtures. + Both precision variants failed that overly broad guard; this was not evidence + of the production cursor defect. Log: `/tmp/sleep28-precision-before.log`. +- Corrected guard counts candidate-page reads. In a fresh database the millisecond + case passed; the submillisecond case failed after 100 repeating pages. Log: + `/tmp/sleep28-precision-clean-before.log`. +- After retaining PostgreSQL precision, ten selected integration cases passed in + a fresh database. Log: `/tmp/sleep28-projection-fixed-tests.log`. +- Broader final selection: 31 integration cases passed (1,398 unselected), + covering milestone authority, native progress, lock contention, question + continuations and authored finals. Log: `/tmp/sleep28-projection-acceptance.log`. + These overlap the earlier ten and are not additive. +- All 20 publication policy unit cases passed; server production typecheck passed. + Logs: `/tmp/sleep28-publication-unit.log`, `/tmp/sleep28-server-typecheck.log`. + Server typecheck excludes test files. No full-workspace acceptance or new PR + readiness is claimed for this preparatory checkpoint. + +Parent #17592 at `f53ae3d6d27c2642e7dfe647a2c0a9bbf2d582ac` now has passing +Actions/security/Greptile gates, no inline review findings, and a neutral +Paperclip Review result. Neutral is not a completed positive review. #17590 at +`e501d6c99d721d6428f8c490206417765c76f4f6` has passing gates with Paperclip Review +queued at inspection. #17586 also has a neutral Paperclip Review. Final review +acceptance remains separate. Refreshed master is `2e47c42fad`; stack reconciliation +remains required. No stack PR was merged. + + +## Scoped worker and initial producer checkpoint + +`chat-run-milestone-work.ts` now reuses `registerScopedDeadlineWork`. Its loader +shares the lifecycle candidate selector with the explicit projector, retaining +all existing SQL eligibility fences and timestamp precision. It reconstructs +one entry per company/issue, with a bounded 100-publication projection batch. +A full batch remains immediately due; a short batch has exhausted the selector +and disarms, including currently suppressed lineage. Real selection/write +failures retain the shared retry backoff. The scheduler owns coalescing, +company fairness, first writes during drain, concurrent changes, and shutdown. +No new persistent queue or independent timer is introduced. + +`notifyChatRunMilestones` carries only company/issue identity through the existing +outer-transaction protocol. Issue-scoped invalidation includes descendants; +company scope is available for broader configuration or lifecycle changes. +Unrelated issues/companies are not reconstructed for an issue hint. + +Initial production integrations now await intent before writing: + +- `heartbeat/run-lifecycle.ts::writeRunStatus`: both shared status transition + callers signal the known company in their existing transaction. This is a + conservative company hint; it is not proof that all raw run writes are covered. +- `heartbeat/run-completion.ts`: the late presentation-decision write now owns + a short transaction and signals its issue (company fallback when unbound). + Provider execution and presentation generation remain outside that transaction. +- `question-response-delivery.ts::recordTerminal`: the existing transaction + signals the issue before recording the durable target run and outcome. + +The production app still uses the combined periodic projector. The new worker +is exercised through actual coordinator registration in tests but is not yet +registered by `app.ts`; do not remove the sweep until the remaining producer +families in the table above are implemented. In particular: queue admission and +recovery writers, native finalization and semantic receipts, question claim/retry +transitions, interaction changes, publication settlement, conversation/endpoint +changes, and decision/wake ledger changes remain to be covered. The late legacy +presentation signal is implemented, but its own full completion integration +proof is still required. Current Cloud blockers and single-active-writer limits +remain unchanged. + +### Worker and producer verification + +15 real-PostgreSQL worker cases pass in +`/tmp/sleep28-worker-question-tests.log`. They cover configured idle/internal +runs/old history, startup and restart, first committed work, outer commit and +nested rollback, first wake during drain, issue/company isolation, suppressed +lineage reactivation, late presentation eligibility, 205 notices plus another +company, selection failure through drain, publication-write retry, a concurrent +commit during projection, shutdown, real cancellation and real question delivery. +Quiet assertions advance two minutes and observe zero DB selects/transactions. +The deadline test verifies no scans during the ten seconds before retry. + +The two real producer proofs are causal, not just notification-spy assertions: + +- Removing the actual `writeRunStatus` signal makes the cancellation test fail: + the queued notice exists but the terminal notice is absent. Restoring it passes. + Log: `/tmp/sleep28-cancellation-without-signal.log`. +- The question test creates and answers a real interaction, supplies a durable + published prompt, and invokes the real delivery service. Its wake callback + inserts the continuation without a milestone signal and proves no notice is + projected yet. The terminal receipt then activates the worker. Removing that + exact signal strands the notice and fails the assertion. Log: + `/tmp/sleep28-question-without-signal.log`. + +The 31-case existing chat integration selection passes after the selector +refactor and producer additions, including presentation authority and native +progress. Log: `/tmp/sleep28-scoped-acceptance.log`. Expected contention/retry +warnings occurred, with all assertions passing. Server production typecheck +passes in `/tmp/sleep28-producer-typecheck.log`. No full-workspace or PR gate +completion is claimed yet. + +Preserved fixture failures: + +1. `/tmp/sleep28-worker-tests.log`: all 11 initial cases failed setup because the + fixture used provider name `slack` as connection transport. The schema requires + `chat_sdk`; no production transport change was made. +2. `/tmp/sleep28-worker-fixed-tests.log`: ten passed; the same-company isolation + fixture violated the application-name uniqueness constraint. Fixture names now + include their identity. +3. `/tmp/sleep28-worker-producer-tests.log`: eleven passed; cancellation required + a responsible-user membership, and teardown omitted its new run-event rows. + Both are now represented/cleaned by the fixture. +4. `/tmp/sleep28-worker-producer-fixed-tests.log`: assertions reached cancellation + projection, but default follow-on recovery admitted another execution and + teardown deadlocked against it. The test now uses cancellation's existing + `suppressImmediateRecovery` option to exercise Stop without starting successor + execution. This does not change the default production recovery policy. +5. All 12 corrected cases passed in `/tmp/sleep28-worker-final-tests.log`; two + concurrency/retry cases made 14 pass in `/tmp/sleep28-worker-expanded-tests.log`; + the real question case brings the latest total to 15. These counts overlap. + +This checkpoint is progress on slice 28, not a new independently complete reason +or a claim that milestone polling is retired. The original full PR-stack scope, +older unresolved CI/review findings, final master reconciliation and each PR's +latest-head acceptance remain required. + + +## Native finalization producer checkpoint + +Native finalization now registers issue-scoped milestone intent in its existing +transactions before failure recording, committed terminal projection, and the +normal final run projection. All three committed-response repair branches signal +only after their authorization/no-op guards and immediately before comment and +presentation writes. The status arbiter, provider interpretation, and execution +policy remain with their native owners; the milestone signal carries only company +and issue identity. + +Governed review-presentation restoration also signals an actual change. Previously +it rewrote identical metadata (including updated_at) on every accepted repair. +The owner now locks the run, compares the exact proposed JSON merge against the +current result, and returns successfully without a write or signal when unchanged. +It still returns true for an already valid projection: callers may need to +materialize the response even when its metadata is already present. Existing +coordinator/proof/digest/company checks remain before this decision. + +Evidence at this checkpoint: + +- The new real-finalizer regression failed before the signal: a persisted committed + native cancellation projected its terminal run state but never generated the + terminal chat notice. Log: `/tmp/sleep28-native-before.log`. +- After wiring, all 16 milestone-worker PostgreSQL cases pass. The new case calls + the real finalizer without a manual milestone notification and verifies that + replaying the settled finalization causes no deferred worker SQL. Its input is + a seeded committed audit-only result; this is a recovery-producer proof, not a + test of accepting provider results. Log: `/tmp/sleep28-native-worker.log`. +- The new governed-review repair test failed first because an unchanged repair + advanced updated_at. Log: `/tmp/sleep28-review-before.log`. +- After repair, nine selected real PostgreSQL chat integration cases pass. They + cover unchanged metadata, changed metadata intent before the write and settlement + after commit, repeat repair, Slack/Telegram accepted-response recovery, canonical + response recovery, already-selected/deleted answer protection, contention, and + review authority changes. Expected contention is retained as retryable. + Log: `/tmp/sleep28-native-recovery-tests.log` (1,421 cases not selected). +- Server production typecheck passes: `/tmp/sleep28-native-typecheck.log`. + It excludes test files. No full-workspace or remote PR acceptance is claimed. + +The periodic projector remains enabled. Next producer audit: status-decision +commit/replay and pending-effect delivery, then admission/recovery, interaction, +publication, conversation/configuration, and monitor/semantic-receipt writers +from the table above. Signals added here do not establish complete producer +coverage. The full original stack, unresolved earlier gates, and master +reconciliation remain required; no PR has been merged by this checkpoint. + + +## Native decision and effect-receipt checkpoint + +`commitNativeStatusDecision` registers company/issue milestone intent after the +existing committed/replayed-decision exits, before inserting or applying a new +decision and its effects. `dispatchPendingNativeStatusEffects` registers intent +only when pending/failed effects exist and their targets have passed validation, +before their delivered receipt and run effect metadata are written. Both use the +existing owning transactions; no caller wrapper or new persistence is needed. +Pure committed replay and an empty pending-effect set remain unsignalled. + +Why the effect receipt matters: `readNativeContinuationSource` requires the +applied decision, delivered same-agent continuation effect, original wake intent, +dispatched wake, and responsible-user continuity. A target run can already exist +while its missing delivered receipt prevents publication. A notification only at +run insertion cannot recover that later eligibility transition. + +Two new PostgreSQL regressions failed before these integrations: +`/tmp/sleep28-native-effects-before.log`. One observed no milestone intent in the +real decision transaction; the other committed a real effect-delivery receipt but +still had no successor notice. After the repair all 18 milestone worker cases +pass (`/tmp/sleep28-native-effects-tests.log`). The new cases exercise the real +committer and effect dispatcher, outer-commit settlement, committed replay, late +lineage admission and quiet repeated recovery. The continuation is initially +suppressed even though its run and dispatch already exist, and becomes publishable +only after the real receipt producer runs. Fixtures preserve real foreign keys. + +All nine selected native chat integration cases still pass after the committer +change (`/tmp/sleep28-native-effects-acceptance.log`); expected lock contention +remains retryable. Server production typecheck passes +(`/tmp/sleep28-native-effects-typecheck.log`). These are local checks, not full +workspace or PR acceptance, and do not include test-file typechecking. + +Next concrete audit is heartbeat queue admission, coalescing, cancellation, +claim/release and wake-receipt writers in `heartbeat/queue.ts`, then the other +recovery writers. Some run mutations still use the root DB directly; place any +new transaction at the narrow mutation owner rather than wrapping execution or +provider I/O. Avoid signalling no-op periodic inspections and lease renewals. +Remaining interaction, publication, conversation/configuration, question +claim/retry, and monitor/semantic-receipt dependencies still require coverage. +The new worker is still not production-registered, and the periodic combined +projector remains until that audit and its real producer tests are complete. + + +## Queue admission, claim, release and coalescence checkpoint + +The issue-bound admission transaction signals before inserting its run. Ordinary, +legacy-comment and queued-comment claims signal before changing run/wake state. +Queued-comment discard and committed chat-close cancellation branches also signal +before their terminal writes. The native review claim owner signals after its +review/ownership checks and before changing the run to running. No claim inspection +or admission-marker-only update gets a signal merely because it was inspected. + +Drain release now selects and locks the still-running candidate in its existing +transaction, obtains its scope, then registers intent before releasing the run, +wake and issue lock. A repeated release after the run is queued returns without +an intent or update. The added selection follows the same run-first locking order +as the previous UPDATE; it is not a periodic discovery query. + +The wake-queue PostgreSQL admission adapter owns coalesced run-context writes and +signals from that existing transaction. Its port carries company identity, so +this writer uses company-scoped reconstruction rather than adding feature fields +to the module contract or interpreting chat/provider payloads. The actual +conversation/lineage decision remains in the chat projection owner. + +The fallback company-run admission/coalescing branch of `heartbeat/queue.ts` is +outside the issue-bound path and cannot create a chat milestone. It intentionally +has no new notification or wrapper. Any later task-binding writer still needs +its own signal. This is separate from claiming a historically bound run, for +which a missing context issue ID conservatively uses company scope. + +Verification and failures: + +- Initial claim fixtures incorrectly inserted a run before its referenced wake. + Three failed on the real FK, while the drain-release missing-wake regression + failed at its publication assertion. `/tmp/sleep28-queue-claim-before.log`. + Fixing insertion and cleanup dependency order required no production change. +- With valid fixtures, all four claim/release regressions failed at the missing + notice assertion before wiring: `/tmp/sleep28-queue-claim-corrected-before.log`. + After repair all 22 then-current worker cases passed: + `/tmp/sleep28-queue-claim-tests.log`. +- Real issue admission during a held execution drain and the actual PostgreSQL + coalescing writer each reproduced a missing notice independently before their + notification was added: `/tmp/sleep28-queue-admission-before.log` and + `/tmp/sleep28-coalescing-before.log`. +- All 24 worker cases now pass: `/tmp/sleep28-queue-producers-tests.log`. Queue + tests call the actual admission/claim/release implementations and database; + dependency callbacks hold execution and prevent provider dispatch. The + coalescing case exercises the actual adapter inside an outer transaction. + These do not claim provider execution or a full native reviewer admission test. +- All 92 existing queue-module and wake-admission policy tests pass: + `/tmp/sleep28-queue-module-tests.log`. The queue module now accepts the standard + guarded external test URL through `PAPERCLIP_QUEUE_TEST_DATABASE_URL`; its + destructive per-case fixture reset ran only on the new isolated database + `paperclip_sleep28_queue_claim_20261025_test`. Embedded tests remain the default. +- Server production typecheck passes: `/tmp/sleep28-queue-producers-typecheck.log`. + This excludes test files and is not workspace/PR acceptance. + +Remaining queue producer audit is explicit: the general chat-control recovery +gate's terminal write, early case-handoff cancellation, stale retry/reassignment +cancellation, native status-wake dispatch receipts, deferred-run promotion in the +wake-queue adapter, queued-comment mutations and run-dispatch cancellation. The +native review claim signal also needs its dedicated real admission acceptance +case before final slice acceptance. Other recovery, interaction, presentation, +conversation/configuration, question claim/retry and monitor/semantic receipt +writers from the dependency table remain. Production still uses the periodic +combined projector; the new worker is not registered yet. No original inventory +or PR-stack acceptance requirement is removed by this checkpoint. + + +## Retry promotion and recovery admission checkpoint + +The run-dispatch PostgreSQL adapter now signals before due-retry promotion, +suppressed-retry cancellation, and stale-run cancellation. Its existing locked +state/gate checks still own authorization and no-op exits. Promotion passes the +already-resolved issue identity to its private writer; no application port or +provider contract changes. Missing issue identity uses company reconstruction. +Repeated attempts after settlement do not schedule another milestone sweep. + +The wake-queue transaction now signals before deferred promotion inserts its +run and before immediate recovery inserts its wake/run pair. The existing issue +lock, deferred claim, run linkage, and execution lock remain atomic. Tests use +these real adapter ports; external host dependencies return fixture values and +no provider execution is started. Notice creation occurs only after commit. + +Evidence: + +- `/tmp/sleep28-run-dispatch-before.log`: both real retry paths committed their + expected status but failed the missing-notice assertion. The first stale-run + fixture failed earlier because chat comments intentionally bypass terminal + task status; no production policy was changed to accommodate that fixture. +- `/tmp/sleep28-staleness-corrected-before.log`: a max-turn continuation correctly + cancelled but its terminal notice was absent before the signal was added. +- `/tmp/sleep28-wake-recovery-before.log`: both deferred and immediate admission + created their queued runs but failed the missing-notice assertions. +- `/tmp/sleep28-recovery-producers-tests.log`: all 166 selected cases pass, + comprising 29 milestone worker cases and existing run-dispatch/wake-queue + application and run-dispatch policy suites. Earlier 97-case verification + overlaps this set and is not an additional unique count. +- `/tmp/sleep28-recovery-producers-typecheck.log`: server production types pass; + test files are excluded from that command. `git diff --check` passes. + +Queued-comment changes, raw queue cancellation and wake-receipt writers, native +review admission acceptance, and remaining dependency producers are still open. +The periodic combined projector remains in production; this checkpoint does +not claim slice publication, full-suite/PR acceptance, or instance sleep. + + +## Native wake dispatch receipt checkpoint + +`dispatchPendingNativeStatusWakeups` now registers milestone intent when it +repairs an interrupted dispatch receipt, coalesces another same-scope intent, +or records the result of fresh admission. Each receipt UPDATE has a narrow +transaction; enqueue/provider execution remains outside it. Existing status, +claim, provenance, and update guards are unchanged. The parsed issue identity +already owned by this dispatcher scopes the notification (company fallback +when absent). Claims, retry timestamps, and rejected targets do not acquire +milestone intent just because they were inspected. + +The existing continuation reader requires both the delivered decision effect +and the coalesced original wake plus matching dispatch receipt. Consequently, +the run-admission notification alone is insufficient: it can be consumed while +the original intent is still claimed. The new normal-dispatch regression forces +that ordering using the dispatch suppression callback, then verifies the final +receipt reactivates projection without starting provider execution. + +Evidence and limitations: + +- `/tmp/sleep28-native-wake-receipt-before.log`: interrupted dispatch recovery + linked the run successfully but its queued notice remained absent. +- `/tmp/sleep28-native-wake-fresh-without-signal.log`: removing only the final + receipt notification reproduced a missing notice after the early admission + sweep. The production signal was restored in a finally block. +- `/tmp/sleep28-native-wake-restored-tests.log`: all 56 milestone/queue cases + pass (31 milestone cases, 25 existing queue cases), including normal and + recovered receipts, repeat recovery quiet, and drain-held dispatch. +- `/tmp/sleep28-native-wake-dependency-tests.log`: all 13 existing dependency + scheduling cases pass on a new isolated PostgreSQL database. The suite logs + responsible-user setup denial for some existing fixture paths; this is not + evidence of successful provider execution or full product acceptance. +- The dependency suite now accepts guarded `PAPERCLIP_DEPENDENCY_TEST_DATABASE_URL` + and removes its fixed fixture user/closes its client during teardown. Its + existing broad cleanup must run only on a dedicated database. Initial reuse + retained the old fixture user; a manual cleanup command used the wrong physical + table name, and the following run failed in setup with duplicate `user_pkey`: + `/tmp/sleep28-native-wake-dependency-final.log`. Corrected teardown removed the + fixture; a fresh read confirmed zero rows before the successful four native + cases in `/tmp/sleep28-native-wake-dependency-clean.log` (nine unselected). +- Server production typecheck passes: `/tmp/sleep28-native-wake-typecheck.log`. + Test files remain outside that typecheck. No whole-slice PR acceptance claimed. + +Raw queue cancellation, queued-comment/lineage changes, native reviewer claim +acceptance, and the remaining dependency producer audit still prevent retiring +the production periodic projector. All original stack scope remains required. + + +## Remaining queue cancellation writers + +The general chat-control recovery gate now registers intent before its guarded +terminal write, after the existing proof/no-op decisions. Invalid admission at +dispatch still fails closed and never calls the provider callback. Repeated +attempts against a settled run exit before signaling. + +Superseded case handoffs previously cancelled with a root-client UPDATE. That +single status change now owns a narrow transaction, locks/selects the still-queued +run, and registers company-scoped intent before updating it. The company scope +avoids trusting a superseded handoff's issue identity as the run's current chat +binding. Status/company guards and the existing post-update wake-status callback +remain unchanged. A repeated claim finds no queued row and creates no intent. +The added run-first lock does not enclose provider I/O or introduce an issue lock. + +Admission's stale scheduled-retry and reassigned execution-owner cancellations +also signal before their existing transaction writes. Those transactions may +later insert/notify a replacement run, but that later intent is insufficient for +the protocol's before-first-write requirement. Company/issue identity comes from +the owning admission state; no new module port or provider branch is added. + +Evidence: + +- `/tmp/sleep28-case-handoff-before.log` reproduces committed cancellation with + no terminal notice before the fix. +- `/tmp/sleep28-chat-control-before.log` reproduces the same gap for a failed + recovery gate; the repaired case-handoff regression passes in that run. +- Both gate tests verify no provider execution and quiet repeated attempts. +- The two real replacement-admission cases cover queued and scheduled-retry + predecessors, their distinct cancellation codes, the old run's terminal notice, + and the drain-held replacement. These are behavioral coverage, not claimed + missing-notice-before reproductions: later admission signaling can also wake + the same issue. Their early intent placement is established by code audit. +- `/tmp/sleep28-cancellation-final-tests.log`: all 60 milestone/queue cases pass, + including 35 milestone worker cases. Earlier 58-case and two-case runs overlap. +- `/tmp/sleep28-cancellation-final-typecheck.log`: server production types pass. + Test files are excluded. `git diff --check` passes. No PR acceptance claimed. + +The identified raw queue cancellation and native wake receipt writers are now +covered. Still required: queued-comment/lineage mutations, native reviewer claim +acceptance, publication and interaction changes, conversation/configuration, +question claim/retry, and monitor/semantic receipt dependencies. Production still +uses the combined periodic projector until those producers are complete. + + +## Queued-comment lineage and native reviewer claim coverage + +The queued-comment PostgreSQL adapter now signals before rewriting a run's +canonical comment IDs, deleting a queued comment, or cancelling its emptied +queued run. Its private transaction builder receives the already-locked issue +identity; exported application ports and caller wrappers are unchanged. Body-only +edits, issue timestamp touches, and wake-payload bookkeeping do not independently +signal: milestone binding reads comment identities/links and run lineage, not the +queued message text. The same transaction's run rewrite owns the wake for reorder +or nonempty discard; deletion owns the first intent for discard. + +The reconstruction regression deliberately starts with the saved queue and run +context out of sync. The real reorder use case reprojects the run context from +that authoritative queue. It is recovery/compatibility coverage, not a claim that +ordinary reordering introduces new comment identities. The discard regression +uses the actual use case and adapter inside a caller-owned transaction and proves +that intent settles only at outer commit. Existing adapter tests retain rollback +and company/issue authorization coverage. Reference/external-object callbacks in +these fixtures are no-ops; this does not claim their integration behavior. + +The previously outstanding native-review claim test now exercises the real +PostgreSQL claim against a seeded valid review assignment: applied decision, +matching projected status version, pending confirmation with a separate reviewer, +source native run, queued reviewer run and matching wake. It checks issue/run/wake +ownership and the working notice, then verifies repeated claim is quiet. This +covers claim/notification integration, not the entire review-creation workflow. + +Evidence: + +- `/tmp/sleep28-queued-comment-before.log`: reconstructed lineage failed to wake + its notice; discard emitted no transaction intent. Both failed before repair. +- `/tmp/sleep28-native-review-claim-without-signal.log`: removing the real claim + notification reproduced the missing working notice. The signal was restored + in finally; no production native-claim edit remains in this checkpoint. +- `/tmp/sleep28-lineage-final-tests.log`: all 74 selected cases pass across the + 38 milestone worker tests, queued-comment adapter/use cases, and native review + participant policy. Earlier 51-case and targeted native-claim runs overlap. +- `/tmp/sleep28-queued-comment-typecheck.log`: server production types pass; + test files are outside that command. `git diff --check` passes. +- The existing queued-comment adapter tests now accept guarded + `PAPERCLIP_QUEUED_COMMENT_TEST_DATABASE_URL` and close their client. Their broad + fixture cleanup ran only on freshly created/migrated + `paperclip_sleep28_queued_comment_20261025_test`; embedded mode remains default. + +The queued-comment producer and native reviewer claim gaps are closed for this +slice. Publication/interaction, conversation/configuration, question claim/retry, +and monitor/semantic receipt producers still need their own audit and proof. +The production periodic projector remains; no slice publication or full-goal +completion is claimed. + +## Question claim and retry checkpoint + +Question response claim state is publication lineage, not merely delivery +bookkeeping. A delivering answer can temporarily bind its source run to an +already published prompt on another conversation. Returning that claim to +pending restores the original chat lineage and can make a previously suppressed +working notice eligible. + +The question service now registers issue-scoped milestone intent before entering +that claim state and before an owned retry releases it. Retry release locks and +rechecks the existing attempt generation before signaling or updating; exhausted +claims retain ownership until the existing terminal receipt commits. Stale-claim +reset locks the actual stale rows, signals their distinct issue scopes, and resets +only those rows in the same transaction. Updates use batches of 500 IDs to avoid +an unbounded statement parameter count. This preserves the existing reset-all +semantics; it does not make the stale-row scan itself bounded. Live lease renewal, +still-owned exhausted claims and no-op claim attempts add no milestone wake. +Provider cancellation and execution remain outside these transactions. + +Verification: + +- `/tmp/sleep28-question-retry-corrected-before.log` reproduces a missing working + notice after real delivery retry. The callback captures observations for + assertions outside the service's error handler so assertions cannot be swallowed. +- `/tmp/sleep28-question-claims-before.log`: retry and stale reset fail before + the fix; a live claim remains unchanged and emits no notification. +- `/tmp/sleep28-question-final-tests.log`: 76 pass, one test expectation failed: + rollback emits settlement again when shared XID reconciliation confirms abort. + State and publication assertions passed. The corrected test checks intent, + eventual settlement, preserved claim, absent notice and subsequent idle quiet + without assuming settlement notifications are delivered exactly once. +- `/tmp/sleep28-question-final-fixed-tests.log`: all 77 cases pass: 42 milestone + worker cases plus 35 existing question-delivery cases, including lease renewal, + stale-worker fencing, retries and adapter behavior. +- `/tmp/sleep28-question-typecheck.log`: server production typecheck passed. + Test files are outside that compiler configuration. `git diff --check` passed. + +Existing question-delivery tests now support guarded +`PAPERCLIP_QUESTION_DELIVERY_TEST_DATABASE_URL`, retaining embedded mode by default +and closing their client. Their broad cleanup ran only on the dedicated migrated +`paperclip_sleep28_question_delivery_20261025_test` database. Milestone fixtures +used their separate dedicated database. + +Fresh origin/master inspected at `8c6bd5a83b`; reconciliation of the stack remains +outstanding. Publication/interaction, conversation/configuration and monitor +receipt producers remain under audit. Production registration and periodic-loop +removal are still pending, so this checkpoint is not a completed migration PR. + +## Monitor receipt ordering checkpoint + +The normal native monitor owner does not need another milestone signal when it +saves a schedule. `PaperclipRunnerToolAuthority.#setTaskMonitor` writes the issue +monitor and its semantic tool receipt inside the same `#withMutationReceipt` +transaction. The future deadline cannot be consumed before that commit. A later +monitor wake's real queue admission already signals the issue scope, and the +projector validates the committed receipt through `readNativeMonitorSource`. +Do not add a redundant signal to the generic semantic tool wrapper merely because +the projector reads a receipt. + +The new test executes the actual `set_task_monitor` tool authority, then admits +the due wake through the real heartbeat queue after the source has finished. It +checks the queued notice and idle quiet. Source completion is fixture-controlled +and provider execution is held by task drain; this is receipt/admission/lineage +coverage, not full monitor scheduler or harness acceptance. Alternate monitor +wake rewrites and recovery paths still require audit before the whole producer +family is declared complete. + +The initial proposed late-receipt test seeded an admitted wake before creating +its future monitor, which did not represent the normal ordering. Its first run +also failed the actual future-deadline validator after advancing the test clock +(`/tmp/sleep28-monitor-receipt-before.log`). That failure was a fixture problem, +not a production regression. The replacement follows the real ownership order +instead of manufacturing a missing signal for an unsupported ordering. + +- `/tmp/sleep28-monitor-admission-tests.log`: targeted real producer path passes. +- `/tmp/sleep28-monitor-final-tests.log`: all 43 milestone worker cases pass. +- No production monitor code changed; `git diff --check` passed. + +Remote gates refreshed at this checkpoint: #17490 at `31bcd1988823` still has +failed E2E shard 3 and aggregate checks; #17590 at `e501d6c99d72` still has queued +Paperclip Review; #17592 at `f53ae3d6d27c` still has neutral Paperclip Review. +These are not claimed ready and no merge was attempted. + +## Publication outcome and operator-resolution checkpoint + +`notifyChatPublicationWork` now accepts an optional company/issue scope. It +retains its existing delivery and receipt notifications. Callers supply that +scope when a publication outcome can change run lineage or release the run's +provider response slot. The shared scheduler still receives only a domain scope; +it does not inspect provider payloads or select presentation policy. + +The publication worker supplies the scope for successful receipts, cancellations +and terminal preparation failures. Its attempt claim, lease renewal and paused +attempt restoration do not signal milestone work. The failure finalizer supplies +it for failed/cancelled outcomes, while retry and delivery_unknown retain the +response slot. Operator mark-delivered/cancel resolution supplies the scope; +retry-anyway does not authorize presentation until its actual delivery settles. +All additions use existing transactions and execute before the relevant write. +Provider I/O remains outside the receipt transaction. No table was added. + +Real PostgreSQL regression evidence: + +- `/tmp/sleep28-publication-preparation-before.log`: initial fixture failed its + setup expectation because answering a question correctly cancels an unsent + prompt. This was not a production failure. The corrected fixture models a + claimed send still in flight when the user answers through the UI. +- `/tmp/sleep28-publication-failure-before.log`: definite delivery failure left + the eligible source failure notice asleep. Retry and unknown-outcome controls + passed and retained suppression. The new scoped signal fixes the failure. +- `/tmp/sleep28-prompt-receipt-tests.log`: a user answers during the controlled + provider send, the real question delivery saves a target receipt, and the + worker consumes that wake while the prompt is still streaming. Only the later + real publication receipt makes the target notice eligible. +- `/tmp/sleep28-prompt-receipt-without-signal.log`: removing only the new receipt + signal reproduces the missing target notice. The source was restored in + finally. Transport and credential authorization ports are controlled; actual + attempt leases, question receipts and publication commit use PostgreSQL. This + test does not establish provider authentication or live network behavior. +- `/tmp/sleep28-publication-resolution-before.log`: both cancel and + mark-delivered operator resolutions failed to wake their newly eligible notice. + The real resolution owner now signals before its state change. Cancel exposes + the source failure; marking delivered enables the question continuation. +- `/tmp/sleep28-publication-final-tests.log`: all 48 milestone and 35 error-policy + cases passed. The three arbitration cases did not run because their embedded + PostgreSQL setup failed after its own startup retries. That suite now supports + guarded `PAPERCLIP_ARBITRATION_TEST_DATABASE_URL`, closes its client, and uses + unique company prefixes. `/tmp/sleep28-arbitration-tests.log` shows all three + pass against dedicated `paperclip_sleep28_arbitration_20261025_test`. +- `/tmp/sleep28-publication-acceptance.log`: four existing chat integration cases + pass: malformed/transient preparation, parsed Discord question continuation, + expired provider-accepted receipt ownership, and ambiguous close confirmation. +- `/tmp/sleep28-publication-final-typecheck.log`: production server typecheck + passes. Test files are excluded from that configuration. Diff checks pass. + +This covers the primary publication worker/failure/resolution owners. Interaction +mutation and cancellation, provider-specific receipt writers, inbound/binding +changes and configuration producers remain to audit. In particular, a helper +called after an interaction mutation cannot replace intent before that earlier +mutation. Production registration and removal of the combined periodic owner +remain pending. Publish this slice as a draft for CI feedback, not as a completed +sleep migration or merge-ready PR. + +## Interaction resolution checkpoint + +Question answers, question cancellation, withdrawal, skipping and terminal-task +expiry now signal milestone work before their relevant mutations. Withdrawal, +skipping and expiry restrict the signal to ask_user_questions and +request_confirmation, the two kinds used by lifecycle presentation arbitration. +Existing actor checks, linked tool/secret resolution, pending-state CAS, caller +hooks and transaction ownership remain in place. No service wrapper or new table +was introduced. Dismissal alone does not change the arbitration predicate and +adds no notification. + +Historical answers are a distinct case: an ordinary deferred question on a done +task remains answerable, but deliberately enqueues no question delivery. That +path cannot rely on a later delivery claim to wake the milestone projector. +The actual answer transaction now owns its notification without reopening work. + +- `/tmp/sleep28-interaction-resolution-before.log`: answer, skip and historical + answer all leave an eligible failure notice asleep before the fix. +- `/tmp/sleep28-interaction-transitions-before.log`: six real producer variants + fail the missing-notice assertion: answer, skip, cancel, withdraw, terminal + expiry and historical answer. Cancel/withdraw also exposed a fixture cleanup + error because they create durable chat actions referencing the conversation. +- `/tmp/sleep28-interaction-transitions-tests.log`: notice assertions pass after + the fix; cancel/withdraw still fail the missing chat-action cleanup. Cleanup + now deletes this fixture's actions before its conversations. The old + `paperclip_sleep28_worker_final_20261025_test` database retains partial failed + fixtures and must not be reused for global worker quiet assertions. +- `/tmp/sleep28-interaction-final-tests.log`: all 160 cases pass across the full + milestone and existing interaction-service suites, on fresh separate databases + `paperclip_sleep28_interaction_worker_20261025_test` and + `paperclip_sleep28_interaction_service_20261025_test`. The former is the new + self-cleaning milestone test database; the latter is exclusive to the service + suite's broad cleanup. +- `/tmp/sleep28-interaction-typecheck.log`: production server typecheck passes; + test files are excluded. `git diff --check` passes. + +Draft PR #17606 targets #17592. Its first head `f398314ecd406d2e0eebbfb81a5d9dd7edc7c437` +started CI run 40198190088 and review run 40198190000. At the last inspection, +24 checks succeeded and 27 remained in progress, with no reported failures. +That is progress evidence, not final acceptance of this subsequent checkpoint. + +Remaining producer work includes confirmation accept/reject and invalidation, +interaction creation/source changes, provider-specific receipts, conversation +bindings, configuration and alternate recovery writers. Production registration, +removal of only the lifecycle polling lane, complete latest-head checks and final +stack/master reconciliation remain required. No merge or full-goal completion. + + +## Confirmation verdicts and comment invalidation + +The interaction service owns verdict and invalidation writes. Acceptance and +rejection now register milestone intent before the caller's optional in-transaction +resolution hook. Existing authorization, issue/proposal/interaction lock order, +workspace finalization checks and postcommit effects remain unchanged. Only +request_confirmation affects the current milestone response-slot policy; +checkbox confirmations do not acquire this additional intent. + +Live human-comment invalidation registers intent for question/confirmation rows +before changing their state. Historical-comment reconciliation registers one +issue-scoped intent per relevant bulk write. A historical checkbox-only group +remains outside this notification. No new wrapper, table or periodic retry was +introduced. All paths continue using the existing outer-transaction protocol. + +Evidence: + +- `/tmp/sleep28-confirmation-before.log`: real acceptance/rejection both fail to + produce the newly eligible failure notice; confirmation skip/withdraw/terminal + expiry already pass using the previous checkpoint's signals. +- `/tmp/sleep28-confirmation-invalidation-before.log`: six missing-notice + regressions fail: acceptance/rejection and question/confirmation supersession + by live/historical comments. The invalidation tests verify the actual service + expired their original interaction before asserting the missing notice. +- `/tmp/sleep28-confirmation-tests.log`: 169 cases pass, comprising all 63 worker + cases and 106 existing interaction-service cases, using the same separate + self-cleaning PostgreSQL databases as the previous checkpoint. +- `/tmp/sleep28-confirmation-typecheck.log`: production server typecheck passes. + It excludes test files. `git diff --check` passes. + +The latest fetched master is 3536b3649f98155d92b45cfb5dc3abcc2fea97d6; it has +not yet been reconciled into this stack. CI run 40198785531 for parent checkpoint +20f8c301f5 was still live with no reported failures at inspection. This does not +establish current-head final acceptance. + +Remaining interaction producers include document-target invalidation, deleted +addressees, startup supersession and creation/replacement. Provider receipt, +conversation/configuration and alternate recovery audits also remain. Production +worker registration and lifecycle polling removal are still intentionally pending. + + +## Creation and sibling replacement + +Creating a question or confirmation can expire a prior card from the same agent, +release the prior run's provider response slot, and cancel its unsent prompt. +The creation transaction now registers a single issue-scoped milestone intent +before inserting the new row. This precedes both sibling invalidation and prompt +publication changes. Existing idempotent reuse and validation return/throw before +this boundary; unrelated interaction kinds remain outside it. Provider policy +and sibling-supersession semantics are unchanged. + +The replacement regressions create two runs before worker startup, suppress the +older failed run with its pending card, let the worker become quiet, then create +the replacement through the actual service. They verify that the old card is +expired and its failure notice appears without another run transition or poll. + +- `/tmp/sleep28-interaction-replacement-before.log`: question and confirmation + replacement both fail the missing-notice assertion before the creation signal. +- `/tmp/sleep28-interaction-replacement-tests.log`: all 184 cases pass across + the 65 milestone worker, 106 interaction-service integration and 13 interaction + service unit cases. PostgreSQL suites use separate isolated databases. +- `/tmp/sleep28-interaction-replacement-typecheck.log`: production server + typecheck passes; test files are excluded. `git diff --check` passes. + +The remaining direct terminal-interaction writers are document-target expiry +(the resolution-time helper and explicit document-change method), deletion of +an addressed agent, and startup supersession. Document-change callers are in +`routes/issues.ts`; the owning state mutation is in the interaction service. +Audit those before removing the lifecycle poll. This checkpoint still does not +register the worker in production or establish final CI/review acceptance. + + +## Remaining terminal transitions in the interaction service + +The resolution-time stale-document helper, explicit document-target invalidation, +deleted-addressee cancellation and startup supersession now register milestone +intent before their first interaction write. The guards follow the existing +response-slot policy: question/confirmation for addressee cancellation, and +confirmation for document targets. No validity, permission, locking, publication, +or expiry behavior changes. Existing transactions own the notification. + +New regressions use actual document creation/update/deletion and the interaction +service's document-invalidation/accept/reject entry points. An early document +write is allowed to settle before interaction expiry, proving that the last +state transition still wakes projection. Deleted-addressee cases use another +agent, so the source run and its chat endpoint remain valid. Startup supersession +creates both pending cards before startup with ordinary sibling replacement +explicitly disabled; the actual repair selects and expires the older one. + +- `/tmp/sleep28-fixture-extraction-tests.log`: all 65 existing worker cases pass + after extracting only shared fixture creation/cleanup into + `__tests__/helpers/chat-milestone-fixtures.ts`. Worker, clock and quiet assertions + remain in the tests; cleanup still occurs after coordinator shutdown. The + helper tracks only fixture company/user IDs and now cleans fixture documents. +- `/tmp/sleep28-terminal-interactions-before.log`: all seven new cases reach + the correct terminal interaction state but fail to produce the eligible + failure notice: document change/deletion, stale accept/reject, two deleted- + addressee variants and startup supersession. There are no setup failures. +- `/tmp/sleep28-terminal-interactions-tests.log`: all 191 cases pass across + 72 worker, 106 service integration and 13 service unit cases. The two real-DB + suites use their separate existing isolated databases. +- `/tmp/sleep28-terminal-interactions-typecheck.log`: production server + typecheck passes; tests are excluded. `git diff --check` passes. + +The raw interaction-writer audit also found owners outside this service. These +must be reconciled with their outer transaction signals before removing polling: +`agent-conversations.ts`, `secret-proposals.ts`, `tool-action-review.ts`, +`tool-gateway.ts`, native automatic/obsolete review retirement, and native +external-review supersession. Result/summary-only writes, connection-intent-only +writes, and writes already covered by an outer status-decision/run transaction +need explicit disposition rather than indiscriminate new notifications. + +Provider receipts, conversation/configuration changes and alternate recovery +writers remain under audit. This checkpoint does not register the new worker +in production or remove lifecycle polling, and is not final PR acceptance. + + +## Governed interaction writers: proposals and tool reviews + +The proposal service bypasses the ordinary interaction resolver when reflecting +binding approval/rejection/withdrawal/expiry and rejection of dependent binding +proposals. Its existing transaction now registers scoped milestone intent only +when the linked interaction changes status. Repeated accepted-result metadata +updates remain quiet. Binding creation only inserts a new pending card, without +superseding another; that operation makes response eligibility more restrictive +and does not independently release an existing notice. + +Human tool-review verdicts similarly register intent in commitToolActionReview, +after current authority/state checks and before the first request mutation. +The gateway's separate projection transaction registers intent when a pending +interaction becomes expired/cancelled. Result-only tool-execution metadata is +not treated as a response-slot transition. Tool-request execution receipts are +also lineage inputs; their broader writer audit remains required separately. +No provider execution, signing policy or request ownership changed. + +`chat-run-milestone-governed-work.test.ts` is a separate cohesive suite using +`PAPERCLIP_MILESTONE_GOVERNED_TEST_DATABASE_URL`, currently the dedicated migrated +`paperclip_sleep28_governed_worker_20261025_test` database. It reuses only the +company/chat fixture helper; worker startup, clock, settlement and quiet assertions +remain explicit. Cleanup is company-scoped and runs after worker shutdown. + +- `/tmp/sleep28-governed-before.log`: all four ordinary binding transitions + fail to produce the newly eligible notice before the fix. +- `/tmp/sleep28-governed-transitions-before.log`: six cases fail, adding nested + rollback followed by successful rejection, and parent-proposal rejection that + cascades to a binding. The rollback itself preserves pending state and quiet. +- `/tmp/sleep28-governed-all-before.log`: retains the proposal failures and an + invalid tool fixture assertion: the real gateway correctly creates a prompt + publication, so an empty-publications expectation was wrong. The corrected + fixture represents a previously failed prompt delivery and asserts absence of + run notices rather than absence of all publications. A pending review still + owns that run's response until resolved. +- `/tmp/sleep28-tool-review-before.log`: all three corrected tool cases fail at + the missing-notice assertion after actual approval/rejection/expiry. The real + gateway creates and signs the request; a controlled remote port forbids any + provider calls. The persisted failed prompt is fixture state, not a provider + delivery claim. +- `/tmp/sleep28-governed-tests.log`: 200 cases pass across the 72 lifecycle + worker, 9 governed producer, 106 interaction integration and 13 interaction + unit cases on separate isolated PostgreSQL databases. +- `/tmp/sleep28-tool-review-acceptance.log`: all 109 existing tool-gateway cases + pass on newly migrated `paperclip_sleep28_tool_review_20261025_test`, selected + through its existing PAPERCLIP_TOOL_REVIEW_TEST_DATABASE_URL guard. +- `/tmp/sleep28-governed-typecheck.log`: production server typecheck passes; + it excludes tests. `git diff --check` passes. + +The native external-review supersession helper has one production caller in +status-decision-committer: its existing milestone intent precedes decision writes +and that helper in the same outer transaction. It needs no extra notification. +Native automatic/obsolete review retirement, conversation reset, authorization +prompt reactivation and tool-request execution lineage still need disposition, +alongside remaining publication/configuration/recovery producers. Production +registration, lifecycle polling removal and full-stack final checks remain pending. + + +## Native review retirement, conversation reset, and CI isolation + +Automatic fallback review retirement and obsolete policy review retirement now +register milestone intent inside their existing transactions, after finding and +locking the issue and before changing the interaction. Existing selection, +explicit-review protections, lock order, and terminal publication handling remain +with their owners. The real selectors are tested with retained prompt history, +no prompt history, and explicit requests that must remain pending. Current +system-authored cards are not newly externalized; the retained-prompt cases prove +recovery of stored history rather than claiming a new provider send path. + +Conversation reset registers intent before its existing issue/comment/question +writes. The regression exercises the actual reset service with a valid saved +conversation identity and chat binding. It establishes the service contract for +stored bindings, not a claim that the current UI creates combined Agent Chat and +external-chat conversations. Its failed prompt remains historical; expiring the +question releases response ownership and reactivates the failed-run notice. + +Evidence: +- `/tmp/sleep28-native-retirement-before.log`: both retained-prompt retirement + cases fail at the missing-notice assertion. Four control cases pass. The reset + case initially fails fixture setup because conversationState was missing. +- `/tmp/sleep28-conversation-reset-before.log`: after supplying the required + active conversation state, reset expires the question but fails to produce + the newly eligible notice before the signal is added. +- `/tmp/sleep28-native-retirement-tests.log`: all 16 governed producer cases pass. +- `/tmp/sleep28-retirement-combined-tests.log`: 101 tests pass (72 lifecycle, + 16 governed, 13 interaction unit). This command used the wrong integration + filename, so it did not run that suite; its count is not reported as 207. +- `/tmp/sleep28-retirement-interaction-service-tests.log`: the correctly selected + existing interaction integration suite separately passes all 106 cases. +- `/tmp/sleep28-native-retirement-typecheck.log`: server production types pass. + +CI at 3603b45c43 failed chat shard 1 in the newly added native review metadata +repair test: expected no signal, received an initial intent. The shared database +owner retains failed transactions until XID reconciliation; subscribing correctly +replays unresolved intent even when this test's operation is a no-op. A deliberate +unrelated rolled-back producer reproduces the same failure in +`/tmp/sleep28-ci-intent-before.log`. The repair test now uses a separate database +signal owner, preserving the intentional rollback and all commit/no-op assertions. +`/tmp/sleep28-ci-intent-tests.log` passes this test plus both current-authority +revocation cases (3 total). Production uncertainty handling is unchanged. Remote +acceptance still needs a fresh head; no blind CI retry was requested. + +Fresh origin/master is aec86c794b. No changes landed in the three service owners +above or chat-run-publications since the preceding 3536b3649f inspection. Full +stack reconciliation remains outstanding. The lifecycle projector remains +unregistered and its periodic production scan remains until the rest of the +publication, configuration, and recovery producer audit is complete. + + +## Interactive callback receipt recovery + +The shared interactive callback can recover a question/confirmation prompt from +`delivery_unknown` to `published` before attempting to resolve the interaction. +That receipt alone can authorize existing continuation lineage, even when the +later resolution does not write. It now supplies the issue scope to the existing +publication notification in that receipt transaction. A published row receiving +only its missing provider message ID keeps its existing delivery wake without +adding a milestone wake: prompt lineage reads publication state, not that ID. +No provider protocol, action authorization, or dispatch policy moved into wiring. + +The existing full Slack closed-choice test now observes the scoped settled signal +at its pre-resolution barrier. This prevents the later answer transaction from +masking a missing receipt wake. `/tmp/sleep28-callback-receipt-before.log` fails +there before the change; `/tmp/sleep28-callback-receipt-tests.log` passes all five +selected Slack/Discord/Telegram callback, modal, and continuation cases. The test +still verifies actual publication receipt, answer, continuation, deduplication, +unauthorized actions and acknowledged callback behavior. Provider transports are +controlled test ports, not live-provider qualification. Server production types +pass in `/tmp/sleep28-callback-receipt-typecheck.log`; diff whitespace checks pass. + +Additional producer dispositions inspected at this checkpoint: +- Discord and Telegram large-publication preparation explicitly reject cards and + interactionId. They preserve the root publication identity while splitting text + or attachments; they do not confirm an interactive prompt receipt. +- Slack file-receipt admission rejects cards/interactionId. Teams file-transfer + authority also rejects those plus progress/control/wake publications. Their + file delivery settlement is not an interactive prompt lineage producer. +- Authored-final suppression tests durable publication/comment identity, not the + transport state. File receipt outcomes do not remove that identity. +- Ordinary provider question/confirmation winner-action writes run in the same + resolution transaction as the already-covered interaction signal. Winner + actions restrict eligible destinations; they do not create a separate callback + polling obligation. Alternate recovery writers still require review. +- Tool-action continuation admission selects already-terminal requests joined to + existing delivery receipts before calling heartbeat.wakeup. Delivery's later + deliveredAt write is not read by milestone lineage. Normal admission therefore + observes the earlier durable request/receipt and supplies its existing wake. + Do not add an unconditional signal to every tool-request metadata write merely + because the lineage query joins that table. +- Gateway connection-authorization creation/reopening inserts a pending review + without superseding another or changing sourceRunId. It restricts eligibility; + its later ordinary resolution is covered by interaction-service signals. + +These dispositions do not finish the entire publication/configuration/recovery +audit. In particular, endpoint and conversation reactivation, policy changes and +alternate recovery receipts still need complete real-producer coverage before +production registration and removal of the lifecycle scan. + + +## Shared drain repair acceptance + +The voice-recovery review exposed dirty hints retaining a settled intent hold +while dispatch is paused. The repair is owned by #17576 and propagated through +#17578's shared scoped-deadline worker into this slice. The milestone first-write +and failed-selection drain tests now also assert zero active work and a stable +idle snapshot, then process the retained notice after resume without a new write. +All 99 lifecycle/governed/voice recovery cases pass in +`/tmp/sleep28-propagated-drain-tests.log`; server types pass. Both Teams scheduled +consumer groups pass 24 cases in `/tmp/sleep28-teams-drain-consumers.log`. + +Broader acceptance found two tests relying on the old held counter to wait for +resume: report and retention flush helpers return as soon as active work is zero, +which is now correct during drain even before the retained retry fires. The run +in `/tmp/sleep28-shared-drain-consumers.log` has 36 passes and those two failures. +Confirm the saved deadline runs on resume, strengthen idle assertions, and repair +these tests at their owning parent PRs before claiming propagated acceptance. + +The report and retention test corrections were verified at their owning parents +and propagated. /tmp/sleep28-shared-drain-final-tests.log now passes all 38 cases +in the previously failing combined selection. Together with the 99 milestone/ +voice-recovery and 24 Teams cases, every shared dispatcher consumer has selected +passing evidence; the full 108 voice integration cases passed at its parent. +This establishes the changed dispatch/drain boundary, not final PR readiness or +host sleep. The same pending predicate in three earlier independent workers is +still a required parent repair. Remaining milestone producers/wiring are unchanged. + +## Resource reactivation producers (2026-10-25) + +The current selector admits active/waiting conversations. Resource inventory and +provider lifecycle recovery can reopen an unavailable conversation after the +milestone worker exhausted its earlier scan. Both paths lacked an issue wake. +The real-owner regressions in /tmp/sleep28-resource-before.log fail after commit +with no failed-run publication, establishing the gap independently of manual +notification helpers. + +Resource reconciliation now owns one local availability transition function. It +locks the unavailable bindings it will reopen, signals their distinct issue +scopes before mutation, and retains the existing availability/completed-history +rules. Both inventory and lifecycle callers use it. Unchanged inventory produces +no milestone intent; status/health metadata is not indiscriminately promoted to +company-wide milestone scans. No shared scheduler contract or provider protocol +changes. The tests use the real credential lease and nested resource transaction, +prove savepoint release does not publish, prove outer rollback leaves the worker +quiet, and prove outer commit creates the expected publication. Completed history +stays completed. They then verify two minutes of quiet and no intent from a +repeated unchanged inventory. + +All 90 milestone lifecycle/governed/resource cases pass in +/tmp/sleep28-resource-combined-tests.log. That command also named a nonexistent +__tests__/chat-run-publications.test.ts path; it did not run projection units. +The separate corrected service-path run is recorded in +/tmp/sleep28-resource-projection-tests.log. Six existing provider acceptance cases +pass in /tmp/sleep28-resource-acceptance.log (1,424 deselected, not counted), covering +Slack membership/reinstall, Slack archive/unarchive, Teams/Telegram lifecycle, +GitHub resource identity preservation, durable lifecycle retry, and canonical +Chat SDK resource identifiers. Server typecheck passes in +/tmp/sleep28-resource-typecheck.log. The dedicated resource database is +paperclip_sleep28_resource_20261025_test. + +The earlier three configuration-owner drain repairs are now propagated. All 109 +health/Discord/receiver cases pass together at 9775b2bc4a in +/tmp/sleep28-configuration-drain-tests.log. An earlier combined run had 108 passes +and one schema mismatch because the parent test databases lacked the downstream +voice cleanup column; the retained log is +/tmp/sleep28-configuration-drain-schema-before.log. Applying the actual tip +migrations to those disposable databases fixes that setup error. + +This closes resource-owner reactivation, not every configuration producer. +GitHub-specific repository/installation restoration, other conversation rebinding, +late authority/lineage recovery, and remaining producer dispositions still need +review before production registration and retirement of the lifecycle scan. +Endpoint status/health are not direct milestone selector predicates; changing +them alone does not justify wiring a broad company wake to every health check. +The delivery owner retains its current authority checks. + +Final resource checkpoint: all 20 projection units pass at the corrected path in +/tmp/sleep28-resource-projection-tests.log, and server build passes in +/tmp/sleep28-resource-build.log. Combined selected evidence is 116 distinct +milestone/projection/provider cases, plus the 109 propagated configuration-owner +cases. Full remote checks/reviews remain required for the new head. + +## GitHub repository restoration follow-up + +GitHub's signed repository callback has a separate restoration path. Its real +owner regression fails in /tmp/sleep28-resource-github-before.log. The provider +module now receives a narrow restoreResourceConversations callback from the +existing resource owner, using the same transaction and issue notification. +Provider payload parsing stays in github/resources.ts. All three resource cases +pass in /tmp/sleep28-resource-github-tests.log; five GitHub integration cases +pass in /tmp/sleep28-resource-github-acceptance.log, including rename/transfer, +concurrent installation events, foreign-installation canonical inventory, +credential rotation, and paused endpoints. Server build/typecheck passes in +/tmp/sleep28-resource-github-build.log. Installation recovery already invokes +reconcileProviderResourceRows; its restoration uses the covered inventory path. + +Next concrete producer: telegram/resources.ts::migrateTelegramResourceConversations +sets moved conversations active before the generic availability transition runs, +so that transition cannot discover the earlier unavailable state. Preserve the +existing migration semantics and notify its affected issue scopes before the +migration write. Normal chat admission also writes links and activates the +conversation in chat-channels.ts; establish its wake ordering explicitly. The +Slack waiting-to-active helpers preserve selector eligibility (both states are +already admitted). Email uses explicit publication mode, which the lifecycle +selector and lineage resolver exclude. These dispositions narrow the remaining +configuration audit; they do not authorize removing the periodic scan yet. + + +## Telegram migration producer + +Telegram group migration reactivates conversations before the generic resource +availability helper runs. A real owner regression at b634353059 fails to publish +the existing failed run after commit (/tmp/sleep28-resource-telegram-before.log). +The Telegram module now locks the bindings it moves and registers distinct issue +milestone intents before its existing update. It preserves the caller's outer +transaction, current migration/authorization semantics, and canonical topic IDs. +No shared scheduler contract or new durable table is introduced. + +All four resource producer cases pass, including nested-savepoint/outer rollback, +commit reactivation, unrelated completed history, two minutes of quiet, unchanged +inventory and duplicate Telegram migration without a new milestone intent: +/tmp/sleep28-resource-telegram-tests.log. All 88 existing lifecycle/governed cases +pass in /tmp/sleep28-resource-telegram-lifecycle.log. Eight existing Telegram +migration integration cases pass in /tmp/sleep28-resource-telegram-acceptance.log +(1,422 deselected): unset/inherited/cleared/configured execution defaults, with +and without an existing destination, topic identity and subsequent reply routing. +Server production typecheck and build pass in the corresponding +/tmp/sleep28-resource-telegram-{typecheck,build}.log files. Total: 100 distinct +selected tests; no live Telegram or host-suspension qualification is claimed. + +The cancellation E2E diagnostic completed at b634353059: focused two cases pass, +the full Agent Chat projects file passes all 16, and the two cases with a +100ms database-only running-row delay pass. Evidence is retained in +/tmp/sleep-cancel-e2e-20261025, /tmp/sleep-cancel-full-e2e-20261025 and +/tmp/sleep-cancel-delayed-e2e-20261025. Captured lock waits do not establish the +CI deadlock cycle. These passing runs do not resolve the actual #17572/#17490 +E2E failures; no blind retry or cancellation-error masking was applied. + +Normal inbound conversation/link activation and remaining authority/lineage +producer dispositions still gate production registration and polling removal. +Fresh master is 7ebf98eafd; final base reconciliation remains outstanding. + + +## Ordinary inbound conversation activation + +The actual Slack ingress owner reproduces another missing wake: a prior failed +run has valid inbound lineage, but its unavailable conversation is restored by a +new message after the projector exhausted its startup work. The harness accepts +the new wake without creating a new run, so this does not rely on later queue +admission to hide the gap. Before repair the expected failed-run publication +never appears (/tmp/sleep28-inbound-before.log). + +persistTaskMutation now registers the existing issue scope before its mutations, +or the newly created issue scope before inserting its conversation. The same +transaction creates the inbound link and reactivates the binding. Existing +endpoint/resource/principal authorization runs before this owner; provider calls +remain outside it. This is a direct use of the shared transaction intent helper, +not a new wrapper or background scan. No business policy or public API changes. + +The regression passes (/tmp/sleep28-inbound-tests.log), and the combined selection +passes 12 cases: ordinary ingress reactivation, three existing Slack/Teams/Telegram +reach-revocation cases, and eight Telegram migration cases +(/tmp/sleep28-inbound-acceptance.log; 1,419 deselected). Server production +typecheck/build pass in /tmp/sleep28-inbound-{typecheck,build}.log. The isolated +DB is paperclip_sleep28_inbound_20261025_test. Existing resource and lifecycle +proofs remain recorded at their prior checkpoints; the whole integration file +was not run locally for this edit. + +Remaining audit: alternate monitor/wakeup receipt rewrites, board reply identity, +provider-specific configuration/authority, and any remaining direct lineage +writers. Endpoint update does not assign a new agent or change publication mode; +its unrelated health/configuration writes must not cause blanket milestone scans. +Production registration/poll removal, complete remote gates, prior parent findings, +and the full original stack remain unfinished. + + +## Interrupted inbound receipt settlement + +freshChatSourceAfterPublishedControl requires both the inbound action and its +delivery to be processed. A durable admitted run can exist before action +settlement, and operator recovery can repair a delivery while keeping its already +processed action/run. Earlier queue/admission signals cannot cover either later +commit. Two real Discord post-close recovery cases reproduce missing working +notices before these producer signals. + +The inbound wakeup owner now registers issue intent before successful action +settlement, and before repaired delivery acceptance when the action was already +processed. It uses the existing validated durable request/receipt and transaction; +no new validation layer, queue, scheduler or provider-specific branch is added. +Failed/issued action settlement does not create milestone work. Existing claim +fences, authorization, retry semantics and duplicate-admission protection remain. + +Evidence: +- /tmp/sleep28-inbound-receipt-before.log: action settlement becomes processed + without publishing the expected working notice (causal before-fix failure). +- /tmp/sleep28-inbound-receipt-tests.log: action recovery passes after repair. +- /tmp/sleep28-inbound-receipt-delivery-before.log: delivery repair misses the + publication. The action case also hit its existing NOWAIT contention retry; + its immediate processed assertion was invalid. The test now drives the owning + recovery path through its saved retry and waits for both durable receipts. +- /tmp/sleep28-inbound-receipts-tests.log: both cases pass with no second wake. +- /tmp/sleep28-inbound-receipts-acceptance.log: all 41 selected cases pass, + comprising the full 29-case fresh-after-close authority group and the previous + 12 ingress/revocation/Telegram migration cases (1,392 deselected). +- Production server typecheck/build pass in + /tmp/sleep28-inbound-receipts-{typecheck,build}.log. + +Further producer audit remains required before production wiring. In particular, +run-identity.ts::activate changes responsibleUserId, which the monitor and native +continuation lineage readers compare between source and successor. Audit actual +identity activation/steering ownership and test legitimate reactivation before +adding a signal. Native safe replacement and provider-capacity retry create +scheduled_retry rows, which are outside this projector's current status selector; +the already-covered promotion/claim paths own their later eligibility. Task retry +strategy-only wake changes do not alter lineage fields. Slack Board dispatch uses +source issue.comment and its own response path; do not infer milestone eligibility +from the shared binding reader alone. These are dispositions and next audits, +not proof that all remaining producers are complete. + + +## Run identity activation + +Monitor/continuation lineage requires matching responsibleUserId on source and +successor. Initialization and accepted steering can restore that match after the +worker exhausted its candidates. The identity owner now reads the bound issue +and registers its intent before the existing activation write. Pending steering +and replayed acceptance do not activate work. Production callers already own a +transaction; standalone tests now use reconcileSteeredIdentity, the existing +transaction/lock owner, instead of accepting directly on a raw client. + +Two real-PostgreSQL worker cases reproduced missing notices before the fix and +pass afterward: initialization and accepted steering, each with pending/quiet +state, outer rollback, outer commit, replay, and settled two-minute SQL quiet. +Evidence: /tmp/sleep28-identity-before.log and /tmp/sleep28-identity-tests.log. +All 43 existing identity/GitHub credential tests pass in +/tmp/sleep28-identity-acceptance-fixed.log. The first acceptance run found two +lock fixtures still using the embedded-only URL; those now use the configured +connection URL, retain their one-second lock timeout, and close the extra pool. +Server typecheck/build passed in /tmp/sleep28-identity-{typecheck,build}.log. + +Fresh master is 34409a92c4 (native failover backup filtering); it was fetched, +not merged into this draft. Recovery service orphan terminalization and adapter +failure classification are the next direct run writers to audit. Production +registration/poll removal and the original stack's gates remain unfinished. + + +## Orphan-run terminalization + +The stale-lock recovery owner can move a running run to interrupted/cancelled +without passing through normal run lifecycle writers. Its terminal update now +registers an issue-scoped milestone intent in a short transaction. The existing +conditional update still rechecks status, runtime, native ownership/finalization, +and legacy controller generation/lease. Process checks and telemetry remain +outside that transaction; cleanup and audit handling retain their prior order. + +Two real-producer cases (dead process and cancelled issue with a live process) +first exhausted the worker after its working notice, then recovered the run. +Both lacked the terminal notice before this change. Both now publish it once, +return to two-minute SQL quiet, and remain quiet after a repeated empty sweep. +Evidence: /tmp/sleep28-recovery-before.log and /tmp/sleep28-recovery-tests.log. +All 40 existing stale-lock recovery cases pass against a separate test database +(/tmp/sleep28-stale-lock-acceptance.log), including native ownership arrival, +controller/finalization races, shared references, explicit retry ownership and +failed audit insertion. Its test harness now supports the guarded external DB +path; no recovery assertions were weakened. Server typecheck/build and diff +whitespace checks pass (/tmp/sleep28-recovery-{typecheck,build}.log). + +Adapter-failure classification remains a separate producer audit: it changes +errorCode/resultJson/updatedAt after issue-monitor or escalation writes, so an +earlier issue notification cannot by itself prove coverage of the later run +commit. Production registration and removal of the lifecycle poll remain gated. + + +## Late adapter-failure classification + +The owning recovery path updates the issue monitor/escalation before persisting +its run classification. That later run write refreshes updatedAt and can bring +an old failed run into the projector's existing 24-hour window. The earlier +issue wake is insufficient: a real quota-recovery case established the monitor +and classification but produced no notice after the worker exhausted startup. + +Classification now registers the issue intent in its own short write transaction. +All four original-assignee/review-participant classification paths pass their +already-owned issue identity. The update is company scoped and preserves its +JSON merge so concurrent workspace receipts remain intact. Provider execution, +classification policy, monitor deadlines, escalation and existing recovery +ownership are unchanged. Repeated recovery with an existing quota monitor is +quiet and does not dispatch another run. + +Evidence: /tmp/sleep28-classification-before.log fails on the missing notice; +/tmp/sleep28-classification-tests.log passes all three recovery milestone cases. +All 89 existing recovery-action cases pass in +/tmp/sleep28-recovery-actions-acceptance.log. Their guarded external DB harness +uses paperclip_sleep28_recovery_actions_20261025_test, separately from worker +fixtures. Server typecheck/build pass in +/tmp/sleep28-classification-{typecheck,build}.log. The earlier 40 stale-lock and +43 identity/credential cases passed at their recorded checkpoints and were not +blindly repeated for this private classification write. + +Still required: finish the authority/configuration producer dispositions, +register the lifecycle worker, retain the separate native-progress duty, remove +only the lifecycle poll, and complete current-head CI/review. Fresh master +34409a92c4 is fetched but not yet reconciled into the stack. Older parent CI and +voice security/deletion findings plus every original remaining reason stay open. + + +## Restoring current publication authority + +Post-control lineage reads current connection enablement/status, resource reach, +endpoint reach/unlinked policy, linked human membership, and task visibility. +Restoring one of those facts can make a retained notice eligible without a run +transition. Nine real service-owner cases now cover that change after the worker +has exhausted startup: connection enablement, resource selection, unlinked reach, +identity confirmation, updateMember, updateMemberAndPermissions, ensureMembership, +setUserCompanyAccess, and task visibility. All keep the original authorization +reader and exact admitted source proof. No new run is dispatched to get a notice. + +Owner changes: +- The existing notifyChatEndpointWork contract now requires the already-known + company identity and also wakes milestones. Its configuration/status callers + pass that identity; ordinary health-only callbacks still use the narrower + notifyChatEndpointConfiguration/notifyChatConnectionConfiguration contracts. + This is a bounded wake on an actual configuration operation, not a health poll. +- Endpoint policy edits signal only newly enabled direct/group/unlinked reach. + Identity confirmation signals inside its existing advisory/row-lock transaction. +- Resource selection signals only a net disabled-to-enabled grant, covering both + individual and bulk GitHub selection through their common owner. Unchanged + inventory/defaults/selection does not create milestone work. +- Membership mutation owners signal human activation or viewer-to-active-role + restoration before writing. ensureMembership now owns its read/write transaction; + agent memberships and unchanged human state do not emit milestone intents. +- The issue update owner signals only hidden-to-visible restoration. Ordinary + chat-bound assignee changes are rejected by the existing chat_binding_agent_locked + guard, not treated as a supported restoration path. + +Evidence and corrected failures: +- /tmp/sleep28-authority-before.log: seven actual missed-wake failures, plus an + invalid Discord group-reach test. Discord channels use resource.enabled rather + than allowGroupChats; that fixture was already eligible before restoration. + Do not count that invalid failure as a production defect. +- /tmp/sleep28-authority-tests.log: the seven valid cases pass after repair; the + incorrect reach test still fails its precondition. It was replaced with actual + connection enablement, which fails before repair in + /tmp/sleep28-connection-authority-before.log. +- /tmp/sleep28-resource-selection-before.log: real resource grant restoration + misses its notice. Existing NOWAIT retry logs are retained; the assertion + failure is absence of the terminally eligible publication. +- /tmp/sleep28-authority-final-acceptance.log: all 50 selected integration cases + pass (the full post-close authority group plus ingress/revocation/Telegram + migration). Nine restoration cases include unchanged replay quietness where + supported; one-time link confirmation retains its existing consumed-token + behavior. Explicit connection updates retain their configuration wake. +- /tmp/sleep28-access-acceptance.log: all 18 access service regressions pass. +- /tmp/sleep28-authority-action-tests.log: all six action-worker cases pass. +- Initial typecheck caught a caller using connection.companyId that its query + did not select. It now uses the owning operation's companyId. Final server + typecheck/build pass in /tmp/sleep28-authority-final-{typecheck,build}.log. + +Alternate human-membership producers remain: middleware/auth.ts Cloud trusted- +header synchronization and board-claim.ts. Startup local-board insertion runs +before worker reconstruction; compatibility backfill only inserts agent members. +Cloud sign-in and board claim can restore humans while a worker is running and +must be covered before removing its poll. Credential rotation advances the +runtime fence; old sources must not regain authority from a new credential +fingerprint. Keep the existing generation/chronology denial tests. Remaining +producer dispositions, production wiring, full remote gates and original stack +scope are still required; this checkpoint is not a completed sleep migration. + + +## Cloud sign-in and board-claim authority restoration + +Cloud trusted-header membership synchronization now owns a short transaction for +its existing membership upsert. It locks the existing membership while comparing +its authority and registers a company-scoped milestone intent before a new, +reactivated, or formerly viewer membership becomes active. Profile-only syncs +remain silent even when the debounce is bypassed by a changed profile. The +trusted-header roles already exclude viewer; existing authentication, company +provisioning, stale instance-admin removal and default-grant behavior are retained. +This adds one indexed membership read and a transaction per actual debounced sync, +not an idle timer. Provider calls are not wrapped in a transaction. + +Board claim uses its existing transaction and signals only actual membership +insertion/reactivation. Existing active membership behavior is unchanged. Its +real database tests verify that nested savepoint completion emits intent, while +settlement waits for the caller's outer commit and the outside connection still +sees the old membership. This is producer/transaction proof; the Cloud integration +case additionally proves the real post-close milestone becomes publishable. + +Evidence retained: +- /tmp/sleep28-cloud-sync-before.log: restored Cloud authority missed its exact + working notice before repair, after the worker had exhausted startup discovery. +- /tmp/sleep28-board-claim-before.log: new and suspended membership cases failed + because no intent was emitted; unchanged active membership passed. +- /tmp/sleep28-cloud-authority-acceptance.log: all 39 post-close integration cases + pass, including Cloud restoration and a forced second profile sync with no + milestone signal. Existing NOWAIT admission retries remain visible in the log. +- /tmp/sleep28-cloud-owner-acceptance.log: all 30 Cloud provisioning, auth route, + transient retry and board-claim cases pass. Dedicated external test DB support + preserves the embedded fallback and closes the owning clients. +- /tmp/sleep28-cloud-authority-typecheck.log retained the initial redundant + viewer comparison failure. Cloud's validated role union excludes viewer; the + redundant condition was removed. Final typecheck and build pass in + /tmp/sleep28-cloud-authority-final-typecheck.log and + /tmp/sleep28-cloud-authority-build.log. Diff whitespace checks pass. + +The pre-push remote head 0f38d1079d fails E2E shard 3 and aggregate in run +40210763612, job 121221379705. The complete completed-job log is retained at +/tmp/sleep28-0f38-e2e3-failure.log. It again fails split-handoff cleanup at +agent-chat-projects.spec.ts:161 (HTTP 500), followed by the experiment-state +assertion at line 568. The log also records a process-identity event append +transaction deadlock at heartbeat.ts:3409 / run-lifecycle.ts:583. This is evidence +for the unresolved cancellation investigation, not proof of its entire lock cycle. +No blind rerun or masking was performed. Production milestone registration, +final producer dispositions, native progress as its separate slice, remote gates, +and every remaining original stack duty remain outstanding. + + +## Latest review and CI repair (2026-10-25) + +Review of `1b0bec0b25` found three defects. All were reproduced before repair: + +- GitHub `linkObservedIdentity` is a separate supported identity writer. It now + signals before creating/reactivating its identity link, inside the existing + membership and principal locks. Reconfirming an already-linked identity is + silent. The real producer test covers outer commit, rollback and reconfirmation. +- Orphan-run terminalization registered intent even when its guarded update + rejected retained native finalization. It now locks an eligible run before + intent and preserves the same predicate on the update. Five retained phases, + each swept three times, emit no milestone signals. All 40 stale-lock tests pass, + including coordinator-arrival races, terminal-issue authority and exhausted + finalization recovery. This does not migrate the stale-lock polling owner. +- The voice retention integration test still supplied the former coordinator + callback name. It now uses `projectNativeProgress`; the original failure was + `input.projectNativeProgress is not a function`. + +CI run `40212695778` also exposed an incomplete Cloud-auth fake and shared-user +fixture leakage. The fake now models transaction and locking calls, retaining +failure injection for optional membership/settings reads. The Cloud authority +integration test restores the shared user's name/email/verification metadata in +its finally block. Previously it left `Owner 1` behind, breaking later Slack +publication assertions that expected `Owner User`. No publication policy or +assertion was weakened to hide the changed name. + +Evidence: + +- `/tmp/sleep28-review-before.log`: five redundant-wake regressions and the + missing GitHub identity intent fail before repair. +- `/tmp/sleep28-voice-callback-before.log`: reproduces the renamed-callback error. +- `/tmp/sleep28-review-fixed.log`: all 40 stale-lock tests pass; one unscoped + signal assertion observed another fixture's notification. The two GitHub + transaction assertions now filter the actual company scope; both pass in + `/tmp/sleep28-review-github-scoped.log`. +- `/tmp/sleep28-voice-callback-fixed.log`: 28 failures on a reused database with + retained voice sessions/reports. Preserve this failed run. The full suite on + fresh migrated `paperclip_sleep28_review_voice_20261025_test` passes all 122 + cases (`/tmp/sleep28-review-voice-fresh.log`). Global reconstruction tests need + an isolated database; reusing a previous suite's retained obligations is invalid. +- `/tmp/sleep28-review-cloud-fakes-final.log`: 38 Cloud-auth/session tests pass. + The first repair exposed an obsolete one-read assertion; the updated assertion + still verifies the exact own-user/active filter on the authorization read. +- `/tmp/sleep28-review-long-board.log`: reproduces the `Owner 1` fixture leak. +- `/tmp/sleep28-review-chat-isolation.log`: 28 cases pass together on fresh + `paperclip_sleep28_review_chat_20261025_test`: all ten authority-restoration + cases, both GitHub signal cases, and all 16 long-publication cases. +- `/tmp/sleep28-review-typecheck.log` and `/tmp/sleep28-review-build.log`: server + production typecheck and build pass. Test files are not typechecked. + +Latest-head CI/review and stack reconciliation remain required. These local +passes do not discharge previous cancellation/deadlock evidence or parent review +findings. No merge or host-sleep qualification is claimed. + + +## Master integration: extracted GitHub credential fixture + +Master extracted the credential fixture into a helper shared with project Git +authorization. Its direct `acceptSteeredIdentity(db, ...)` call bypassed the +existing transaction owner. Combined verification reproduced four credential +failures and one project-authorization failure when activation registered work. +The helper now calls `reconcileSteeredIdentity`, which already owns the task/run +locks and transaction. No production transaction contract changed. + +The project suite initially skipped all 30 cases because embedded PostgreSQL was +unavailable. It now supports the existing validated external-test-database option +through `PAPERCLIP_PROJECT_GIT_TEST_DATABASE_URL` and closes that pool. Use a fresh +disposable database per run: the suite seeds fixed A/B user IDs. Reusing the red +run's database caused a duplicate-user setup failure; a new database resolved +that fixture setup issue without changing product behavior. All 25 credential +cases and all 30 project-authorization cases then passed on the owning branch. +The original failures remain in `/tmp/sleep-master-credentials-tests.log` and +`/tmp/sleep-master-project-git-external-before.log`; corrected results are in +`/tmp/sleep-master-identity-fixture-tests.log` (credentials) and +`/tmp/sleep-master-project-git-fixed.log` (project authorization). + + +### Slack setup identity producer correction (2026-10-27) + +The extracted Slack setup service's `linkSlackAccount` callback was missing the +milestone intent before its identity upsert. It now uses the existing company- +scoped notification after lease/membership/conflict checks and before the write. +An already linked account returns without a notification. No polling, new storage, +or caller wrapper is added; downstream milestone consumers retain their own scope +and current-authority checks. + +Two real setup regressions failed before the fix with no intent observed. A real +PostgreSQL table lock proves the intent exists while the insert is still blocked, +the link is invisible until commit, and settlement follows commit. A failed insert +rolls back identity without sending a test message; repeated successful linking +emits no new milestone work. The complete automatic Slack registration group +passes 115 cases (the other integration groups were not selected), and server +TypeScript checking passes. Combined stack and current-head CI remain separate +verification gates. + + +## Current-parent integration and acceptance (October 28) + +The existing milestone slice is prepared on Teams attempt recovery #17592 at +`24c54399909105984e2a10f5869064984f69c472`. Its previous published head is +`7991c2fa2ce7298bee69b25dbc6cb60a5e2fe250`, with original own base +`e634f655375853421a62692abf95fd859585a800`. This is a normal merge, with no +new scheduler, queue table, or worker duty. Final landed-parent reconciliation, +exact-head CI, and Greptile 5/5 remain separate publication and merge gates. + +Four explicit-base integration conflicts were resolved by preserving both owners: + +- Publication resolution and successful delivery retain milestone-scoped + notifications and the parent's exact voice conversation scope. +- Native finalization retains the server-owned first-task completion guard and + this slice's milestone notifications. +- The run-identity fixture retains current parent coverage for query parameters, + lock timeout options, and the additional authorization read scopes. Its three + steering acceptance calls use the existing reconciliation transaction owner. + The current parent's external PostgreSQL setup and client cleanup remain. + +All unrelated parent paths, including migration 0397, bounded consent recovery, +voice cleanup/retention fixture repairs, and runner checkpoint restoration, are +preserved. No migration regeneration or dependency/workflow change is introduced. + +Fresh isolated PostgreSQL fixtures pass **330 distinct selected cases**: + +- **305 cases across 14 complete suites:** all five milestone worker suites, + run identity, board claim, Cloud provisioning, auth session routes, Cloud actor + authentication, stale-lock recovery, GitHub credentials, project Git + authorization, and app admission wiring. +- **25 selected real chat integration cases** cover saved milestone discovery, + identity and authority producers, post-close receipt settlement, all ten + authority-restoration paths including Cloud synchronization, Slack account + linking and rollback, and changed native review metadata. The remaining + **1,590 chat cases were skipped**, not passed. +- Server production typechecking, build, and module-boundary checks pass. + Production typechecking does not statically check test files; the selected + tests above were executed. + +Commands and results were retained in +`/tmp/sleep28-current-parent-acceptance-command.json`, +`/tmp/sleep28-current-parent-acceptance.log`, +`/tmp/sleep28-current-parent-producers.log`, and the corresponding +`types`, `build`, and `boundaries` logs, each with a zero exit-status receipt. +Every database owner used its own newly created and app-migrated fixture. +The intentional cancellation and audit-write failures in the recovery tests +remain visible in the successful acceptance log. + +Earlier failure records and combined-stack counts above retain their original +revision scope. This acceptance is not a complete local repository run, a live +provider test, an external alarm qualification, physical host/database suspension, +or evidence of fleet savings. Native progress and subsequent sleep slices keep +their separate owners. diff --git a/server/src/__tests__/app-private-hostname-gate.test.ts b/server/src/__tests__/app-private-hostname-gate.test.ts index 2aabbe2f370..09c7ccc1abb 100644 --- a/server/src/__tests__/app-private-hostname-gate.test.ts +++ b/server/src/__tests__/app-private-hostname-gate.test.ts @@ -25,7 +25,7 @@ describe("createChatReconciliationCoordinator", () => { processFailedGitHubWebhookDeliveries, processPendingDeliveries, flushPublications, - projectRunMilestones: async () => 0, + projectNativeProgress: async () => 0, processPendingSlackFileUploadReceipts: async () => undefined, processPendingSlackSessionSyncs: async () => undefined, onError, @@ -142,14 +142,14 @@ describe("createChatReconciliationCoordinator", () => { }); it.each([false, true])( - "isolates a blocked milestone projector and joins its completion (rejects: %s)", + "isolates a blocked native progress projector and joins its completion (rejects: %s)", async (rejects) => { let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); - const failure = new Error("milestone projection failed"); - const projectRunMilestones = vi + const failure = new Error("native progress projection failed"); + const projectNativeProgress = vi .fn(async () => 0) .mockImplementationOnce(async () => { await held; @@ -159,7 +159,7 @@ describe("createChatReconciliationCoordinator", () => { const flushPublications = vi.fn(async () => undefined); const onError = vi.fn(); const coordinator = createChatReconciliationCoordinator({ - projectRunMilestones, + projectNativeProgress, processPendingDeliveries: async () => undefined, flushPublications, processPendingSlackFileUploadReceipts: async () => undefined, @@ -169,7 +169,7 @@ describe("createChatReconciliationCoordinator", () => { try { coordinator.reconcile(); await vi.waitFor(() => { - expect(projectRunMilestones).toHaveBeenCalledTimes(1); + expect(projectNativeProgress).toHaveBeenCalledTimes(1); expect(flushPublications).toHaveBeenCalledTimes(1); }); // A newly committed final/question must dispatch even while a different @@ -179,7 +179,7 @@ describe("createChatReconciliationCoordinator", () => { await vi.waitFor(() => expect(flushPublications).toHaveBeenCalledTimes(2), ); - expect(projectRunMilestones).toHaveBeenCalledTimes(1); + expect(projectNativeProgress).toHaveBeenCalledTimes(1); let drained = false; const draining = coordinator.drain().then(() => { drained = true; @@ -188,17 +188,17 @@ describe("createChatReconciliationCoordinator", () => { expect(drained).toBe(false); release(); await draining; - expect(projectRunMilestones).toHaveBeenCalledTimes(2); + expect(projectNativeProgress).toHaveBeenCalledTimes(2); expect(onError).toHaveBeenCalledTimes(rejects ? 1 : 0); if (rejects) - expect(onError).toHaveBeenCalledWith("run milestones", failure); + expect(onError).toHaveBeenCalledWith("native progress", failure); const completedCounts = [ - projectRunMilestones.mock.calls.length, + projectNativeProgress.mock.calls.length, flushPublications.mock.calls.length, ]; await new Promise((resolve) => setImmediate(resolve)); expect([ - projectRunMilestones.mock.calls.length, + projectNativeProgress.mock.calls.length, flushPublications.mock.calls.length, ]).toEqual(completedCounts); } finally { @@ -209,18 +209,18 @@ describe("createChatReconciliationCoordinator", () => { }, ); - it("joins a blocked milestone projector at shutdown without starting dirty follow-ups", async () => { + it("joins a blocked native progress projector at shutdown without starting dirty follow-ups", async () => { let release!: () => void; const held = new Promise((resolve) => { release = resolve; }); - const projectRunMilestones = vi.fn(async () => { + const projectNativeProgress = vi.fn(async () => { await held; return 1; }); const flushPublications = vi.fn(async () => undefined); const coordinator = createChatReconciliationCoordinator({ - projectRunMilestones, + projectNativeProgress, flushPublications, processPendingDeliveries: async () => undefined, processPendingSlackFileUploadReceipts: async () => undefined, @@ -230,7 +230,7 @@ describe("createChatReconciliationCoordinator", () => { try { coordinator.reconcile(); await vi.waitFor(() => { - expect(projectRunMilestones).toHaveBeenCalledOnce(); + expect(projectNativeProgress).toHaveBeenCalledOnce(); expect(flushPublications).toHaveBeenCalledOnce(); }); coordinator.notifyPublications(); @@ -243,7 +243,7 @@ describe("createChatReconciliationCoordinator", () => { expect(drained).toBe(false); release(); await draining; - expect(projectRunMilestones).toHaveBeenCalledOnce(); + expect(projectNativeProgress).toHaveBeenCalledOnce(); expect(flushPublications).toHaveBeenCalledOnce(); } finally { release(); @@ -269,7 +269,7 @@ describe("createChatReconciliationCoordinator", () => { ); const onError = vi.fn(); const coordinator = createChatReconciliationCoordinator({ - projectRunMilestones: async () => 0, + projectNativeProgress: async () => 0, processPendingDeliveries, flushPublications, processPendingSlackFileUploadReceipts, @@ -317,7 +317,7 @@ describe("createChatReconciliationCoordinator", () => { const processPendingSlackSessionSyncs = vi.fn(async () => undefined); const onError = vi.fn(); const coordinator = createChatReconciliationCoordinator({ - projectRunMilestones: async () => 0, + projectNativeProgress: async () => 0, processPendingDeliveries, flushPublications, processPendingSlackFileUploadReceipts, diff --git a/server/src/__tests__/auth-session-route.test.ts b/server/src/__tests__/auth-session-route.test.ts index e30c09896ac..1ab75bf346f 100644 --- a/server/src/__tests__/auth-session-route.test.ts +++ b/server/src/__tests__/auth-session-route.test.ts @@ -10,12 +10,16 @@ import { import { errorHandler } from "../middleware/error-handler.js"; import { assertCompanyAccess } from "../routes/authz.js"; +function queryResult(rows: T) { + return Object.assign(Promise.resolve(rows), { for: () => Promise.resolve(rows) }); +} + function createSelectChain(rows: unknown[]) { return { from() { return { where() { - return Promise.resolve(rows); + return queryResult(rows); }, }; }, @@ -102,6 +106,7 @@ describe("actorMiddleware authenticated session profile", () => { delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), select: vi.fn(() => createSelectChain([])), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -176,7 +181,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === companyMemberships ? [{ companyId: importedCompanyId, membershipRole: "member", status: "active" }] : [], @@ -186,6 +191,7 @@ describe("actorMiddleware authenticated session profile", () => { insert: vi.fn(() => insertChain), delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -252,7 +258,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === companies ? [{ name: "paperclip-stack-purple-rain" }] : [], @@ -280,7 +286,7 @@ describe("actorMiddleware authenticated session profile", () => { })), delete: vi.fn(() => ({ where: () => Promise.resolve(undefined) })), } as any; - db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { @@ -349,7 +355,7 @@ describe("actorMiddleware authenticated session profile", () => { select: vi.fn(() => ({ from: (table: unknown) => ({ where: () => - Promise.resolve( + queryResult( table === instanceUserRoles && state.staleInstanceAdminRow ? [{ id: "stale-role-row" }] : [], ), }), @@ -362,6 +368,7 @@ describe("actorMiddleware authenticated session profile", () => { }, })), } as any; + db.transaction = vi.fn(async (run: (tx: typeof db) => Promise) => run(db)); const app = express(); app.use( actorMiddleware(db, { diff --git a/server/src/__tests__/board-claim.test.ts b/server/src/__tests__/board-claim.test.ts index f408a8fbc08..b103cbb7d1a 100644 --- a/server/src/__tests__/board-claim.test.ts +++ b/server/src/__tests__/board-claim.test.ts @@ -1,3 +1,5 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -8,6 +10,7 @@ import { createDb, instanceUserRoles, principalPermissionGrants, + subscribeDatabaseWork, } from "@paperclipai/db"; import { claimBoardOwnership, @@ -20,7 +23,8 @@ import { startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_BOARD_CLAIM_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("board claim", () => { @@ -28,8 +32,8 @@ describeEmbeddedPostgres("board claim", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-board-claim-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-board-claim-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); afterEach(async () => { @@ -42,10 +46,11 @@ describeEmbeddedPostgres("board claim", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); - it("lets a signed-in user claim a local-board-only authenticated instance", async () => { + it.each(["new", "suspended", "active"] as const)("claims a local-board-only instance with %s membership and wakes only restored authority", async prior => { const now = new Date(); const userId = `claim-user-${randomUUID()}`; const company = await db @@ -70,6 +75,10 @@ describeEmbeddedPostgres("board claim", () => { role: "instance_admin", }); + if (prior !== "new") await db.insert(companyMemberships).values({ + companyId: company.id, principalType: "user", principalId: userId, + status: prior, membershipRole: "owner", + }); await initializeBoardClaimChallenge(db, { deploymentMode: "authenticated" }); const warningUrl = getBoardClaimWarningUrl("127.0.0.1", 3197); expect(warningUrl).toBeTruthy(); @@ -84,12 +93,19 @@ describeEmbeddedPostgres("board claim", () => { claimedByUserId: null, }); - await expect( - claimBoardOwnership(db, { token, code, userId }), - ).resolves.toEqual({ - status: "claimed", - claimedByUserId: userId, - }); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => events.push(event)); + try { + await db.transaction(async tx => { + await expect(claimBoardOwnership(tx as unknown as typeof db, { token, code, userId })) + .resolves.toEqual({ status: "claimed", claimedByUserId: userId }); + expect(events).toEqual(prior === "active" ? [] : ["intent"]); + // A nested claim must not publish a commit before its caller settles. + const outside = await db.select().from(companyMemberships).where(eq(companyMemberships.companyId, company.id)); + expect(outside.map(row => row.status)).toEqual(prior === "new" ? [] : [prior]); + }); + expect(events).toEqual(prior === "active" ? [] : ["intent", "settled"]); + } finally { unsubscribe(); } await expect( db diff --git a/server/src/__tests__/chat-action-work.test.ts b/server/src/__tests__/chat-action-work.test.ts index 067d6d54e1b..7576c93d4aa 100644 --- a/server/src/__tests__/chat-action-work.test.ts +++ b/server/src/__tests__/chat-action-work.test.ts @@ -47,7 +47,7 @@ describe("chat action scheduling", () => { const work = s.start(); await vi.advanceTimersByTimeAsync(1); expect(s.service.processTelegramSubscriptionDiscovery).toHaveBeenCalledTimes(1); - await s.owner.transaction(tx => notifyChatEndpointWork(tx, "connection-fixture")); + await s.owner.transaction(tx => notifyChatEndpointWork(tx, "connection-fixture", "company-fixture")); expect(s.service.scheduleTelegramSubscriptionDiscovery).toHaveBeenCalledWith("connection-fixture"); s.service.processTelegramSubscriptionDiscovery.mockRejectedValueOnce(new Error("discovery unavailable")); release(); @@ -87,7 +87,7 @@ describe("chat action scheduling", () => { for (const run of s.runs) expect(run).toHaveBeenCalledTimes(2); // Queue claims/settlements do not rediscover every configured endpoint. expect(s.service.processTelegramSubscriptionDiscovery).toHaveBeenCalledTimes(1); - await s.owner.transaction(tx => notifyChatEndpointWork(tx, "connection-fixture")); + await s.owner.transaction(tx => notifyChatEndpointWork(tx, "connection-fixture", "company-fixture")); await vi.advanceTimersByTimeAsync(1); for (const run of s.runs) expect(run).toHaveBeenCalledTimes(3); expect(s.service.processTelegramSubscriptionDiscovery).toHaveBeenCalledTimes(2); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 56fc1a628f0..8428c0cb192 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -1,3 +1,8 @@ +import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "../middleware/auth.js"; +import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { accessService } from "../services/access.js"; +import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; +import { restoreNativeChatReviewPresentationInTransaction } from "../services/native-runtime/native-chat-review-presentation.js"; import { notifyChatEndpointConfiguration, notifyChatCompanyConfiguration } from "../services/chat-channels/configuration-work-notifications.js"; import { deleteCompany } from "../services/company-deletion.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js"; @@ -23,6 +28,7 @@ import { chatSlackManagerGrants, chatSlackRegistrations, toolOauthStates, enviro import type { PaperclipCloudConnector, SealedConnectorCredentials } from "../services/paperclip-cloud-connector.js"; import { buildSlackAppManifest, SLACK_MANAGER_SCOPES } from "@paperclipai/shared"; import { registerChatActionWork, dueChatAction } from "../services/chat-action-work.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { registerChatDeliveryWork } from "../services/chat-delivery-work.js"; import { notifyChatPublicationWork, notifyChatActionWork, CHAT_ACTION_QUEUES } from "../services/chat-work-notifications.js"; @@ -202,6 +208,8 @@ import { } from "../services/chat-interaction-publications.js"; import { enqueueChatRunMilestones, + enqueueChatRunLifecycleMilestones, + enqueueSafeNativeChatProgress, githubRunReplyState, resolveChatRunPresentationAuthorizationReason, CHAT_RUN_PRESENTATION_AUTHORIZATION_REASON, @@ -1144,8 +1152,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const companyPrefixAttempts = 8; const freshCompanyPrefix = () => `C${randomUUID().replaceAll("-", "").slice(0, 7).toUpperCase()}`; - async function seedCompany(nextIssuePrefix = freshCompanyPrefix) { - const companyId = randomUUID(); + async function seedCompany(nextIssuePrefix = freshCompanyPrefix, companyId = randomUUID()) { const assignedAgentId = randomUUID(); const replacementAgentId = randomUUID(); let inserted = false; @@ -6029,6 +6036,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .where(eq(chatEndpointLeases.endpointId, f.endpoint.id)), ).toHaveLength(0); }); + it("wakes GitHub milestone identity restoration only after outer commit and stays quiet on reconfirmation", async () => { + const f = await reviewBotFixture(); + await f.service.revokeLink(f.endpoint.id, f.principal.id); + const verified = { githubUserId: "42", login: "octocat" }; + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, (event, scopes) => { + if ([...scopes ?? []].some(scope => JSON.parse(scope).companyId === f.companyId)) events.push(event); + }); + const link = () => db.select().from(chatIdentityLinks).where(and( + eq(chatIdentityLinks.endpointId, f.endpoint.id), eq(chatIdentityLinks.principalId, f.principal.id), + )).then(rows => rows[0]); + try { + await expect(db.transaction(async tx => { + await githubChatManagementService(tx as unknown as typeof db, f.providerFetch) + .linkObservedIdentity(f.endpoint.id, "owner-user", verified); + expect(events).toEqual(["intent"]); + expect((await link()).status).toBe("revoked"); + throw new Error("rollback identity restoration"); + })).rejects.toThrow("rollback identity restoration"); + expect((await link()).status).toBe("revoked"); + events.length = 0; + await db.transaction(async tx => { + await githubChatManagementService(tx as unknown as typeof db, f.providerFetch) + .linkObservedIdentity(f.endpoint.id, "owner-user", verified); + expect(events).toEqual(["intent"]); + expect((await link()).status).toBe("revoked"); + }); + expect(events).toEqual(["intent", "settled"]); + expect((await link()).status).toBe("linked"); + events.length = 0; + await f.management.linkObservedIdentity(f.endpoint.id, "owner-user", verified); + expect(events).toEqual([]); + } finally { unsubscribe(); } + }); + + it("wakes milestone authority only when GitHub guest settings change, after outer commit", async () => { + const f = await reviewBotFixture(); + const initial = await f.management.configuration(f.endpoint.id, "owner-user"); + const configuration = { ...initial.configuration, people: [{ + kind: "guest" as const, githubUserId: "84", login: "contributor", + sponsorUserId: "owner-user", permissionProfile: "restricted" as const, automaticReviews: false, + }] }; + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, (event, scopes) => { + if ([...scopes ?? []].some(scope => JSON.parse(scope).companyId === f.companyId)) events.push(event); + }); + try { + await db.transaction(async tx => { + await githubChatManagementService(tx as unknown as typeof db, f.providerFetch) + .saveConfiguration(f.endpoint.id, { expectedRevision: initial.revision, configuration }, "owner-user"); + expect(events).toEqual(["intent"]); + expect((await f.service.get(f.endpoint.id)).allowUnlinkedPeople).toBe(false); + }); + expect(events).toEqual(["intent", "settled"]); + expect((await f.service.get(f.endpoint.id)).allowUnlinkedPeople).toBe(true); + events.length = 0; + const current = await f.management.configuration(f.endpoint.id, "owner-user"); + await f.management.saveConfiguration(f.endpoint.id, { expectedRevision: current.revision, configuration }, "owner-user"); + expect(events).toEqual([]); + } finally { unsubscribe(); } + }); + it("scopes guest tasks and rechecks their sponsor before every bot tool", async () => { const f = await reviewBotFixture(); const initial = await f.management.configuration( @@ -8295,11 +8364,62 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(posts(f)).toHaveLength(1); expect(new URLSearchParams(String(posts(f)[0]![1]!.body)).get("channel")).toBe(f.installerId); }); - it("links the actor's own manager identity without requiring a connect command", async () => { + it("signals milestone eligibility around the real Slack account link and stays quiet on reconfirmation", async () => { const f = await managedFixture(); await f.provision(); await db.delete(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id)); - expect(await f.service.sendSlackTestMessage(f.endpoint.id, actor, { requestId: randomUUID() })).toMatchObject({ status: "sent" }); - expect(await db.select().from(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id))).toEqual([expect.objectContaining({ paperclipUserId: actor.userId, status: "linked" })]); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, (event, scopes) => { + if ([...scopes ?? []].some(scope => JSON.parse(scope).companyId === f.companyId)) events.push(event); + }); + let release!: () => void, acquired!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const locked = new Promise(resolve => { acquired = resolve; }); + const lock = db.transaction(async tx => { + await tx.execute(sql`lock table chat_identity_links in share mode`); + acquired(); await gate; + }); + await locked; + const linking = f.service.sendSlackTestMessage(f.endpoint.id, actor, { requestId: randomUUID() }); + void linking.catch(() => undefined); + try { + try { + await vi.waitFor(async () => { + const [waiting] = await db.execute<{ count: number }>(sql`select count(*)::int as count from pg_stat_activity + where datname = current_database() and state = 'active' and wait_event_type = 'Lock' + and query like 'insert into "chat_identity_links"%'`); + expect(waiting.count).toBeGreaterThan(0); + }); + expect(events).toEqual(["intent"]); + expect(await db.select().from(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id))).toEqual([]); + } finally { release(); await lock; await linking; } + expect(events).toEqual(["intent", "settled"]); + expect(await db.select().from(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id))) + .toEqual([expect.objectContaining({ paperclipUserId: actor.userId, status: "linked" })]); + events.length = 0; + expect(await f.service.sendSlackTestMessage(f.endpoint.id, actor, { requestId: randomUUID() })).toMatchObject({ status: "sent" }); + expect(events).toEqual([]); + } finally { unsubscribe(); } + }); + it("settles a failed Slack account-link intent without committing identity or sending a test message", async () => { + const f = await managedFixture(); await f.provision(); + await db.delete(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id)); + f.provider.mockClear(); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, (event, scopes) => { + if ([...scopes ?? []].some(scope => JSON.parse(scope).companyId === f.companyId)) events.push(event); + }); + await db.execute(sql`create function sleep_slack_link_failure() returns trigger language plpgsql as $$ begin raise exception 'link write unavailable'; end $$`); + await db.execute(sql`create trigger sleep_slack_link_failure before insert on chat_identity_links for each row execute function sleep_slack_link_failure()`); + try { + await expect(f.service.sendSlackTestMessage(f.endpoint.id, actor, { requestId: randomUUID() })).rejects.toThrow(); + expect(events).toEqual(["intent", "settled"]); + expect(await db.select().from(chatIdentityLinks).where(eq(chatIdentityLinks.endpointId, f.endpoint.id))).toEqual([]); + expect(posts(f)).toHaveLength(0); + } finally { + unsubscribe(); + await db.execute(sql`drop trigger sleep_slack_link_failure on chat_identity_links`); + await db.execute(sql`drop function sleep_slack_link_failure()`); + } }); it("requires a private Slack challenge for an unknown account before linking and sending the test", async () => { const f = await managedFixture(); await f.provision(); @@ -34933,7 +35053,44 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { ]); }); - it("drains bounded Slack chat-origin milestones without admitting an internal issue run", async () => { + it("wakes an existing chat milestone when ordinary ingress reactivates its conversation", async () => { + const fixture = await seedCompany(); + const { callbacks, endpoint, wakeup } = await configuredSlackEndpoint(fixture); + const thread = makeThread({ channelId: "C-REOPEN-MILESTONE", id: "slack:C-REOPEN-MILESTONE:4100.1", name: "reopen-milestone" }); + const send = (id: string) => deliverMessage({ callbacks, endpointId: endpoint.id, thread: thread.thread, + message: makeMessage({ id, text: "@maya continue this conversation", mentioned: true }), trigger: "mention" }); + await send("4100.1"); + const [conversation] = await readChatConversationsByEndpointId(endpoint.id); + const [link] = await db.select().from(chatMessageLinks).where(and( + eq(chatMessageLinks.conversationId, conversation.id), eq(chatMessageLinks.providerMessageId, "4100.1"))); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: fixture.companyId, agentId: fixture.assignedAgentId, + status: "failed", errorCode: "low_trust_isolation_unavailable", updatedAt: new Date(), + contextSnapshot: { issueId: conversation.issueId, source: "chat:slack", endpointId: endpoint.id, wakeCommentId: link.commentId }, + }); + await db.update(chatConversations).set({ state: "unavailable" }).where(eq(chatConversations.id, conversation.id)); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + const notice = () => db.select().from(chatPublications).where(and(eq(chatPublications.issueId, conversation.issueId), + eq(chatPublications.idempotencyKey, `run:${runId}:failed:${endpoint.id}`))); + try { + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + expect(await notice()).toEqual([]); + const wakesBefore = wakeup.mock.calls.length; + await send("4100.2"); + expect(wakeup.mock.calls.length).toBe(wakesBefore + 1); + // The harness accepts the new wake without creating a run. Existing + // projection debt must wake from ingress, not a later run transition. + await expect.poll(notice, { timeout: 5000 }).toEqual([ + expect.objectContaining({ idempotencyKey: `run:${runId}:failed:${endpoint.id}` }), + ]); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + expect(onError).not.toHaveBeenCalled(); + } finally { await coordinator.stop(); } + }); + + it.each(["milliseconds", "microseconds"])("drains bounded Slack chat-origin milestones without admitting an internal issue run (%s)", async (precision) => { const fixture = await seedCompany(); const { callbacks, endpoint } = await configuredSlackEndpoint(fixture); const thread = makeThread({ @@ -35028,15 +35185,32 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { })), ]); - const inserted: number[] = []; - for (let index = 0; index < 4; index += 1) { - inserted.push( - await enqueueChatRunMilestones(db, { - since: new Date("2026-09-05T14:00:00.000Z"), - limit: 1, - }), - ); + if (precision === "microseconds") { + await db.update(heartbeatRuns).set({ + updatedAt: sql`'2026-09-05T14:59:00.000123Z'::timestamptz`, + }).where(inArray(heartbeatRuns.id, resolverRejectedRuns.map(run => run.id))); } + // Fail a non-advancing pagination loop explicitly instead of leaving a + // background query chain running after the test timeout. + const select = db.select.bind(db); + let reads = 0; + const readGuard = vi.spyOn(db, "select").mockImplementation((...args) => { + if (args[0] && "runUpdatedAt" in args[0] && ++reads > 100) { + throw new Error("Milestone projection did not advance past rejected candidates"); + } + return select(...args); + }); + const inserted: number[] = []; + try { + for (let index = 0; index < 4; index += 1) { + inserted.push( + await enqueueChatRunLifecycleMilestones(db, { + since: new Date("2026-09-05T14:00:00.000Z"), + limit: 1, + }), + ); + } + } finally { readGuard.mockRestore(); } expect(inserted).toEqual([1, 1, 1, 0]); const publications = await db .select() @@ -41727,13 +41901,18 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } }); - it("publishes a closed-choice question, settles its Slack card, and delivers its exact continuation response", async () => { + it("publishes a closed-choice question, settles its Slack card, and delivers its exact continuation response", async ({ onTestFinished }) => { + let receiptWakeObserved = false; const fixture = await seedCompany(); const continuationRunId = randomUUID(); const resolveNativeQuestion = vi.fn(async () => "not_native" as const); const { callbacks, endpoint, runtime, service } = await configuredSlackEndpoint(fixture, { resolveNativeQuestion, + questionResolutionPersistBarrier: async () => { + // Receipt commit must wake projection before the separate answer write. + expect(receiptWakeObserved).toBe(true); + }, wakeup: async (agentId, options) => { if (options.contextSnapshot?.source !== "issue.interaction.respond") { return { accepted: true }; @@ -42056,6 +42235,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .set({ state: "delivery_unknown", providerMessageId: null }) .where(eq(chatPublications.id, publication.id)); + const unsubscribeReceipt = subscribeDatabaseWork(db, "chat-run-milestones", (event, hints) => { + if (event === "settled" && hints?.has(JSON.stringify({ companyId: fixture.companyId, issueId: conversation.issueId }))) { + receiptWakeObserved = true; + } + }); + onTestFinished(unsubscribeReceipt); const providerAcknowledgement = callbacks.onAction(actionEvent()); await expect( Promise.race([ @@ -51802,8 +51987,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const registeredCommandId = "888888888888888819"; let interactionSequence = 0n; - async function commandFixture() { - const fixture = await seedCompany(); + async function commandFixture(companyId?: string) { + const fixture = await seedCompany(undefined, companyId); const applicationId = uniqueDiscordApplicationId(); let holdWork = false; const queued: Array<() => void> = []; @@ -52897,8 +53082,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { : lastFreshDiscordMessageId + 1n; return lastFreshDiscordMessageId.toString(); }; - async function reopenedFixture(includeDuringConfirmationSource = false) { - const f = await commandFixture(); + async function reopenedFixture(includeDuringConfirmationSource = false, companyId?: string) { + const f = await commandFixture(companyId); const threadId = `discord:${guildId}:${channelId}:${rootMessageId}`; const admitMessage = async (message: ReturnType) => { await deliverMessage({ @@ -53046,6 +53231,142 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }; } + it.each(["action", "delivery"] as const)("wakes a post-close milestone when interrupted inbound acceptance settles its %s receipt", async receipt => { + const f = await reopenedFixture(); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + const notice = () => db.select().from(chatPublications).where(and( + eq(chatPublications.issueId, f.conversation.issueId), + eq(chatPublications.idempotencyKey, `run:${f.fresh.runId}:working:${f.endpoint.id}`), + )); + try { + // Crash boundary: the run/wake is durable, but its inbound action has + // not yet recorded settlement. Recovery must not dispatch it again. + if (receipt === "action") { + await db.update(chatActions).set({ status: "issued", result: {}, updatedAt: new Date() }) + .where(eq(chatActions.id, f.fresh.action.id)); + } else { + // Operator repair of a failed ledger keeps the already-admitted + // action/run. Only delivery acceptance needs to commit again. + await db.update(chatDeliveries).set({ state: "retry", processedAt: null, nextAttemptAt: null }) + .where(eq(chatDeliveries.id, f.delivery.id)); + } + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + expect(await notice()).toEqual([]); + const wakesBefore = f.wakeup.mock.calls.length; + await expect.poll(async () => { + await f.service.processPendingDeliveries(25, f.delivery.id); + const [action] = await db.select().from(chatActions).where(eq(chatActions.id, f.fresh.action.id)); + const [delivery] = await db.select().from(chatDeliveries).where(eq(chatDeliveries.id, f.delivery.id)); + return [action.status, delivery.state]; + }, { timeout: 10_000 }).toEqual(["processed", "processed"]); + expect(f.wakeup.mock.calls.length).toBe(wakesBefore); + await expect.poll(notice, { timeout: 5000 }).toEqual([ + expect.objectContaining({ idempotencyKey: `run:${f.fresh.runId}:working:${f.endpoint.id}` }), + ]); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + expect(onError).not.toHaveBeenCalled(); + } finally { await coordinator.stop(); await f.close(); } + }); + + it.each(["connection", "resource", "unlinked", "identity", "member", "member_permissions", "ensure_membership", "company_access", "visibility", "cloud_sync"] as const)( + "wakes a post-close milestone when %s authority is restored", async owner => { + const stackId = randomUUID(); + const f = await reopenedFixture(false, owner === "cloud_sync" ? cloudTenantPrimaryCompanyId(stackId) : undefined); + const previousToken = process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; + const [previousUser] = owner === "cloud_sync" + ? await db.select().from(authUsers).where(eq(authUsers.id, "owner-user")) + : []; + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + const notice = () => db.select().from(chatPublications).where(and( + eq(chatPublications.issueId, f.conversation.issueId), + eq(chatPublications.idempotencyKey, `run:${f.fresh.runId}:working:${f.endpoint.id}`), + )); + const access = accessService(db); + const membership = (await db.select().from(companyMemberships).where(and( + eq(companyMemberships.companyId, f.fixture.companyId), eq(companyMemberships.principalId, "owner-user"), + )))[0]; + try { + let restore: () => Promise; + if (owner === "connection") { + await toolAccessService(db).updateConnection(f.endpoint.connectionId, { enabled: false }); + restore = () => toolAccessService(db).updateConnection(f.endpoint.connectionId, { enabled: true }); + } else if (owner === "resource") { + const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.id, f.conversation.id)); + expect(conversation.resourceId).not.toBeNull(); + await f.service.replaceResources(f.endpoint.id, [{ id: conversation.resourceId!, enabled: false }], "owner-user"); + restore = () => f.service.replaceResources(f.endpoint.id, [{ id: conversation.resourceId!, enabled: true }], "owner-user"); + } else if (owner === "identity" || owner === "unlinked") { + await f.service.update(f.endpoint.id, { allowUnlinkedPeople: false }, "owner-user"); + await f.service.revokeLink(f.endpoint.id, f.fresh.action.principalId!); + if (owner === "unlinked") { + restore = () => f.service.update(f.endpoint.id, { allowUnlinkedPeople: true }, "owner-user"); + } else { + const intent = await f.service.createLinkIntent(f.endpoint.id, f.fresh.action.principalId!, 1800); + restore = () => f.service.confirmIdentityLink(new URL(intent.confirmationUrl).searchParams.get("token")!, "owner-user"); + } + } else if (owner === "visibility") { + await issueService(db).update(f.conversation.issueId, { hiddenAt: new Date() }); + restore = () => issueService(db).update(f.conversation.issueId, { hiddenAt: null }); + } else { + // Model existing denied authority without exercising revocation's + // unrelated credential cleanup; restoration uses its actual owner. + await db.update(companyMemberships).set({ status: "suspended" }).where(eq(companyMemberships.id, membership.id)); + if (owner === "member") restore = () => access.updateMember(f.fixture.companyId, membership.id, { status: "active" }); + else if (owner === "member_permissions") restore = () => access.updateMemberAndPermissions(f.fixture.companyId, membership.id, { status: "active", grants: [] }, "owner-user"); + else if (owner === "cloud_sync") { + process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "milestone-cloud-sync-test"; + let request = 0; + restore = () => resolveCloudTenantActor(db, cloudActorHeaderSourceFromHeaders({ + "x-paperclip-cloud-tenant-token": "milestone-cloud-sync-test", + "x-paperclip-cloud-user-id": "owner-user", + "x-paperclip-cloud-user-email": "owner@example.test", + // Change the profile to force an actual sync on replay. The + // debounce must not hide repeated unchanged membership wakes. + "x-paperclip-cloud-user-name": `Owner ${request++}`, + "x-paperclip-cloud-stack-id": stackId, + "x-paperclip-cloud-stack-role": "owner", + })); + } + else if (owner === "ensure_membership") restore = () => access.ensureMembership(f.fixture.companyId, "user", "owner-user", membership.membershipRole, "active"); + else { + const existing = await db.select().from(companyMemberships).where(and( + eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "owner-user"), + )); + restore = () => access.setUserCompanyAccess("owner-user", existing.map(row => row.companyId)); + } + } + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + expect(await notice()).toEqual([]); + await restore(); + await expect.poll(notice, { timeout: 5000 }).toEqual([ + expect.objectContaining({ idempotencyKey: `run:${f.fresh.runId}:working:${f.endpoint.id}` }), + ]); + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull()); + if (owner !== "identity" && owner !== "connection") { + const notices: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => notices.push(event)); + try { await restore(); expect(notices).toEqual([]); } + finally { unsubscribe(); } + expect(await notice()).toHaveLength(1); + } + expect(onError).not.toHaveBeenCalled(); + } finally { + if (previousToken === undefined) delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; + else process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = previousToken; + await coordinator.stop(); + if (previousUser) await db.update(authUsers).set({ + name: previousUser.name, email: previousUser.email, + emailVerified: previousUser.emailVerified, updatedAt: previousUser.updatedAt, + }).where(eq(authUsers.id, previousUser.id)); + await f.close(); + } + }, + ); + it("publishes working and the exact fresh final after pinned close without resurrecting the old source", async () => { const f = await reopenedFixture(); try { @@ -65325,6 +65646,36 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }; } + it.each(["separate owners", "combined budget"] as const)( + "keeps lifecycle and native progress projection independent with %s", + async (mode) => { + await enqueueChatRunMilestones(db, { since: new Date(0) }); + const context = await safeNativeProgressFixture("telegram", mode === "separate owners" ? "111" : "112"); + try { + const progressRun = await context.createRun("ongoing work"); + await context.addEvent(progressRun); + const failedRun = await context.createRun("failed work"); + await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, failedRun.runId)); + const input = { since: new Date(0), limit: 1 }; + await expect((mode === "separate owners" ? enqueueChatRunLifecycleMilestones : enqueueChatRunMilestones)(db, input)).resolves.toBe(1); + const rows = () => db.select({ key: chatPublications.idempotencyKey }).from(chatPublications) + .where(eq(chatPublications.conversationId, context.conversation.id)); + expect((await rows()).map(row => row.key)).toContain(`run:${failedRun.runId}:failed:${context.endpoint.id}`); + expect((await rows()).some(row => row.key.includes(":native:"))).toBe(false); + if (mode === "separate owners") { + await expect(enqueueChatRunLifecycleMilestones(db, input)).resolves.toBe(0); + } + await expect((mode === "separate owners" ? enqueueSafeNativeChatProgress : enqueueChatRunMilestones)(db, input)).resolves.toBe(1); + expect((await rows()).filter(row => row.key.includes(":native:"))).toEqual([ + { key: `run:${progressRun.runId}:working:${context.endpoint.id}:native:making_progress:1` }, + ]); + await expect(enqueueChatRunMilestones(db, input)).resolves.toBe(0); + } finally { + await retirePublicationFixture(context.service, context.endpoint.id); + } + }, + ); + it.each(["issue", "run"] as const)( "skips a locked native progress %s while unrelated final and question publications deliver", async (lockTarget) => { @@ -72379,6 +72730,48 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }; } + it("restores changed native review metadata once and signals only its committed change", async () => { + const context = await committedNativeReviewPublicationFixture("milestone-repair"); + // Earlier failure cases can leave intent awaiting XID reconciliation on the + // suite's shared owner. Subscribing to that owner immediately replays intent. + await expect(db.transaction(async tx => { + await signalDatabaseWork(tx, "chat-run-milestones"); + throw new Error("unrelated rolled-back producer"); + })).rejects.toThrow("unrelated rolled-back producer"); + const repairDb = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(repairDb, "chat-run-milestones", event => events.push(event)); + try { + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, context.runId)); + const input = { companyId: context.fixture.companyId, issueId: coordinator.issueId, runId: context.runId, + decisionId: coordinator.decisionId!, resultId: coordinator.resultId!, assessmentId: coordinator.assessmentId! }; + const savedAt = new Date(Date.now() - 60_000); + await db.update(heartbeatRuns).set({ updatedAt: savedAt }).where(eq(heartbeatRuns.id, context.runId)); + // Already-restored metadata still authorizes materialization by the caller, + // but must neither rewrite the run nor wake projection again. + await expect(repairDb.transaction(tx => restoreNativeChatReviewPresentationInTransaction(tx as unknown as typeof db, input))).resolves.toBe(true); + expect((await readHeartbeatRunsById(context.runId))[0].updatedAt).toEqual(savedAt); + expect(events).toEqual([]); + await db.update(heartbeatRuns).set({ resultJson: sql`${heartbeatRuns.resultJson} - 'externalChatReviewPresentation'` }) + .where(eq(heartbeatRuns.id, context.runId)); + await repairDb.transaction(async tx => { + expect(await restoreNativeChatReviewPresentationInTransaction(tx as unknown as typeof db, input)).toBe(true); + expect(events).toEqual(["intent"]); + }); + expect(events).toEqual(["intent", "settled"]); + expect((await readHeartbeatRunsById(context.runId))[0].resultJson).toMatchObject({ + externalChatReviewPresentation: { decisionId: coordinator.decisionId, runId: context.runId }, + }); + events.length = 0; + await repairDb.transaction(tx => restoreNativeChatReviewPresentationInTransaction(tx as unknown as typeof db, input)); + expect(events).toEqual([]); + } finally { + unsubscribe(); + await repairDb.$client.end({ timeout: 1 }); + await context.service.shutdown(); + } + }); + it.each(["principal_revoked", "gate_changed"] as const)( "blocks a committed native review response and selected files at transport after %s", async (kind) => { diff --git a/server/src/__tests__/chat-run-milestone-governed-work.test.ts b/server/src/__tests__/chat-run-milestone-governed-work.test.ts new file mode 100644 index 00000000000..22eb097d76b --- /dev/null +++ b/server/src/__tests__/chat-run-milestone-governed-work.test.ts @@ -0,0 +1,230 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + completionContracts, nativeRunResults, nativeRunFinalizations, workAssessments, statusDecisions, statusDecisionEffects, + agents, toolActionDeliveries, toolActionRequests, toolInvocations, toolGatewaySessions, toolCallEvents, toolAccessAuditEvents, toolPolicies, + companySecretBindings, companySecretProposals, companySecrets, createDb, + chatPublications, heartbeatRuns, issueComments, issueThreadInteractions, issues, +} from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; +import { createSecretProposalsService } from "../services/secret-proposals.js"; +import { commitToolActionReview } from "../services/tool-action-review.js"; +import { createToolGatewayService } from "../services/tool-gateway.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { prepareConversationTurn } from "../services/agent-conversations.js"; +import { dismissAutomaticCompletionReviews } from "../services/native-runtime/automatic-completion-reviews.js"; +import { dismissObsoleteNativePolicyReviews } from "../services/native-runtime/obsolete-policy-reviews.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MILESTONE_GOVERNED_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("chat milestones after governed interaction writes", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let fixtures: ReturnType; + let f: Awaited>; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-governed-milestones-"); + db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + fixtures = createChatMilestoneFixtures(db); + }, 60_000); + beforeEach(async () => { + errors.length = 0; f = await fixtures.create(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(companySecretProposals).where(eq(companySecretProposals.companyId, f.companyId)); + await db.delete(companySecretBindings).where(eq(companySecretBindings.companyId, f.companyId)); + await db.delete(companySecrets).where(eq(companySecrets.companyId, f.companyId)); + await db.delete(toolActionDeliveries).where(eq(toolActionDeliveries.companyId, f.companyId)); + await db.delete(toolGatewaySessions).where(eq(toolGatewaySessions.companyId, f.companyId)); + await db.delete(toolCallEvents).where(eq(toolCallEvents.companyId, f.companyId)); + await db.delete(toolAccessAuditEvents).where(eq(toolAccessAuditEvents.companyId, f.companyId)); + await db.delete(toolActionRequests).where(eq(toolActionRequests.companyId, f.companyId)); + await db.delete(toolInvocations).where(eq(toolInvocations.companyId, f.companyId)); + await db.delete(toolPolicies).where(eq(toolPolicies.companyId, f.companyId)); + await fixtures.cleanup(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await flush(); + } + async function flush() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + } + async function quiet() { + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + try { await vi.advanceTimersByTimeAsync(120_000); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); } + finally { reads.mockRestore(); writes.mockRestore(); } + } + const publications = () => db.select().from(chatPublications).where(eq(chatPublications.issueId, f.issueId)); + + async function run(status: "failed" | "running" = "failed") { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + responsibleUserId: f.userId, status, errorCode: "test_run_failed", updatedAt: new Date(), + contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId }, + }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + return runId; + } + + it.each(["approved", "rejected", "withdrawn", "expired", "rolled back", "parent rejected"] as const)("reactivates after a binding proposal is %s", async status => { + const runId = await run(); + // Binding review needs secret metadata, not the provider value. No secret is resolved. + const [secret] = await db.insert(companySecrets).values({ companyId: f.companyId, name: "test/binding", key: "BINDING_TEST" }).returning(); + const parent = status === "parent rejected" ? (await db.insert(companySecretProposals).values({ + companyId: f.companyId, kind: "secret", proposedName: "test/parent", proposedKey: "PARENT_TEST", + justification: "Pending parent secret", proposedByAgentId: f.agentId, originRunId: runId, + originIssueId: f.issueId, expiresAt: new Date(Date.now() + 86_400_000), + }).returning())[0] : null; + const proposals = createSecretProposalsService(db); + const proposal = await proposals.createBinding({ companyId: f.companyId, heartbeatRunId: runId }, { + ...(parent ? { secretProposalId: parent.id } : { secretId: secret.id }), configPath: "env.BINDING_TEST", justification: "Use the existing company credential", bindingTargetPolicy: "self_and_reports", + }); + expect(proposal.interactionId).toBeTruthy(); + await start(); expect(await publications()).toEqual([]); await quiet(); + if (status === "rolled back") { + await expect(db.transaction(async tx => { + await createSecretProposalsService(tx as unknown as typeof db).transition(f.companyId, proposal.id, "rejected"); + throw new Error("rollback proposal"); + })).rejects.toThrow("rollback proposal"); + expect((await proposals.getById(f.companyId, proposal.id))?.status).toBe("pending"); + await flush(); expect(await publications()).toEqual([]); await quiet(); + await proposals.transition(f.companyId, proposal.id, "rejected"); + } else if (parent) { + await proposals.transition(f.companyId, parent.id, "rejected"); + expect((await proposals.getById(f.companyId, proposal.id))?.status).toBe("rejected"); + } else if (status === "approved") { + await proposals.approve(f.companyId, proposal.id, { resolvedByUserId: f.userId }); + } else if (status === "expired") { + expect(await proposals.sweepExpired(new Date(proposal.expiresAt.getTime() + 1))).toBe(1); + } else if (status === "rejected" || status === "withdrawn") { + await proposals.transition(f.companyId, proposal.id, status, { resolvedByUserId: f.userId, proposerAgentId: f.agentId }); + } + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, proposal.interactionId!)))[0].status) + .toBe(status === "approved" ? "accepted" : status === "withdrawn" ? "cancelled" : status === "rolled back" || status === "parent rejected" ? "rejected" : status); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${runId}:failed:${f.endpointId}`); + await quiet(); + }); + it.each(["approved", "rejected", "expired"] as const)("reactivates after a tool review is %s", async status => { + await db.update(agents).set({ adapterType: "process" }).where(eq(agents.id, f.agentId)); + const runId = await run("running"); + await db.insert(toolPolicies).values({ companyId: f.companyId, name: "Ask first", policyType: "require_approval", + selectors: { toolName: "mcp-remote-fixture:update_note" } }); + const remote = vi.fn(async () => { throw new Error("Review must not invoke the provider"); }); + const gateway = createToolGatewayService(db, { toolActionSigningSecret: "milestone-review-fixture", remoteHttpRequest: remote }); + const session = await gateway.createSession({ companyId: f.companyId, agentId: f.agentId, runId }); + await expect(gateway.executeTool({ sessionToken: session.token, tool: "mcp-remote-fixture:update_note", parameters: { noteId: "test" } })) + .rejects.toMatchObject({ reasonCode: "approval_required" }); + const [request] = await db.select().from(toolActionRequests).where(eq(toolActionRequests.companyId, f.companyId)); + expect(request.interactionId).toBeTruthy(); + // The prompt could not be delivered, and the run failed while awaiting review. + // The pending review still owns its response until the board resolves it. + await db.update(chatPublications).set({ state: "failed" }).where(eq(chatPublications.issueId, f.issueId)); + await db.update(heartbeatRuns).set({ status: "failed", updatedAt: new Date() }).where(eq(heartbeatRuns.id, runId)); + await start(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${runId}:`))).toBe(false); + await quiet(); + if (status === "expired") { + await db.update(toolActionRequests).set({ expiresAt: new Date(Date.now() - 1) }).where(eq(toolActionRequests.id, request.id)); + await gateway.sweepExpiredActionReviews(); + } else { + await commitToolActionReview(db, { companyId: f.companyId, actionRequestId: request.id, decision: status, actor: { userId: f.userId } }); + } + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, request.interactionId!)))[0].status) + .toBe(status === "approved" ? "accepted" : status); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${runId}:failed:${f.endpointId}`); + await quiet(); expect(remote).not.toHaveBeenCalled(); + }); + + it.each([ + ["automatic", "prompt"], ["automatic", "no prompt"], ["automatic", "explicit request"], + ["obsolete", "prompt"], ["obsolete", "no prompt"], ["obsolete", "explicit request"], + ] as const)("handles %s review retirement with %s", async (kind, history) => { + const runId = await run(), contractId = randomUUID(), resultId = randomUUID(), assessmentId = randomUUID(), decisionId = randomUUID(); + const prompt = kind === "automatic" + ? "Review the persisted native-run evidence and confirm whether this issue may be completed." + : "Review the superseding native policy assessment."; + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1", risk: kind === "automatic" ? "low" : "standard", + completionAuthority: kind === "automatic" ? "agent_claim_policy" : "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { objective: "Respond to chat" }, canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native", nativeIssueId: f.issueId, completionContractId: contractId, + resultJson: { presentationDecision: { source: "none" } } }).where(eq(heartbeatRuns.id, runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId, + completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId, + resultJson: { terminal: { runTerminalState: "succeeded" } } }); + await db.insert(nativeRunFinalizations).values({ runId, companyId: f.companyId, issueId: f.issueId, phase: "committed", resultId }); + await db.insert(workAssessments).values({ id: assessmentId, companyId: f.companyId, issueId: f.issueId, runId, + contractId, resultId, triggerKind: "run_result", triggerActorCompanyId: f.companyId, priorIssueStatus: "in_progress", + priorStatusVersion: 0, policyVersion: "phase6-v1", assessmentJson: { attentionRequests: [] }, inputDigest: assessmentId }); + await db.insert(statusDecisions).values({ id: decisionId, companyId: f.companyId, issueId: f.issueId, runId, assessmentId, + decisionVersion: 1, policyVersion: "phase6-v1", fromStatus: "in_progress", toStatus: "in_review", applicationState: "applied", + reasonCode: kind === "automatic" ? "completion_claim_incomplete" : "completion_review_required", decisionDigest: decisionId, + decisionJson: { projectedStatusVersion: 1, effects: [{ kind: "bind_reviewer", prompt, ownerUserId: null }, + ...(kind === "obsolete" ? [{ kind: "append_superseding_assessment" }] : [])] } }); + await db.update(issues).set({ status: "in_review", statusVersion: 1, lastStatusDecisionId: decisionId }).where(eq(issues.id, f.issueId)); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, + kind: "request_confirmation", sourceRunId: runId, idempotencyKey: `native-review:${decisionId}`, status: "pending", + payload: { version: 1, prompt: history === "explicit request" ? "Review the release before publishing." : prompt, + target: { type: "custom", key: "native_completion_review", revisionId: decisionId } } }).returning(); + await db.insert(statusDecisionEffects).values({ companyId: f.companyId, issueId: f.issueId, decisionId, + ordinal: 0, effectKind: "bind_reviewer", targetType: "issue_thread_interaction", targetId: interaction.id, + idempotencyKey: decisionId, payload: {}, deliveryState: "delivered" }); + // System cards are no longer externalized, but stored prompt rows still own + // the response slot. Exercise their supported retirement rather than insert + // a synthetic new provider send through today's author-validation boundary. + if (history !== "no prompt") await db.insert(chatPublications).values({ companyId: f.companyId, issueId: f.issueId, + endpointId: f.endpointId, conversationId: f.conversationId, state: "pending", payload: { text: prompt, interactionId: interaction.id }, + idempotencyKey: `interaction:${interaction.id}:${f.endpointId}` }); + const notices = async () => (await publications()).filter(row => row.idempotencyKey === `run:${runId}:completed:${f.endpointId}`); + const retire = () => kind === "automatic" ? dismissAutomaticCompletionReviews(db, f.issueId) : dismissObsoleteNativePolicyReviews(db, [runId]); + await start(); expect(await notices()).toHaveLength(history === "no prompt" ? 1 : 0); await quiet(); + await retire(); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interaction.id)))[0].status) + .toBe(history === "explicit request" ? "pending" : "cancelled"); + await flush(); expect(await notices()).toHaveLength(history === "explicit request" ? 0 : 1); await quiet(); + await retire(); await flush(); expect(await notices()).toHaveLength(history === "explicit request" ? 0 : 1); await quiet(); + }); + + it("rechecks a stored chat binding when a conversation reset expires its question", async () => { + const runId = await run(); + await db.update(issues).set({ conversationAgentId: f.agentId, conversationUserId: f.userId, conversationState: "active" }).where(eq(issues.id, f.issueId)); + const interaction = await issueThreadInteractionService(db).create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", sourceRunId: runId, continuationPolicy: "wake_assignee", + payload: { version: 1, title: "Choose", questions: [{ id: "choice", prompt: "Continue?", selectionMode: "single", + options: [{ id: "yes", label: "Yes" }], required: true }] }, + }, { agentId: f.agentId, runId }); + // Failed delivery leaves the pending question itself as the response owner. + await db.update(chatPublications).set({ state: "failed" }).where(eq(chatPublications.issueId, f.issueId)); + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: f.userId, body: "/new" }).returning(); + const [resetRun] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, + status: "running", contextSnapshot: { issueId: f.issueId, source: "issue.comment", wakeCommentId: comment.id } }).returning(); + await start(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${runId}:`))).toBe(false); await quiet(); + expect(await prepareConversationTurn(db, resetRun)).toMatchObject({ reset: true }); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interaction.id)))[0].status).toBe("expired"); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${runId}:failed:${f.endpointId}`); + await quiet(); + }); + +}); diff --git a/server/src/__tests__/chat-run-milestone-identity-work.test.ts b/server/src/__tests__/chat-run-milestone-identity-work.test.ts new file mode 100644 index 00000000000..0afb3fb63a7 --- /dev/null +++ b/server/src/__tests__/chat-run-milestone-identity-work.test.ts @@ -0,0 +1,204 @@ +import { randomUUID } from "node:crypto"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agentWakeupRequests, chatActions, chatExternalPrincipals, chatIdentityLinks, chatMessageLinks, chatPublications, companyMemberships, createDb, heartbeatRuns, issueComments, issues, subscribeDatabaseWork } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; +import { initializeRunIdentity, reserveSteeredIdentity, acceptSteeredIdentity } from "../services/run-identity.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; +import { accessService } from "../services/access.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MILESTONE_IDENTITY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("chat milestones after identity activation", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let fixtures: ReturnType; + let f: Awaited>; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-identity-milestones-"); + db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + fixtures = createChatMilestoneFixtures(db); + }, 60_000); + beforeEach(async () => { + errors.length = 0; f = await fixtures.create(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await fixtures.cleanup(); expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function flush() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + } + async function quiet() { + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + try { await vi.advanceTimersByTimeAsync(120_000); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); } + finally { reads.mockRestore(); writes.mockRestore(); } + } + const publications = () => db.select().from(chatPublications).where(eq(chatPublications.issueId, f.issueId)); + + it.each([false, true])("rechecks concurrent membership suspension before restoring notice access (rollback=%s)", async rollback => { + const membershipWhere = and(eq(companyMemberships.companyId, f.companyId), eq(companyMemberships.principalId, f.userId)); + // A Board-origin reply uses the linked author and current membership. + // Remove the fixture's unrelated provider-ingress provenance. + await db.delete(chatMessageLinks).where(eq(chatMessageLinks.commentId, f.commentId)); + await db.update(issueComments).set({ authorUserId: f.userId }).where(eq(issueComments.id, f.commentId)); + const [principal] = await db.insert(chatExternalPrincipals).values({ companyId: f.companyId, + provider: "slack", providerAccountId: "", externalId: f.userId, + }).returning(); + await db.insert(chatIdentityLinks).values({ companyId: f.companyId, endpointId: f.endpointId, + principalId: principal.id, paperclipUserId: f.userId, status: "linked", + }); + await db.insert(chatActions).values({ companyId: f.companyId, endpointId: f.endpointId, + conversationId: f.conversationId, kind: "slack_board_message", status: "processed", + providerActionId: `board-comment:${f.commentId}`, payload: { issueId: f.issueId, commentId: f.commentId, + userId: f.userId, principalId: principal.id, agentId: f.agentId }, + }); + const runId = randomUUID(), sourceId = randomUUID(); + const nextCheckAt = new Date(Date.now() - 1000).toISOString(); + await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded", responsibleUserId: f.userId, + contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId }, + resultJson: { semanticToolReceipts: { schedule: { operationId: "set_task_monitor", + result: { taskId: f.issueId, monitor: { status: "scheduled", nextCheckAt } } } } }, + }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "running", responsibleUserId: f.userId, + contextSnapshot: { issueId: f.issueId, source: "issue.monitor" }, + }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, runId, + source: "automation", reason: "issue_monitor_due", status: "claimed", + idempotencyKey: `issue-monitor:${f.issueId}:${nextCheckAt}`, payload: { issueId: f.issueId, nextCheckAt }, + }); + await initializeRunIdentity(db, { companyId: f.companyId, runId, issueId: f.issueId, + responsibleUserId: f.userId, cause: "dispatch" }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + let releaseSuspension!: () => void; + const suspensionHeld = new Promise(resolve => { releaseSuspension = resolve; }); + let markSuspended!: () => void; + const suspended = new Promise(resolve => { markSuspended = resolve; }); + let releaseRestoration!: () => void; + const restorationHeld = new Promise(resolve => { releaseRestoration = resolve; }); + let restored = false; + let restorationPid = 0; + const suspending = db.transaction(async tx => { + await tx.update(companyMemberships).set({ status: "suspended" }).where(membershipWhere); + markSuspended(); + await suspensionHeld; + }); + await suspended; + const restoring = db.transaction(async tx => { + const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + restorationPid = backend.pid; + await accessService(tx as unknown as typeof db).ensureMembership(f.companyId, "user", f.userId, "operator"); + restored = true; + await restorationHeld; + if (rollback) throw new Error("outer membership rollback"); + }); + void restoring.catch(() => undefined); + try { + // The old read sees active membership, then its UPDATE waits. The fixed + // read waits first and observes the suspension after that writer commits. + await vi.waitFor(async () => { + expect(restorationPid).toBeGreaterThan(0); + const [state] = await db.execute<{ blocked: boolean }>(sql`select cardinality(pg_blocking_pids(${restorationPid})) > 0 as blocked`); + expect(state.blocked).toBe(true); + }); + releaseSuspension(); + await suspending; + await vi.waitFor(() => expect(restored).toBe(true)); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await vi.advanceTimersByTimeAsync(1000); + expect(await publications()).toEqual([]); + expect((await db.select().from(companyMemberships).where(membershipWhere))[0].status).toBe("suspended"); + releaseRestoration(); + if (rollback) await expect(restoring).rejects.toThrow("outer membership rollback"); + else await restoring; + await flush(); + expect((await db.select().from(companyMemberships).where(membershipWhere))[0].status) + .toBe(rollback ? "suspended" : "active"); + expect(await publications()).toEqual(rollback ? [] : [expect.objectContaining({ idempotencyKey: `run:${runId}:working:${f.endpointId}` })]); + await quiet(); + if (!rollback) { + const notices: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => notices.push(event)); + try { + await accessService(db).ensureMembership(f.companyId, "user", f.userId, "operator"); + expect(notices).toEqual([]); + } finally { unsubscribe(); } + await quiet(); + } + } finally { + releaseSuspension(); + releaseRestoration(); + await Promise.allSettled([suspending, restoring]); + } + }); + + it.each(["initialization", "steering"] as const)("wakes matching monitor lineage after %s commits, preserving rollback and replay", async kind => { + const nextCheckAt = new Date(Date.now() - 1000).toISOString(); + const sourceId = randomUUID(), successorId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: sourceId, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "succeeded", responsibleUserId: f.userId, + contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId }, + resultJson: { semanticToolReceipts: { schedule: { operationId: "set_task_monitor", + result: { taskId: f.issueId, monitor: { status: "scheduled", nextCheckAt } } } } }, + }); + await db.insert(heartbeatRuns).values({ id: successorId, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "running", responsibleUserId: null, + contextSnapshot: { issueId: f.issueId, source: "issue.monitor" }, + }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, runId: successorId, + source: "automation", reason: "issue_monitor_due", status: "claimed", + idempotencyKey: `issue-monitor:${f.issueId}:${nextCheckAt}`, payload: { issueId: f.issueId, nextCheckAt }, + }); + const identityInput = { companyId: f.companyId, runId: successorId, issueId: f.issueId, responsibleUserId: f.userId, cause: "dispatch" }; + if (kind === "steering") { + await initializeRunIdentity(db, { ...identityInput, responsibleUserId: null }); + await db.update(issueComments).set({ authorUserId: f.userId }).where(eq(issueComments.id, f.commentId)); + } + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await flush(); expect(await publications()).toEqual([]); await quiet(); + const pending = kind === "steering" ? await reserveSteeredIdentity(db, { + companyId: f.companyId, runId: successorId, issueId: f.issueId, messageId: f.commentId, + }) : null; + // Reserving an unacknowledged message must not activate its identity. + if (kind === "steering") expect(pending?.status).toBe("pending"); + await flush(); expect(await publications()).toEqual([]); await quiet(); + const activate = (rollback: boolean) => db.transaction(async tx => { + if (kind === "steering") await acceptSteeredIdentity(tx, pending!); + else await initializeRunIdentity(tx as unknown as typeof db, identityInput); + await vi.advanceTimersByTimeAsync(1000); + expect(await publications()).toEqual([]); + if (rollback) throw new Error("outer rollback"); + }); + await expect(activate(true)).rejects.toThrow("outer rollback"); + await flush(); expect(await publications()).toEqual([]); await quiet(); + await activate(false); await flush(); + expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${successorId}:working:${f.endpointId}` })]); + await quiet(); + const notices: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => notices.push(event)); + try { + await db.transaction(async tx => { + if (kind === "steering") await acceptSteeredIdentity(tx, pending!); + else await initializeRunIdentity(tx as unknown as typeof db, identityInput); + }); + expect(notices).toEqual([]); + } finally { unsubscribe(); } + await flush(); await quiet(); + }); +}); diff --git a/server/src/__tests__/chat-run-milestone-recovery-work.test.ts b/server/src/__tests__/chat-run-milestone-recovery-work.test.ts new file mode 100644 index 00000000000..b1d3cd9bf24 --- /dev/null +++ b/server/src/__tests__/chat-run-milestone-recovery-work.test.ts @@ -0,0 +1,103 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { chatPublications, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; +import { recoveryService } from "../services/recovery/service.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MILESTONE_RECOVERY_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("chat milestones after run recovery", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let fixtures: ReturnType; + let f: Awaited>; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-recovery-milestones-"); + db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + fixtures = createChatMilestoneFixtures(db); + }, 60_000); + beforeEach(async () => { + errors.length = 0; f = await fixtures.create(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await fixtures.cleanup(); expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function flush() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + } + async function quiet() { + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + try { await vi.advanceTimersByTimeAsync(120_000); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); } + finally { reads.mockRestore(); writes.mockRestore(); } + } + const publications = () => db.select().from(chatPublications).where(eq(chatPublications.issueId, f.issueId)); + + it.each(["process_gone", "issue_cancelled"] as const)("publishes the recovered terminal state after %s and becomes quiet", async reason => { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + status: "running", runtimeMode: "legacy", responsibleUserId: f.userId, + processPid: reason === "process_gone" ? 2_000_000_000 : process.pid, + contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId }, + }); + await db.update(issues).set({ executionRunId: runId, + status: reason === "issue_cancelled" ? "cancelled" : "in_progress", + }).where(eq(issues.id, f.issueId)); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await flush(); + expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${runId}:working:${f.endpointId}` })]); + await quiet(); + + const recovery = recoveryService(db, { enqueueWakeup: vi.fn() }); + expect(await recovery.sweepStaleIssueLocks()).toEqual({ cleared: 1, issueIds: [f.issueId], terminalizedRunIds: [runId] }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0].status) + .toBe(reason === "process_gone" ? "interrupted" : "cancelled"); + await flush(); + expect((await publications()).map(row => row.idempotencyKey).sort()).toEqual([ + `run:${runId}:failed:${f.endpointId}`, `run:${runId}:working:${f.endpointId}`, + ].sort()); + await quiet(); + expect(await recovery.sweepStaleIssueLocks()).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] }); + await flush(); await quiet(); expect(await publications()).toHaveLength(2); + }); + + it("wakes an older failed run when recovery refreshes its quota classification", async () => { + const runId = randomUUID(), old = new Date(Date.now() - 48 * 60 * 60_000); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + status: "failed", runtimeMode: "legacy", responsibleUserId: f.userId, + createdAt: old, startedAt: old, finishedAt: old, updatedAt: old, + error: "You've hit your usage limit for GPT-5. Try again at 12:00 AM (UTC).", errorCode: "adapter_failed", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: f.endpointId, wakeCommentId: f.commentId }, + }); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await flush(); expect(await publications()).toEqual([]); await quiet(); + const enqueueWakeup = vi.fn(async () => null); + const recovery = recoveryService(db, { enqueueWakeup }); + expect(await recovery.reconcileStrandedAssignedIssues()).toMatchObject({ providerQuotaMonitored: 1 }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)))[0].errorCode).toBe("provider_quota"); + expect(enqueueWakeup).not.toHaveBeenCalled(); + await flush(); + expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${runId}:failed:${f.endpointId}` })]); + await quiet(); + expect(await recovery.reconcileStrandedAssignedIssues()).toMatchObject({ providerQuotaMonitored: 0, skipped: 1 }); + await flush(); await quiet(); expect(await publications()).toHaveLength(1); + }); + +}); diff --git a/server/src/__tests__/chat-run-milestone-resource-work.test.ts b/server/src/__tests__/chat-run-milestone-resource-work.test.ts new file mode 100644 index 00000000000..8574ba2a70a --- /dev/null +++ b/server/src/__tests__/chat-run-milestone-resource-work.test.ts @@ -0,0 +1,125 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { chatConversations, chatDeliveries, chatEndpointResources, chatEndpoints, chatPublications, createDb, heartbeatRuns, issues, subscribeDatabaseWork } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; +import { chatCredentialMutationLease } from "../services/chat-credential-mutation-lease.js"; +import { createChatResourceReconciliation } from "../services/chat-channels/resource-reconciliation.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MILESTONE_RESOURCE_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("chat milestones after resource recovery", () => { + let db: ReturnType; + let temporary: Awaited> | undefined; + let fixtures: ReturnType; + let f: Awaited>; + let coordinator: ReturnType | undefined; + const errors: unknown[] = []; + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-resource-milestones-"); + db = createDb(external ?? temporary!.connectionString, { idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + fixtures = createChatMilestoneFixtures(db); + }, 60_000); + beforeEach(async () => { + errors.length = 0; f = await fixtures.create(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await db.delete(chatDeliveries).where(eq(chatDeliveries.companyId, f.companyId)); + await fixtures.cleanup(); expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + async function flush() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + } + async function quiet() { + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + try { await vi.advanceTimersByTimeAsync(120_000); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); } + finally { reads.mockRestore(); writes.mockRestore(); } + } + const publications = () => db.select().from(chatPublications).where(eq(chatPublications.issueId, f.issueId)); + + it.each(["inventory", "lifecycle", "github repository", "telegram migration"] as const)("wakes only reopened conversations after %s commits, preserving rollback and completed history", async path => { + const provider = path === "github repository" ? "github" : path === "telegram migration" ? "telegram" : "slack"; + const resourceType = provider === "github" ? "repository" : provider === "telegram" ? "chat" : "channel"; + const providerResourceId = provider === "github" ? "owner/repo" : provider === "telegram" ? "-100222" : "recovered-channel"; + await db.update(chatEndpoints).set({ provider }).where(eq(chatEndpoints.id, f.endpointId)); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, + responsibleUserId: f.userId, status: "failed", errorCode: "test_run_failed", updatedAt: new Date(), + contextSnapshot: { issueId: f.issueId, source: `chat:${provider}`, endpointId: f.endpointId, wakeCommentId: f.commentId }, + }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, f.issueId)); + const [endpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId)); + const [resource] = await db.insert(chatEndpointResources).values({ companyId: f.companyId, endpointId: f.endpointId, + type: resourceType, providerResourceId: provider === "telegram" ? "-111" : providerResourceId, label: "Recovered", availability: "unavailable", enabled: true, + }).returning(); + await db.update(chatConversations).set({ resourceId: resource.id, state: "unavailable", ...(provider === "telegram" ? { externalConversationId: "telegram:-111", externalThreadId: "telegram:-111:42" } : {}) }).where(eq(chatConversations.id, f.conversationId)); + const completedId = randomUUID(); + await db.insert(chatConversations).values({ id: completedId, companyId: f.companyId, endpointId: f.endpointId, + // Telegram migration intentionally rebinds all conversations on the old resource. + // Unrelated completed history must remain passive. + resourceId: provider === "telegram" ? null : resource.id, issueId: f.issueId, state: "completed", externalConversationId: completedId, externalThreadId: completedId, externalLabel: "History" }); + const [delivery] = await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpointId, + providerEventId: randomUUID(), deduplicationKey: randomUUID(), eventKind: "installation", normalizedEvent: {}, state: "processing", attempts: 1, + }).returning(); + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => true).ready; + await flush(); expect(await publications()).toEqual([]); await quiet(); + const withLease = chatCredentialMutationLease(db); + const recover = (rollback: boolean, alreadyPublished = false) => withLease(endpoint, lease => db.transaction(async tx => { + const resources = createChatResourceReconciliation(tx as unknown as typeof db, { + endpointRecord: async () => ({ endpoint }), withCredentialMutationLease: withLease, runtimeGeneration: () => 0, maxErrorText: 1000, + }); + if (path === "inventory") { + await resources.reconcileProviderResourceRows(endpoint, { provider, resources: [{ type: resourceType, providerResourceId, label: "Recovered" }] }, lease); + } else if (path === "github repository") { + await resources.reconcileGitHubWebhookRepository(endpoint, { repository: { id: 123, full_name: providerResourceId, name: "repo", owner: { id: 321, login: "owner" } } }, lease); + } else { + await resources.applyProviderAvailability({ currentEndpoint: endpoint, credentialLease: lease, staleResourceEffect: false, + claimed: delivery, candidate: { attempts: 0 }, effect: { kind: "resource", provider, providerEventId: delivery.providerEventId, + providerResourceId, resourceType, ...(provider === "telegram" ? { previousProviderResourceId: "-111" } : {}), label: "Recovered", availability: "available" }, + }); + } + // The owner used a nested transaction. Its savepoint release cannot publish. + await vi.advanceTimersByTimeAsync(1000); + expect(await publications()).toHaveLength(alreadyPublished ? 1 : 0); + if (rollback) throw new Error("outer rollback"); + })); + await expect(recover(true)).rejects.toThrow("outer rollback"); + await flush(); expect(await publications()).toEqual([]); await quiet(); + await recover(false); await flush(); + expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${runId}:failed:${f.endpointId}` })]); + expect((await db.select().from(chatConversations).where(eq(chatConversations.id, completedId)))[0].state).toBe("completed"); + if (provider === "telegram") { + expect((await db.select().from(chatConversations).where(eq(chatConversations.id, f.conversationId)))[0]) + .toMatchObject({ externalConversationId: "telegram:-100222", externalThreadId: "telegram:-100222:42", state: "active" }); + } + await quiet(); + // Unchanged inventory must not turn a provider refresh into milestone polling. + const notices: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => notices.push(event)); + try { + await withLease(endpoint, lease => createChatResourceReconciliation(db, { + endpointRecord: async () => ({ endpoint }), withCredentialMutationLease: withLease, runtimeGeneration: () => 0, maxErrorText: 1000, + }).reconcileProviderResourceRows(endpoint, { provider, resources: [{ type: resourceType, providerResourceId, label: "Recovered" }] }, lease)); + expect(notices).toEqual([]); + if (provider === "telegram") { + // A duplicate migration has no old bindings left to reactivate. + await recover(false, true); + expect(notices).toEqual([]); + } + } finally { unsubscribe(); } + await flush(); await quiet(); + }); +}); diff --git a/server/src/__tests__/chat-run-milestone-work.test.ts b/server/src/__tests__/chat-run-milestone-work.test.ts new file mode 100644 index 00000000000..d8bb45530ca --- /dev/null +++ b/server/src/__tests__/chat-run-milestone-work.test.ts @@ -0,0 +1,987 @@ +import { documentService } from "../services/documents.js"; +import { createChatMilestoneFixtures } from "./helpers/chat-milestone-fixtures.js"; +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, agentWakeupRequests, caseHandoffs, cases, chatConversations, chatEndpoints, chatPublications, completionContracts, createDb, nativeRunFinalizations, nativeRunResults, statusDecisions, statusDecisionEffects, workAssessments, subscribeDatabaseWork, heartbeatRuns, issueComments, issueQuestionResponseDeliveries, issueThreadInteractions, issues } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { chatRunMilestoneWork } from "../services/chat-run-milestone-work.js"; +import { notifyChatRunMilestones } from "../services/chat-run-milestone-notifications.js"; +import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; +import { questionResponseDeliveryService } from "../services/question-response-delivery.js"; +import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js"; +import { createDiscardQueuedComment, createReorderQueuedComments } from "../modules/wake-queue/application/queued-comment-use-cases.js"; +import { createRunDispatch } from "../modules/run-dispatch/index.js"; +import { createAdmissionTransactionScope, createPostgresWakeQueueAdapter, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; +import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; +import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY, CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE } from "../services/chat-control-recovery-stop.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { commitNativeStatusDecision, dispatchPendingNativeStatusEffects } from "../services/native-runtime/status-decision-committer.js"; +import { claimQueuedNativeReviewRun } from "../services/native-runtime/native-review-dispatch.js"; +import { createChatPublicationResolution } from "../services/chat-channels/publications/resolution.js"; +import { createChatPublicationWorker } from "../services/chat-channels/publications/worker.js"; +import { CAPABILITIES } from "../services/chat-channels/provider-catalog.js"; +import { createChatPublicationFailures } from "../services/chat-channels/publications/failures.js"; +import { PaperclipRunnerToolAuthority } from "../services/native-runtime/paperclip-runner-tool-authority.js"; +import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; + +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MILESTONE_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("chat lifecycle milestone worker", () => { + let temporary: Awaited> | undefined; + let db: ReturnType; + let coordinator: ReturnType | undefined; + let enabled = true; + const errors: unknown[] = []; + let fixtures: ReturnType; + type Fixture = Awaited>; + let f: Fixture; + + beforeAll(async () => { + if (!external) temporary = await startEmbeddedPostgresTestDatabase("chat-milestones-"); + db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + fixtures = createChatMilestoneFixtures(db); + }, 60_000); + const fixture = (companyId?: string) => fixtures.create(companyId); + beforeEach(async () => { + enabled = true; errors.length = 0; f = await fixture(); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); + }); + afterEach(async () => { + await coordinator?.stop(); coordinator = undefined; vi.useRealTimers(); vi.restoreAllMocks(); + await fixtures.cleanup(); + expect(errors).toEqual([]); + }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + function runValues(scope = f, patch: Partial = {}) { + return { id: randomUUID(), companyId: scope.companyId, agentId: scope.agentId, responsibleUserId: scope.userId, status: "running", updatedAt: new Date(), + contextSnapshot: { issueId: scope.issueId, source: "chat:slack", endpointId: scope.endpointId, wakeCommentId: scope.commentId }, ...patch }; + } + async function run(scope = f, patch: Partial = {}) { + const values = runValues(scope, patch); + await db.transaction(async tx => { await notifyChatRunMilestones(tx, scope); await tx.insert(heartbeatRuns).values(values); }); + return values; + } + async function start() { + coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => enabled, canReconcile: () => true, onError: error => errors.push(error) }); + await chatRunMilestoneWork(db).register(coordinator, () => enabled).ready; + } + async function flush() { + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(1000); + expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); + }, { timeout: 10_000 }); + } + async function quiet(ms = 120_000) { + const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + try { await vi.advanceTimersByTimeAsync(ms); expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); } + finally { reads.mockRestore(); writes.mockRestore(); } + } + const publications = (scope = f) => db.select().from(chatPublications).where(eq(chatPublications.issueId, scope.issueId)); + + it("leaves configured idle, internal runs and old history quiet", async () => { + await run(f, { contextSnapshot: { issueId: f.issueId, source: "issue.comment" } }); + await run(f, { updatedAt: new Date(Date.now() - 2 * 86_400_000) }); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + }); + it("reconstructs missing notices at startup and leaves published history quiet across restart", async () => { + const row = await run(); await start(); await flush(); + expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${row.id}:working:${f.endpointId}` })]); + await quiet(); await coordinator!.stop(); await start(); await flush(); await quiet(); + expect(await publications()).toHaveLength(1); + }); + it("wakes on the first committed row after becoming quiet", async () => { + await start(); await flush(); await quiet(); await run(); await flush(); + expect(await publications()).toHaveLength(1); await quiet(); + }); + it("wakes from the real run cancellation transaction without a manual milestone signal", async () => { + const row = await run(f, { status: "queued", runtimeMode: "legacy" }); + await start(); await flush(); await quiet(); + // This fixture exercises the Stop producer, not successor execution. + await heartbeatService(db).cancelRun(row.id, "Operator stopped queued work", { suppressImmediateRecovery: true }); + await flush(); + expect((await publications()).map(publication => publication.idempotencyKey).sort()).toEqual([ + `run:${row.id}:failed:${f.endpointId}`, `run:${row.id}:queued:${f.endpointId}`, + ].sort()); + await quiet(); + }); + it.each(["promote", "suppress"] as const)("wakes when real retry dispatch chooses to %s", async action => { + const row = await run(f, { status: "scheduled_retry", runtimeMode: "legacy", scheduledRetryAt: new Date(Date.now() - 1000), scheduledRetryReason: "transient_error" }); + if (action === "suppress") await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + const dispatch = createRunDispatch(db); + const result = await dispatch.promoteScheduledRetry({ companyId: f.companyId, runId: row.id }); + expect(result.outcome).toBe(action === "promote" ? "promoted" : "gate_suppressed"); + await flush(); + expect((await publications()).map(publication => publication.idempotencyKey)).toContain( + `run:${row.id}:${action === "promote" ? "queued" : "failed"}:${f.endpointId}`, + ); + await quiet(); + expect((await dispatch.promoteScheduledRetry({ companyId: f.companyId, runId: row.id })).outcome).toBe("not_promoted"); + await quiet(); + }); + it("wakes when the real staleness gate cancels queued work", async () => { + const row = await run(f, { status: "queued", runtimeMode: "legacy", scheduledRetryReason: "max_turns_continuation" }); + await start(); await flush(); await quiet(); + await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + const dispatch = createRunDispatch(db); + await dispatch.cancelStaleQueuedRun({ companyId: f.companyId, runId: row.id, expectedStatus: "queued" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, row.id)))[0].status).toBe("cancelled"); + await flush(); + expect((await publications()).map(publication => publication.idempotencyKey)).toContain(`run:${row.id}:failed:${f.endpointId}`); + await quiet(); + await dispatch.cancelStaleQueuedRun({ companyId: f.companyId, runId: row.id, expectedStatus: "queued" }); + await quiet(); + }); + it.each(["deferred", "immediate"] as const)("wakes when real %s recovery admits chat work", async kind => { + const finishing = await run(f, { status: "succeeded", runtimeMode: "legacy", finishedAt: new Date(), + contextSnapshot: { issueId: f.issueId, source: "issue_assigned" }, + }); + const wakeId = randomUUID(); + if (kind === "deferred") await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.companyId, agentId: f.agentId, + source: "on_demand", status: "deferred_issue_execution", payload: { issueId: f.issueId }, + }); + const adapter = createPostgresWakeQueueAdapter(db, { + resolveResponsibleUserId: async () => f.userId, + getRoutineEnv: async () => ({ routineId: null, env: null, responsibleUserId: null }), + resolveSessionBeforeForWakeup: async () => null, + }); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + let admittedId: string | undefined; + await adapter.withIssueExecutionLock({ companyId: f.companyId, runId: finishing.id, now: new Date() }, async (locked, ports) => { + const input = { companyId: f.companyId, issue: locked.primaryIssue, finishingRun: locked.run, + contextSnapshot: runValues().contextSnapshot, responsibleUserId: f.userId, sessionBefore: null, now: new Date(), + }; + const agent = { id: f.agentId, companyId: f.companyId, name: "Maya", invokable: true }; + if (kind === "deferred") { + expect(await ports.transaction.claimDeferredWakeForPromotion({ companyId: f.companyId, wakeId, now: new Date() })).toBe(true); + admittedId = (await ports.transaction.finalizePromotedWake({ ...input, wakeId, deferredAgent: agent, + reason: "issue_execution_promoted", source: "on_demand", triggerDetail: null, payload: {}, + })).id; + } else { + admittedId = (await ports.transaction.queueImmediateRecoveryRun({ ...input, recoveryAgent: agent, reason: "issue_execution_recovery" })).id; + } + expect(await publications()).toEqual([]); + return { outcome: { kind: "released" }, postCommitEffects: [] }; + }); + expect(admittedId).toBeDefined(); + await flush(); + expect((await publications()).map(publication => publication.idempotencyKey)).toContain(`run:${admittedId}:queued:${f.endpointId}`); + await quiet(); + }); + async function queue() { + const dependencies = createHeartbeatQueueTestDependencies(db); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + dependencies.getAgent.mockResolvedValue(agent); + dependencies.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental()); + return { service: createHeartbeatQueue(db, dependencies), dependencies }; + } + it("wakes on real queue admission while execution is held by drain", async () => { + const { service, dependencies } = await queue(); + dependencies.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" }); + await start(); await flush(); await quiet(); + const admitted = await service.enqueueWakeup(f.agentId, { source: "on_demand", reason: "issue_commented", + contextSnapshot: runValues().contextSnapshot, requestedByActorType: "user", requestedByActorId: f.userId, + }); + expect(admitted).toMatchObject({ status: "queued", companyId: f.companyId }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${admitted!.id}:queued:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + }); + it.each(["ordinary", "legacy_comments", "queued_comments"] as const)("wakes on a real %s queue claim", async kind => { + const wakeId = randomUUID(); + const context = runValues().contextSnapshot; + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.companyId, agentId: f.agentId, + source: "on_demand", status: "queued", + payload: kind === "queued_comments" ? { _paperclipWakeContext: { wakeCommentIds: [f.commentId] } } : {}, + }); + const seeded = await run(f, { status: "queued", runtimeMode: "legacy", wakeupRequestId: wakeId, + contextSnapshot: kind === "ordinary" ? context : { ...context, wakeCommentIds: [f.commentId] }, + }); + await db.update(agentWakeupRequests).set({ runId: seeded.id }).where(eq(agentWakeupRequests.id, wakeId)); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)); + const { service, dependencies } = await queue(); + await start(); await flush(); await quiet(); + expect(await service.claimQueuedRun(before)).toMatchObject({ id: seeded.id, status: "running" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:working:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + expect(await service.claimQueuedRun(before)).toBeNull(); + await quiet(); + }); + it("wakes when claim rejects a superseded case handoff and leaves repeated claims quiet", async () => { + const wakeId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.companyId, agentId: f.agentId, + source: "assignment", status: "queued", payload: { issueId: f.issueId }, + }); + const seeded = await run(f, { status: "queued", runtimeMode: "legacy", wakeupRequestId: wakeId }); + const [record] = await db.insert(cases).values({ companyId: f.companyId, caseNumber: 1, identifier: "M-CASE-1", + caseType: "work", title: "Chat task", workIssueId: f.issueId, handoffVersion: 2 }).returning(); + await db.insert(caseHandoffs).values({ companyId: f.companyId, caseId: record.id, issueId: f.issueId, + version: 1, toAgentId: f.agentId, phase: "implementation", instructions: "Build the change", actorType: "system", + actorId: "test", idempotencyKey: "superseded", inputFingerprint: "fixture", wakeupRequestId: wakeId, + expectedIssueStatusVersion: 0, appliedIssueStatusVersion: 0, state: "delivered" }); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)); + const { service, dependencies } = await queue(); + await start(); await flush(); await quiet(); + expect(await service.claimQueuedRun(before)).toBeNull(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)))[0]) + .toMatchObject({ status: "cancelled", errorCode: "case_handoff_superseded" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:failed:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + expect(await service.claimQueuedRun(before)).toBeNull(); + await quiet(); + }); + it("wakes when dispatch fails an unverifiable chat recovery and leaves repeated gates quiet", async () => { + const seeded = await run(f, { status: "running", runtimeMode: "legacy", invocationSource: "automation", + runnerProfileJson: { [CHAT_CONTROL_RECOVERY_ADMISSION_KEY]: {} }, + }); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)); + const { service, dependencies } = await queue(); + const dispatch = vi.fn(async () => null); + await start(); await flush(); await quiet(); + await service.withChatControlRecoveryGate(before, "dispatch", dispatch); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)))[0]) + .toMatchObject({ status: "failed", errorCode: CHAT_CONTROL_RECOVERY_UNRESOLVED_CODE }); + expect(dispatch).not.toHaveBeenCalled(); expect(dependencies.executeRun).not.toHaveBeenCalled(); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:failed:${f.endpointId}`); + await quiet(); + await service.withChatControlRecoveryGate(before, "dispatch", dispatch); + await quiet(); + }); + it.each(["queued", "scheduled_retry"] as const)("publishes cancellation when admission replaces a reassigned %s owner", async status => { + const old = await run(f, { status, runtimeMode: "legacy", scheduledRetryAt: status === "scheduled_retry" ? new Date(Date.now() + 60_000) : null }); + const [replacement] = await db.insert(agents).values({ companyId: f.companyId, name: "Replacement", adapterType: "paperclip_runner" }).returning(); + await db.update(issues).set({ assigneeAgentId: replacement.id, executionRunId: old.id }).where(eq(issues.id, f.issueId)); + const { service, dependencies } = await queue(); + dependencies.getAgent.mockResolvedValue(replacement); + dependencies.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" }); + await start(); await flush(); await quiet(); + const admitted = await service.enqueueWakeup(replacement.id, { source: "on_demand", reason: "issue_commented", + contextSnapshot: runValues().contextSnapshot, requestedByActorType: "user", requestedByActorId: f.userId, + }); + expect(admitted).toMatchObject({ agentId: replacement.id, status: "queued" }); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, old.id)))[0]) + .toMatchObject({ status: "cancelled", errorCode: status === "queued" ? "lock_released_on_reassignment" : "issue_reassigned" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${old.id}:failed:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); await quiet(); + }); + async function editableQueue(staleContext = false) { + const wakeId = randomUUID(); + await db.update(issueComments).set({ authorUserId: f.userId }).where(eq(issueComments.id, f.commentId)); + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.companyId, agentId: f.agentId, + source: "on_demand", status: "queued", payload: { issueId: f.issueId, _paperclipWakeContext: { wakeCommentIds: [f.commentId] } }, + }); + const seeded = await run(f, { status: "queued", runtimeMode: "legacy", wakeupRequestId: wakeId, + contextSnapshot: { ...runValues().contextSnapshot, wakeCommentId: staleContext ? randomUUID() : f.commentId }, + }); + await db.update(agentWakeupRequests).set({ runId: seeded.id }).where(eq(agentWakeupRequests.id, wakeId)); + const issue = { id: f.issueId, companyId: f.companyId, assigneeAgentId: f.agentId, executionRunId: seeded.id }; + const actor = { actorType: "user" as const, actorId: f.userId, agentId: null, runId: null, agentApiKeyId: null }; + const issueLock = createQueuedCommentIssueLockWriter(db, { + syncCommentReferences: async () => {}, deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {}, + }); + const snapshot = await issueLock.withLockedQueue({ issue, actor, queueId: wakeId }, async locked => locked.queue); + return { seeded, issueLock, input: { issue, actor, queueId: wakeId, revision: snapshot.revision, now: new Date() } }; + } + it("wakes when queued-comment mutation reconstructs run lineage from its authoritative queue", async () => { + const { seeded, issueLock, input } = await editableQueue(true); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + await createReorderQueuedComments({ issueLock })({ ...input, orderedCommentIds: [f.commentId] }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:queued:${f.endpointId}`); + await quiet(); + }); + it("settles queued-comment discard intent only after the owning transaction commits", async () => { + const { seeded, input } = await editableQueue(); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "chat-run-milestones", event => events.push(event)); + try { + await db.transaction(async tx => { + const writer = createQueuedCommentIssueLockWriter(tx as unknown as typeof db, { + syncCommentReferences: async () => {}, deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {}, + }); + await createDiscardQueuedComment({ issueLock: writer })({ ...input, commentId: f.commentId }); + expect(events).toContain("intent"); expect(events).not.toContain("settled"); + }); + expect(events).toContain("settled"); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)))[0]) + .toMatchObject({ status: "cancelled", errorCode: "queued_comment_discarded" }); + } finally { unsubscribe(); } + }); + it("wakes when the real coalescing writer gives an existing run chat lineage", async () => { + const seeded = await run(f, { contextSnapshot: { issueId: f.issueId, source: "issue_assigned" } }); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + await db.transaction(async tx => { + await createWakeAdmissionWriter().coalesceIntoActiveExecutionRun( + createAdmissionTransactionScope(f.companyId, tx as unknown as typeof db), { + companyId: f.companyId, activeExecutionRunId: seeded.id, mergedContextSnapshot: runValues().contextSnapshot, + agentId: f.agentId, source: "on_demand", triggerDetail: null, payload: { issueId: f.issueId }, + requestedByActorType: "user", requestedByActorId: f.userId, idempotencyKey: null, + }); + expect(await publications()).toEqual([]); + }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:working:${f.endpointId}`); + await quiet(); + }); + it("wakes on a claim released for drain and leaves repeated release quiet", async () => { + const seeded = await run(); + const { service } = await queue(); + await start(); await flush(); await quiet(); + await service.releaseRunClaimedJustBeforeSuppression(seeded.id); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:queued:${f.endpointId}`); + await quiet(); + await service.releaseRunClaimedJustBeforeSuppression(seeded.id); + await quiet(); + }); + it("wakes an admitted monitor through its previously committed source receipt", async () => { + const source = await run(f, { runtimeMode: "native", nativeIssueId: f.issueId }); + await db.update(issues).set({ executionRunId: source.id }).where(eq(issues.id, f.issueId)); + await start(); await flush(); await quiet(); + const nextCheckAt = new Date(Date.now() + 60_000).toISOString(); + const authority = new PaperclipRunnerToolAuthority(db, { ...f, runId: source.id }); + const request = { tool: "set_task_monitor", callId: randomUUID(), arguments: { + idempotencyKey: "milestone-monitor", monitor: { nextCheckAt, notes: "Check the saved task" }, + } }; + await expect(authority.execute(request)).resolves.toMatchObject({ taskId: f.issueId, replayed: false }); + // The monitor and source receipt commit together, before due admission. This + // fixture finishes the source without executing a harness; admission below + // uses the real queue, with execution held by task drain. + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, source.id)); + await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId)); + await quiet(); + const { service, dependencies } = await queue(); + dependencies.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" }); + const admitted = await service.enqueueWakeup(f.agentId, { source: "automation", reason: "issue_monitor_due", + idempotencyKey: `issue-monitor:${f.issueId}:${nextCheckAt}`, + payload: { issueId: f.issueId, nextCheckAt }, + contextSnapshot: { issueId: f.issueId, source: "issue.monitor" }, + requestedByActorType: "user", requestedByActorId: f.userId, + }); + expect(admitted).toMatchObject({ status: "queued" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${admitted!.id}:queued:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + }); + async function nativeFixture(phase = "committed") { + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ + id: contractId, companyId: f.companyId, issueId: f.issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "phase6-v1", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { objective: "Respond to chat" }, canonicalSha256: "milestone-contract", + createdByActorType: "system", createdByActorId: "test", + }); + // Persisted audit-only routing can commit without a status decision. Recovery + // still owes the terminal run projection after the controller restarts. + const row = await run(f, { runtimeMode: "native", nativeIssueId: f.issueId, completionContractId: contractId, + resultJson: { nativeAttentionRouting: [{ decisionId: null, reasonCode: "attention_duplicate_suppressed", + materializedTargets: [{ kind: "existing_attention" }] }] }, + }); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, + runId: row.id, completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", + resultJson: { terminal: { runTerminalState: "cancelled" } }, canonicalSha256: "milestone-result", + }); + await db.insert(nativeRunFinalizations).values({ runId: row.id, companyId: f.companyId, issueId: f.issueId, + phase, resultId, + }); + return { row, contractId, resultId }; + } + async function continuationDecisionFixture() { + const native = await nativeFixture("assessing"); + const assessmentId = randomUUID(); + await db.insert(workAssessments).values({ id: assessmentId, companyId: f.companyId, issueId: f.issueId, + runId: native.row.id, contractId: native.contractId, resultId: native.resultId, triggerKind: "run_result", + triggerActorCompanyId: f.companyId, priorIssueStatus: "in_progress", priorStatusVersion: 0, + policyVersion: "phase6-v1", assessmentJson: {}, inputDigest: assessmentId, + }); + return { companyId: f.companyId, issueId: f.issueId, runId: native.row.id, assessmentId, + priorStatus: "in_progress", priorStatusVersion: 0, priorDecisionId: null, + decision: { policyVersion: "phase6-v1", statusAction: "preserve" as const, toStatus: "in_progress" as const, + reasonCode: "live_continuation_registered", unblockDescriptor: null, + effects: [{ kind: "enqueue_continuation" as const, agentId: f.agentId, continuationKind: "same_agent" as const, + summary: "Continue responding", idempotencyKey: `continuation:${native.row.id}` }], + }, + }; + } + it("wakes from a governed native reviewer claim and leaves repeated claims quiet", async () => { + const input = await continuationDecisionFixture(); + const { decision } = await commitNativeStatusDecision({ db, ...input }); + const [reviewer] = await db.insert(agents).values({ companyId: f.companyId, name: "Reviewer", adapterType: "paperclip_runner" }).returning(); + await db.update(statusDecisions).set({ decisionJson: { ...decision.decisionJson, projectedStatusVersion: 1 } }).where(eq(statusDecisions.id, decision.id)); + await db.update(issues).set({ status: "in_review", statusVersion: 1, lastStatusDecisionId: decision.id, executionRunId: null }).where(eq(issues.id, f.issueId)); + await db.update(chatEndpoints).set({ assignedAgentId: reviewer.id }).where(eq(chatEndpoints.id, f.endpointId)); + const [interaction] = await db.insert(issueThreadInteractions).values({ companyId: f.companyId, issueId: f.issueId, + kind: "request_confirmation", status: "pending", sourceRunId: input.runId, addresseeAgentId: reviewer.id, + effectiveResolverPolicy: "anyone", resolverPolicyProvenance: "inherited", + payload: { version: 1, prompt: "Review the work", target: { type: "custom", key: "native_completion_review", revisionId: decision.id } }, + }).returning(); + const wakeId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId: f.companyId, agentId: reviewer.id, source: "automation", status: "queued" }); + const seeded = await run(f, { agentId: reviewer.id, status: "queued", runtimeMode: "native", nativeIssueId: f.issueId, wakeupRequestId: wakeId, + contextSnapshot: { ...runValues().contextSnapshot, nativeReviewInteractionId: interaction.id, nativeReviewDecisionId: decision.id }, + }); + await db.update(agentWakeupRequests).set({ runId: seeded.id }).where(eq(agentWakeupRequests.id, wakeId)); + const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, seeded.id)); + const claim = { run: before, agentNameKey: "reviewer", claimedAt: new Date(), claimValues: {} }; + await start(); await flush(); await quiet(); + expect(await claimQueuedNativeReviewRun(db, claim)).toMatchObject({ id: seeded.id, status: "running" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${seeded.id}:working:${f.endpointId}`); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].executionRunId).toBe(seeded.id); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wakeId)))[0].status).toBe("claimed"); + await quiet(); + expect(await claimQueuedNativeReviewRun(db, claim)).toBeNull(); await quiet(); + }); + it("wakes from native committed-result projection and leaves repeated finalization quiet", async () => { + const { row } = await nativeFixture(); + await start(); await flush(); await quiet(); + await finalizeNativeRun({ db, runId: row.id, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + await flush(); + expect((await publications()).map(publication => publication.idempotencyKey).sort()).toEqual([ + `run:${row.id}:failed:${f.endpointId}`, `run:${row.id}:working:${f.endpointId}`, + ].sort()); + await quiet(); + await finalizeNativeRun({ db, runId: row.id, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + // A no-op repair must not enqueue another scan. This also catches deferred + // coordinator callbacks that start after the finalizer returns. + await quiet(); + expect(await publications()).toHaveLength(2); + }); + it("signals a new native decision at outer commit but not an already committed replay", async () => { + const input = await continuationDecisionFixture(); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, "chat-run-milestones", event => events.push(event)); + try { + await db.transaction(async tx => { + await commitNativeStatusDecision({ db: tx as unknown as typeof db, ...input }); + expect(events).toEqual(["intent"]); + }); + expect(events).toEqual(["intent", "settled"]); + events.length = 0; + await expect(commitNativeStatusDecision({ db, ...input })).resolves.toMatchObject({ replayed: true }); + expect(events).toEqual([]); + } finally { unsubscribe(); } + }); + it("wakes a suppressed continuation when its native effect delivery receipt commits", async () => { + const input = await continuationDecisionFixture(); + const { decision } = await commitNativeStatusDecision({ db, ...input }); + const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + // Reconstruct an interrupted delivery with its already admitted successor. + // The target has no direct chat identity: only the delivered ledger can + // authorize inheriting the source run's conversation. + await db.update(statusDecisionEffects).set({ deliveryState: "pending", deliveredAt: null }) + .where(eq(statusDecisionEffects.decisionId, decision.id)); + const target = await run(f, { status: "queued", contextSnapshot: { + issueId: f.issueId, source: "native_status_decision", nativeStatusWakeIntentId: intent.id, + } }); + await db.update(agentWakeupRequests).set({ status: "coalesced", runId: target.id }).where(eq(agentWakeupRequests.id, intent.id)); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + reason: "issue_status_changed", requestedByActorType: "system", requestedByActorId: `native-status-wake-dispatch:${intent.id}`, + runId: target.id, status: "queued", idempotencyKey: intent.idempotencyKey, payload: { issueId: f.issueId }, + }); + await start(); await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${target.id}:`))).toBe(false); + await quiet(); + await dispatchPendingNativeStatusEffects({ db, ...input, decisionId: decision.id }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${target.id}:queued:${f.endpointId}`); + await quiet(); + await expect(dispatchPendingNativeStatusEffects({ db, ...input, decisionId: decision.id })).resolves.toEqual([]); + await quiet(); + }); + it("wakes an admitted continuation when native wake dispatch repairs its original receipt", async () => { + const input = await continuationDecisionFixture(); + await commitNativeStatusDecision({ db, ...input }); + const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + const target = await run(f, { status: "queued", contextSnapshot: { + issueId: f.issueId, source: "native_status_decision", nativeStatusWakeIntentId: intent.id, + } }); + await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agentId, source: "automation", + reason: "issue_status_changed", requestedByActorType: "system", requestedByActorId: `native-status-wake-dispatch:${intent.id}`, + runId: target.id, status: "queued", idempotencyKey: intent.idempotencyKey, payload: { issueId: f.issueId }, + }); + await start(); await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${target.id}:`))).toBe(false); + await quiet(); + const { service, dependencies } = await queue(); + await expect(service.dispatchPendingNativeStatusWakeups({ companyId: f.companyId })).resolves.toMatchObject({ recovered: 1, dispatched: 0 }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${target.id}:queued:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + await expect(service.dispatchPendingNativeStatusWakeups({ companyId: f.companyId })).resolves.toMatchObject({ scanned: 0 }); + await quiet(); + }); + it("rechecks a fresh native continuation after admission finishes its wake receipt", async () => { + const input = await continuationDecisionFixture(); + await commitNativeStatusDecision({ db, ...input }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + await start(); await flush(); await quiet(); + const { service, dependencies } = await queue(); + let admittedId: string | undefined; + dependencies.getSchedulingSuppression.mockImplementation(async () => { + const rows = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const admitted = rows.find(row => row.id !== input.runId); + if (admitted && !admittedId) { + admittedId = admitted.id; + // Admission already committed and signalled, but the original wake + // receipt is still claimed. Let that first projection finish now. + await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${admittedId}:`))).toBe(false); + } + return { suppressed: true, reason: "task_drain" }; + }); + await expect(service.dispatchPendingNativeStatusWakeups({ companyId: f.companyId })).resolves.toMatchObject({ dispatched: 1 }); + expect(admittedId).toBeDefined(); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${admittedId}:queued:${f.endpointId}`); + expect(dependencies.executeRun).not.toHaveBeenCalled(); + await quiet(); + }); + it("projects a continuation only after the real question delivery receipt commits", async () => { + const source = await run(f, { status: "succeeded" }); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: source.id, + title: "Choose", payload: { version: 1, title: "Choose", questions: [{ + id: "choice", prompt: "Continue?", selectionMode: "single", required: true, + options: [{ id: "yes", label: "Yes" }], + }] }, + }, { agentId: f.agentId, runId: source.id }); + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId, status: "in_progress" }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: f.userId }); + // A provider acknowledgement is durable input to this projection test. + await db.insert(chatPublications).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: f.conversationId, + issueId: f.issueId, idempotencyKey: `interaction:${interaction.id}:${f.endpointId}`, + payload: { text: "Continue?", interactionId: interaction.id }, state: "published", providerMessageId: "question-1", + }).onConflictDoNothing(); + await db.update(chatPublications).set({ state: "published", providerMessageId: "question-1" }).where(eq(chatPublications.issueId, f.issueId)); + await start(); await flush(); await quiet(); + let targetId!: string; + const delivery = questionResponseDeliveryService(db, { heartbeat: { + cancelRun: async () => { throw new Error("The source is already terminal"); }, + wakeup: async (_agentId, options) => { + const values = runValues(f, { status: "queued", contextSnapshot: options.contextSnapshot }); + const [target] = await db.insert(heartbeatRuns).values(values).returning(); + targetId = target.id; + await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${target.id}:`))).toBe(false); + return target; + }, + } }); + await expect(delivery.deliver(interaction.id)).resolves.toMatchObject({ status: "fallback_queued" }); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${targetId}:queued:${f.endpointId}`); + await quiet(); + }); + it.each([ + ["question", "answer"], ["question", "skip"], ["question", "cancel"], + ["question", "withdraw"], ["question", "expire"], ["question", "historical answer"], + ["confirmation", "accept"], ["confirmation", "reject"], ["confirmation", "skip"], + ["confirmation", "withdraw"], ["confirmation", "expire"], + ["question", "comment"], ["question", "historical comment"], + ["confirmation", "comment"], ["confirmation", "historical comment"], + ["question", "replacement"], ["confirmation", "replacement"], ["confirmation", "startup supersession"], + ["question", "addressee deletion"], ["confirmation", "addressee deletion"], + ] as const)("rechecks an unsent %s after %s", async (kind, action) => { + const source = await run(f, { status: "failed", errorCode: "test_run_failed" }); + const interactions = issueThreadInteractionService(db); + const issue = { id: f.issueId, companyId: f.companyId, status: "in_progress" }; + const addressee = action === "addressee deletion" + ? (await db.insert(agents).values({ companyId: f.companyId, name: "Reviewer", adapterType: "paperclip_runner" }).returning())[0] : null; + const createInteraction = (sourceRunId: string, supersedePendingSiblingInteractions = true) => interactions.create(issue, kind === "confirmation" ? { + kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId, addresseeAgentId: addressee?.id, + title: "Confirm", payload: { version: 1, prompt: "Continue?", supersedeOnUserComment: true }, + } : { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId, addresseeAgentId: addressee?.id, + title: "Choose", payload: { version: 1, title: "Choose", supersedeOnUserComment: true, questions: [{ + id: "choice", prompt: "Continue?", selectionMode: "single", required: true, + options: [{ id: "yes", label: "Yes" }], + }] }, + }, { agentId: f.agentId, runId: sourceRunId }, { supersedePendingSiblingInteractions }); + const interaction = await createInteraction(source.id); + const replacement = action === "replacement" || action === "startup supersession" ? await run() : null; + if (action === "startup supersession") { + await createInteraction(replacement!.id, false); + await db.update(issueThreadInteractions).set({ createdAt: new Date(Date.now() - 1000) }).where(eq(issueThreadInteractions.id, interaction.id)); + } + if (action === "historical answer") { + await interactions.dismissInteraction(issue, interaction.id, { userId: f.userId }); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + } + await start(); await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${source.id}:`))).toBe(false); + await quiet(); + if (action === "addressee deletion") { + expect((await interactions.cancelPendingForDeletedAddressee(f.companyId, addressee!.id)).map(row => row.id)).toContain(interaction.id); + } else if (action === "startup supersession") { + expect(await interactions.sweepSupersededPendingRequestConfirmations()).toMatchObject({ expired: 1 }); + } else if (action === "replacement") { + await createInteraction(replacement!.id); + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interaction.id)))[0].status).toBe("expired"); + } else if (action === "comment" || action === "historical comment") { + const [comment] = await db.insert(issueComments).values({ companyId: f.companyId, issueId: f.issueId, + authorType: "user", authorUserId: f.userId, origin: "comment", body: "Ignore the earlier request" }).returning(); + const expired = action === "comment" + ? await interactions.expireRequestConfirmationsSupersededByComment(issue, comment, { userId: f.userId }) + : await interactions.expireRequestConfirmationsSupersededByHistoricalComments(issue); + expect(expired.map(row => row.id)).toContain(interaction.id); + } else if (action === "accept") { + await interactions.acceptInteraction({ ...issue, projectId: null, goalId: null }, interaction.id, {}, { userId: f.userId }); + } else if (action === "reject") { + await interactions.rejectInteraction(issue, interaction.id, { reason: "No longer needed" }, { userId: f.userId }); + } else if (action === "skip") { + await interactions.skipInteraction(issue, interaction.id, { reason: "No longer needed" }, { userId: f.userId }); + } else if (action === "cancel") { + await interactions.cancelQuestions(issue, interaction.id, { reason: "No longer needed" }, { userId: f.userId }); + } else if (action === "withdraw") { + await interactions.withdrawInteraction(issue, interaction.id, { reason: "No longer needed" }, { userId: f.userId }); + } else if (action === "expire") { + await db.update(issues).set({ status: "cancelled" }).where(eq(issues.id, f.issueId)); + await interactions.expirePendingInteractionsForTerminalIssue({ ...issue, status: "cancelled" }, { userId: f.userId }); + } else { + await interactions.answerQuestions(issue, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: f.userId }); + } + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${source.id}:failed:${f.endpointId}`); + if (action === "historical answer") { + expect(await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id))).toEqual([]); + } + await quiet(); + }); + it.each(["change", "delete", "accept", "reject"] as const)("rechecks a document confirmation after %s detects a stale target", async mode => { + const source = await run(f, { status: "failed", errorCode: "test_run_failed" }); + const interactions = issueThreadInteractionService(db), docs = documentService(db); + const issue = { id: f.issueId, companyId: f.companyId, status: "in_progress", projectId: null, goalId: null }; + const input = { issueId: f.issueId, key: "plan", format: "markdown", body: "First plan", createdByUserId: f.userId }; + const { document } = await docs.upsertIssueDocument(input); + const interaction = await interactions.create(issue, { + kind: "request_confirmation", continuationPolicy: "wake_assignee", sourceRunId: source.id, + payload: { version: 1, prompt: "Apply this plan?", target: { type: "issue_document", issueId: f.issueId, + documentId: document.id, key: document.key, revisionId: document.latestRevisionId!, revisionNumber: document.latestRevisionNumber } }, + }, { agentId: f.agentId, runId: source.id }); + await start(); await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${source.id}:`))).toBe(false); + await quiet(); + const changed = mode === "delete" + ? { ...(await docs.deleteIssueDocument(f.issueId, document.key))!, latestRevisionId: null, latestRevisionNumber: null } + : (await docs.upsertIssueDocument({ ...input, body: "Revised plan", baseRevisionId: document.latestRevisionId })).document; + await flush(); await quiet(); + if (mode === "accept" || mode === "reject") { + await expect(mode === "accept" + ? interactions.acceptInteraction(issue, interaction.id, {}, { userId: f.userId }) + : interactions.rejectInteraction(issue, interaction.id, {}, { userId: f.userId })) + .rejects.toMatchObject({ status: 409, details: { code: "interaction_stale_target" } }); + } else { + expect((await interactions.expireStaleRequestConfirmationsForIssueDocument(issue, changed, { userId: f.userId })).map(row => row.id)).toContain(interaction.id); + } + expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interaction.id)))[0].status).toBe("expired"); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${source.id}:failed:${f.endpointId}`); + await quiet(); + }); + it("wakes a question continuation after its in-flight prompt receipt commits", async () => { + await db.update(chatEndpoints).set({ provider: "discord" }).where(eq(chatEndpoints.id, f.endpointId)); + const source = await run(f, { status: "failed", contextSnapshot: { ...runValues().contextSnapshot, source: "chat:discord" } }); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: source.id, + title: "Choose", payload: { version: 1, title: "Choose", questions: [{ + id: "choice", prompt: "Continue?", selectionMode: "single", required: true, + options: [{ id: "yes", label: "Yes" }], + }] }, + }, { agentId: f.agentId, runId: source.id }); + const prompt = (await publications()).find(row => row.idempotencyKey === `interaction:${interaction.id}:${f.endpointId}`)!; + const [endpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId)); + const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.id, f.conversationId)); + const targetId = randomUUID(); + let beforeReceiptKeys: string[] | undefined; + await start(); await flush(); await quiet(); + const deliver = questionResponseDeliveryService(db, { heartbeat: { + wakeup: async () => { + const [target] = await db.insert(heartbeatRuns).values(runValues(f, { + id: targetId, status: "queued", contextSnapshot: { issueId: f.issueId }, + })).returning(); + return target; + }, + } }); + const post = vi.fn(async () => { + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId, status: "in_progress" }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: f.userId }); + await deliver.deliver(interaction.id); + await flush(); + beforeReceiptKeys = (await publications()).map(row => row.idempotencyKey); + return { id: "published-question" }; + }); + // Controlled transport/authorization ports; actual attempt lease, receipt + // transaction, question delivery receipt and milestone worker use PostgreSQL. + const worker = createChatPublicationWorker(db, { + CAPABILITIES, settleGitHubAutomaticPublication: async () => false, + ensureDurablePublicationTransport: async publication => publication, + tryProcessTeamsFilePublication: async () => false, + endpointRecord: async () => ({ endpoint, assignedAgentName: "Maya", connectionName: "Chat", applicationId: "test", credentialSecretRefs: [] }), + runtimeContextForRecord: () => ({ generation: 0, localEpoch: 0, credentialFingerprint: "test", version: "test" }), + hasCommittedTaskControlCompletion: async () => false, nonDirectDestinationAllowed: () => true, + runOwnershipMilestoneSupersessionReason: async () => null, + progressSupersededByFastResponse: async () => false, runProgressSupersededByPublishedInteraction: async () => false, + currentTaskControlPayload: async publication => publication.payload, + withCredentialMutationLease: async (_endpoint, operation) => operation({ assertOwned: async () => {} }), + closeProgressToReplace: async () => null, interactionResolutionPublicationToReplace: async () => null, + interactionPromptPublicationToReplace: async () => null, runPublicationToReplace: async () => null, + inboundWakePublicationToReplace: async () => null, taskStatusPublicationToReplace: async () => null, + claimPublicationTransportAuthorization: async () => ({ endpoint, conversation, authorizationActionId: null, preparedText: null }), + postSafePublication: post, commitTaskControlCompletion: async () => {}, receiptReactionCompletionRunId: async () => null, + finalizePublicationFailure: async (_publication, error) => { errors.push(error); return false; }, + redactError: String, + } as Parameters[1]); + await worker.processSelectedPublication(prompt); + expect(post).toHaveBeenCalledTimes(1); + expect((await db.select().from(chatPublications).where(eq(chatPublications.id, prompt.id)))[0]).toMatchObject({ state: "published", providerMessageId: "published-question" }); + expect(beforeReceiptKeys).toBeDefined(); + expect(beforeReceiptKeys).not.toContain(`run:${targetId}:queued:${f.endpointId}`); + await flush(); + expect((await publications()).map(row => row.idempotencyKey)).toContain(`run:${targetId}:queued:${f.endpointId}`); + await quiet(); + }); + it.each([ + ["retry", null], ["failed", null], ["delivery_unknown", "cancel"], ["delivery_unknown", "mark_delivered"], + ] as const)("rechecks resolved prompt ownership after delivery %s (%s)", async (state, resolution) => { + const source = await run(f, { status: "failed", errorCode: "test_run_failed" }); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: source.id, + title: "Choose", payload: { version: 1, title: "Choose", questions: [{ + id: "choice", prompt: "Continue?", selectionMode: "single", required: true, + options: [{ id: "yes", label: "Yes" }], + }] }, + }, { agentId: f.agentId, runId: source.id }); + // Model a send already claimed when the user answers through the UI. + const [prompt] = await db.update(chatPublications).set({ state: "streaming", attempts: 1 }) + .where(eq(chatPublications.idempotencyKey, `interaction:${interaction.id}:${f.endpointId}`)).returning(); + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId, status: "in_progress" }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: f.userId }); + expect(prompt).toBeDefined(); + await start(); await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${source.id}:`))).toBe(false); + await quiet(); + const failures = createChatPublicationFailures(db, { + FailedChatRetryPublicationReadError: class extends Error {}, + MAX_ERROR_TEXT: 1000, getTaskBaseUrl: () => null, + invalidateRuntime: async () => false, runtimeCallbackEndpoint: async () => null, + }); + const failure = Object.assign(new Error("provider rejected the send"), { status: state === "retry" ? 429 : 400 }); + // The finalizer receives the selection from before the attempt increment. + await expect(failures.finalizePublicationFailure({ ...prompt, attempts: 0 }, failure, state === "delivery_unknown")).resolves.toBe(true); + expect((await db.select().from(chatPublications).where(eq(chatPublications.id, prompt.id)))[0].state).toBe(state); + await flush(); + expect((await publications()).some(row => row.idempotencyKey === `run:${source.id}:failed:${f.endpointId}`)).toBe(state === "failed"); + await quiet(); + if (resolution) { + const targetId = randomUUID(); + await questionResponseDeliveryService(db, { heartbeat: { wakeup: async () => { + const [target] = await db.insert(heartbeatRuns).values(runValues(f, { + id: targetId, status: "queued", contextSnapshot: { issueId: f.issueId }, + })).returning(); + return target; + } } }).deliver(interaction.id); + await flush(); + expect((await publications()).some(row => row.idempotencyKey.startsWith(`run:${targetId}:`))).toBe(false); + await quiet(); + const [endpoint] = await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId)); + const resolver = createChatPublicationResolution(db, { + endpointRecord: async () => ({ endpoint, assignedAgentName: "Maya", connectionName: "Chat", applicationId: "test", credentialSecretRefs: [] }), + withCredentialMutationLease: async (_endpoint, operation) => operation({ assertOwned: async () => {} }), + commitTaskControlCompletion: async () => {}, processPendingPublications: async () => {}, + } as Parameters[1]); + await resolver.resolvePublication(f.endpointId, prompt.id, resolution, f.userId); + await flush(); + const keys = (await publications()).map(row => row.idempotencyKey); + expect(keys.includes(`run:${source.id}:failed:${f.endpointId}`)).toBe(resolution === "cancel"); + expect(keys.includes(`run:${targetId}:queued:${f.endpointId}`)).toBe(resolution === "mark_delivered"); + await quiet(); + } + }); + it.each(["retry", "stale claim", "live claim", "rolled-back reset"] as const)("rechecks question prompt binding after %s", async (mode) => { + const source = await run(); + const interactions = issueThreadInteractionService(db); + const interaction = await interactions.create({ id: f.issueId, companyId: f.companyId }, { + kind: "ask_user_questions", continuationPolicy: "wake_assignee", sourceRunId: source.id, + title: "Choose", payload: { version: 1, title: "Choose", questions: [{ + id: "choice", prompt: "Continue?", selectionMode: "single", required: true, + options: [{ id: "yes", label: "Yes" }], + }] }, + }, { agentId: f.agentId, runId: source.id }); + await interactions.answerQuestions({ id: f.issueId, companyId: f.companyId, status: "in_progress" }, interaction.id, + { answers: [{ questionId: "choice", optionIds: ["yes"] }] }, { userId: f.userId }); + const other = await fixture(f.companyId); + await db.update(chatEndpoints).set({ assignedAgentId: f.agentId }).where(eq(chatEndpoints.id, other.endpointId)); + await db.update(chatConversations).set({ issueId: f.issueId }).where(eq(chatConversations.id, other.conversationId)); + await db.insert(chatPublications).values({ companyId: f.companyId, endpointId: other.endpointId, conversationId: other.conversationId, + issueId: f.issueId, idempotencyKey: `interaction:${interaction.id}:${other.endpointId}`, + payload: { text: "Continue?", interactionId: interaction.id }, state: "published", providerMessageId: "other-question", + }); + const events: string[] = []; + let claimEvents: string[] = []; + let duringClaimNotices: string[] | undefined; + const unsubscribe = subscribeDatabaseWork(db, "chat-run-milestones", event => events.push(event)); + const delivery = questionResponseDeliveryService(db, { heartbeat: { + cancelRun: async () => { + claimEvents = [...events]; + await start(); await flush(); + duringClaimNotices = (await publications()).map(row => row.idempotencyKey); + throw new Error("temporary cancellation failure"); + }, + wakeup: async () => { throw new Error("Must retain the still-active source"); }, + } }); + try { + if (mode === "retry") { + await expect(delivery.deliver(interaction.id)).resolves.toBeNull(); + } else { + await db.update(issueQuestionResponseDeliveries).set({ status: "delivering", attemptCount: 1, + lastAttemptAt: new Date(Date.now() - (mode === "live claim" ? 0 : 60_000)), + }).where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)); + await start(); await flush(); + duringClaimNotices = (await publications()).map(row => row.idempotencyKey); + if (mode === "rolled-back reset") { + await expect(db.transaction(async tx => { + await questionResponseDeliveryService(tx as unknown as typeof db, { heartbeat: {} as never }).sweepPending(0); + expect(events).toEqual(["intent"]); + throw new Error("rollback reset"); + })).rejects.toThrow("rollback reset"); + } else { + await delivery.sweepPending(0); + } + } + expect(duringClaimNotices).toBeDefined(); + expect(duringClaimNotices!.some(key => key.startsWith(`run:${source.id}:`))).toBe(false); + expect((await db.select().from(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.interactionId, interaction.id)))[0].status).toBe(mode === "live claim" || mode === "rolled-back reset" ? "delivering" : "pending"); + await flush(); + const noticeExists = (await publications()).some(row => row.idempotencyKey === `run:${source.id}:working:${f.endpointId}`); + expect(noticeExists).toBe(mode === "retry" || mode === "stale claim"); + if (mode === "retry") expect(claimEvents).toEqual(["intent", "settled"]); + else if (mode === "live claim") expect(events).toEqual([]); + else { + expect(events[0]).toBe("intent"); + expect(events.slice(1)).toContain("settled"); + expect(events.filter(event => event === "intent")).toHaveLength(1); + } + await quiet(); + } finally { unsubscribe(); } + }); + it("waits for outer commit and ignores rolled-back nested writes", async () => { + await start(); await flush(); + await expect(db.transaction(async tx => { + await tx.transaction(async nested => { await notifyChatRunMilestones(nested, f); await nested.insert(heartbeatRuns).values(runValues()); }); + throw new Error("rollback fixture"); + })).rejects.toThrow("rollback fixture"); + await flush(); expect(await publications()).toEqual([]); await quiet(); + let release!: () => void, staged!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const stagedPromise = new Promise(resolve => { staged = resolve; }); + const writing = db.transaction(async tx => { + await tx.transaction(async nested => { await notifyChatRunMilestones(nested, f); await nested.insert(heartbeatRuns).values(runValues()); }); + staged(); await gate; + }); + try { await stagedPromise; await vi.advanceTimersByTimeAsync(1000); expect(await publications()).toEqual([]); } + finally { release(); await writing; } + await flush(); expect(await publications()).toHaveLength(1); await quiet(); + }); + it("retains the first committed wake during drain without periodic SQL", async () => { + await start(); await flush(); enabled = false; + await run(); await vi.advanceTimersByTimeAsync(1000); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + const drained = idleWorkSnapshot(); + expect(await publications()).toEqual([]); await quiet(); expect(idleWorkSnapshot()).toEqual(drained); + enabled = true; await vi.advanceTimersByTimeAsync(30_000); await flush(); + expect(await publications()).toHaveLength(1); await quiet(); + }); + it("scopes mutation wakes to their issue and company", async () => { + const sibling = await fixture(f.companyId), foreign = await fixture(); + await start(); await flush(); + // Unsupported raw writers deliberately do not wake: they provide a sentinel + // proving a supported issue mutation does not perform a global scan. + await db.insert(heartbeatRuns).values([runValues(sibling), runValues(foreign)]); + await run(); await flush(); + expect(await publications()).toHaveLength(1); expect(await publications(sibling)).toEqual([]); expect(await publications(foreign)).toEqual([]); + await db.transaction(tx => notifyChatRunMilestones(tx, { companyId: f.companyId })); + await flush(); expect(await publications(sibling)).toHaveLength(1); expect(await publications(foreign)).toEqual([]); + await quiet(); + }); + it("disarms suppressed lineage and reactivates after its binding changes", async () => { + const row = await run(f, { contextSnapshot: { issueId: f.issueId, source: "chat:slack", endpointId: randomUUID(), wakeCommentId: f.commentId } }); + await start(); await flush(); expect(await publications()).toEqual([]); await quiet(); + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, f); + await tx.update(heartbeatRuns).set({ contextSnapshot: runValues().contextSnapshot }).where(eq(heartbeatRuns.id, row.id)); + }); + await flush(); expect(await publications()).toHaveLength(1); await quiet(); + }); + it("waits quietly for a late presentation decision", async () => { + const row = await run(f, { status: "succeeded" }); await start(); await flush(); + expect(await publications()).toEqual([]); await quiet(); + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, f); + await tx.update(heartbeatRuns).set({ resultJson: { presentationDecision: { source: "none" } } }).where(eq(heartbeatRuns.id, row.id)); + }); + await flush(); expect(await publications()).toEqual([expect.objectContaining({ idempotencyKey: `run:${row.id}:completed:${f.endpointId}` })]); await quiet(); + }); + it("drains more than one batch without duplicate notices or starving another company", async () => { + const other = await fixture(); + await db.insert(heartbeatRuns).values([...Array.from({ length: 205 }, () => runValues()), runValues(other)]); + await start(); await flush(); expect(await publications()).toHaveLength(205); expect(await publications(other)).toHaveLength(1); await quiet(); + }); + it("retains a failed selection for retry through drain", async () => { + await run(); + const select = vi.spyOn(db, "select").mockImplementationOnce(() => { throw new Error("temporary read failure"); }); + await start(); select.mockRestore(); expect(errors.splice(0)).toHaveLength(1); + enabled = false; await vi.advanceTimersByTimeAsync(30_000); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); + const drained = idleWorkSnapshot(); + await quiet(); expect(idleWorkSnapshot()).toEqual(drained); + enabled = true; await vi.advanceTimersByTimeAsync(30_000); await flush(); + expect(await publications()).toHaveLength(1); await quiet(); + }); + it("retries a failed publication write without idle scans before its deadline", async () => { + await run(); + const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("temporary publication failure")); + await start(); transaction.mockRestore(); expect(errors.splice(0)).toHaveLength(1); + expect(await publications()).toEqual([]); await quiet(10_000); + await vi.advanceTimersByTimeAsync(30_000); await flush(); expect(await publications()).toHaveLength(1); await quiet(); + }); + it("preserves a committed change arriving while a projection batch is running", async () => { + await run(); + const transaction = db.transaction.bind(db); + let release!: () => void, entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const staged = new Promise(resolve => { entered = resolve; }); + const held = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => { + entered(); await gate; return transaction(callback, config); + }); + const starting = start(); + try { + await staged; + await transaction(async tx => { await notifyChatRunMilestones(tx, f); await tx.insert(heartbeatRuns).values(runValues()); }); + } finally { release(); await starting; held.mockRestore(); } + await flush(); expect(await publications()).toHaveLength(2); await quiet(); + }); + it("stops future wakes and reconstructs writes made after shutdown", async () => { + await start(); await flush(); await coordinator!.stop(); await run(); await quiet(); + expect(await publications()).toEqual([]); await start(); await flush(); expect(await publications()).toHaveLength(1); + }); +}); diff --git a/server/src/__tests__/cloud-tenant-company-provisioning.test.ts b/server/src/__tests__/cloud-tenant-company-provisioning.test.ts index c57dde6d63e..3c08f90e7e7 100644 --- a/server/src/__tests__/cloud-tenant-company-provisioning.test.ts +++ b/server/src/__tests__/cloud-tenant-company-provisioning.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { createHash } from "node:crypto"; import { afterAll, afterEach, beforeEach, beforeAll, describe, expect, it } from "vitest"; import { and, eq } from "drizzle-orm"; @@ -18,7 +19,8 @@ import { } from "./helpers/embedded-postgres.js"; import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "../middleware/auth.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_CLOUD_PROVISIONING_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -59,8 +61,8 @@ describeEmbeddedPostgres("cloud tenant company provisioning", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tenant-provisioning-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-tenant-provisioning-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); beforeEach(() => { @@ -80,6 +82,7 @@ describeEmbeddedPostgres("cloud tenant company provisioning", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/github-operation-credentials.test.ts b/server/src/__tests__/github-operation-credentials.test.ts index cdcb3a518fe..499134e2b55 100644 --- a/server/src/__tests__/github-operation-credentials.test.ts +++ b/server/src/__tests__/github-operation-credentials.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { githubCredentialsFixture } from "./helpers/github-credentials-fixture.js"; import { createNativeGitHubAccess } from "../services/native-runtime/native-github-access.js"; import express from "express"; @@ -34,8 +35,6 @@ import { import { initializeRunIdentity, captureRunIdentity, - reserveSteeredIdentity, - acceptSteeredIdentity, } from "../services/run-identity.js"; import { resolveGitHubOperationCredentials } from "../services/github-operation-credentials.js"; import { @@ -52,23 +51,25 @@ const vault = vi.hoisted(() => ({ resolveSecretValue: vi.fn(async () => "test-dedicated-token"), })); vi.mock("../services/secrets.js", () => ({ secretService: () => vault })); -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_GITHUB_CREDENTIAL_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)( "operation-time GitHub credential resolution", () => { - let database: Awaited>, + let database: Awaited> | undefined, db: ReturnType; beforeAll(async () => { vi.stubEnv( "PAPERCLIP_AGENT_JWT_SECRET", "test-github-broker-signing-secret", ); - database = await startEmbeddedPostgresTestDatabase( + if (!external) database = await startEmbeddedPostgresTestDatabase( "paperclip-github-operation-", ); - db = createDb(database.connectionString); + db = createDb(external ?? database!.connectionString); }, 30_000); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await database?.cleanup(); vi.unstubAllEnvs(); }, 60_000); @@ -85,7 +86,7 @@ const support = await getEmbeddedPostgresTestSupport(); transport: "mcp_remote" as const, config: { sourceTemplateKey, payload }, transportConfig: { sourceTemplateKey } }; }); await db.insert(toolConnections).values(irrelevant); - const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); + const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { toolConnections }, capture); try { @@ -117,7 +118,7 @@ const support = await getEmbeddedPostgresTestSupport(); resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload, }).where(eq(heartbeatRuns.id, input.runId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); - const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); + const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { diff --git a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts index 5e420f50f9e..5be4f7bd2ff 100644 --- a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts @@ -37,6 +37,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { heartbeatService } from "../services/heartbeat.ts"; import { runningProcesses } from "../adapters/index.ts"; @@ -63,7 +64,8 @@ vi.mock("../adapters/index.ts", async () => { }; }); -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEPENDENCY_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -103,8 +105,8 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () = let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-dependency-scheduling-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-dependency-scheduling-"); + db = createDb(external ?? tempDb!.connectionString); await db.insert(authUsers).values({ id: "responsible-user", name: "Fixture owner", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }); heartbeat = heartbeatService(db); await ensureIssueRelationsTable(db); @@ -185,6 +187,8 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () = }); afterAll(async () => { + await db?.delete(authUsers).where(eq(authUsers.id, "responsible-user")); + await db?.$client.end(); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/helpers/chat-milestone-fixtures.ts b/server/src/__tests__/helpers/chat-milestone-fixtures.ts new file mode 100644 index 00000000000..00302e480a8 --- /dev/null +++ b/server/src/__tests__/helpers/chat-milestone-fixtures.ts @@ -0,0 +1,72 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { + activityLog, agents, agentWakeupRequests, authUsers, + caseHandoffs, cases, companyMemberships, chatActions, + chatConversations, chatEndpoints, chatMessageLinks, chatPublications, + companies, completionContracts, documentRevisions, documents, + issueDocuments, type Db, nativeRunFinalizations, nativeRunResults, + statusDecisions, statusDecisionEffects, workAssessments, heartbeatRunEvents, + heartbeatRuns, issueComments, issueQuestionResponseDeliveries, issueThreadInteractions, + issues, toolApplications, toolConnections, +} from "@paperclipai/db"; + +// This owns only fixture data. Each test retains its worker, clock and quiet +// assertions, and must stop the worker before cleanup. +export function createChatMilestoneFixtures(db: Db) { + const fixtureCompanies: string[] = []; + const fixtureUsers: string[] = []; + async function create(companyId: string = randomUUID()) { + const agentId = randomUUID(), issueId = randomUUID(), endpointId = randomUUID(), conversationId = randomUUID(), commentId = randomUUID(), userId = randomUUID(); + if (!fixtureCompanies.includes(companyId)) { + fixtureCompanies.push(companyId); + await db.insert(companies).values({ id: companyId, name: "Milestones", issuePrefix: `M${companyId.slice(0, 7)}` }); + } + fixtureUsers.push(userId); + await db.insert(authUsers).values({ id: userId, name: "Owner", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole: "owner" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Maya", adapterType: "paperclip_runner" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Chat task", status: "in_progress", assigneeAgentId: agentId, responsibleUserId: userId }); + const applicationId = randomUUID(), connectionId = randomUUID(); + await db.insert(toolApplications).values({ id: applicationId, companyId, name: `Chat ${applicationId}`, type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Chat", uid: connectionId, transport: "chat_sdk", connectionPurpose: "channel", enabled: true, status: "active" }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, assignedAgentId: agentId, publicId: endpointId, provider: "slack", status: "active" }); + await db.insert(chatConversations).values({ id: conversationId, companyId, endpointId, issueId, externalConversationId: conversationId, externalThreadId: conversationId, externalLabel: "Chat" }); + await db.insert(issueComments).values({ id: commentId, companyId, issueId, authorType: "user", body: "Start work" }); + await db.insert(chatMessageLinks).values({ companyId, endpointId, conversationId, commentId, providerMessageId: commentId, direction: "inbound" }); + return { companyId, agentId, issueId, endpointId, conversationId, commentId, userId }; + } + async function cleanup() { + for (const companyId of fixtureCompanies.splice(0)) { + await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(caseHandoffs).where(eq(caseHandoffs.companyId, companyId)); + await db.delete(cases).where(eq(cases.companyId, companyId)); + await db.delete(chatActions).where(eq(chatActions.companyId, companyId)); + await db.delete(chatMessageLinks).where(eq(chatMessageLinks.companyId, companyId)); + await db.delete(chatPublications).where(eq(chatPublications.companyId, companyId)); + await db.delete(chatConversations).where(eq(chatConversations.companyId, companyId)); + await db.delete(chatEndpoints).where(eq(chatEndpoints.companyId, companyId)); + await db.delete(issueQuestionResponseDeliveries).where(eq(issueQuestionResponseDeliveries.companyId, companyId)); + await db.delete(issueThreadInteractions).where(eq(issueThreadInteractions.companyId, companyId)); + await db.delete(issueComments).where(eq(issueComments.companyId, companyId)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.companyId, companyId)); + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, companyId)); + await db.delete(statusDecisionEffects).where(eq(statusDecisionEffects.companyId, companyId)); + await db.delete(statusDecisions).where(eq(statusDecisions.companyId, companyId)); + await db.delete(workAssessments).where(eq(workAssessments.companyId, companyId)); + await db.delete(nativeRunResults).where(eq(nativeRunResults.companyId, companyId)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); + await db.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, companyId)); + await db.delete(completionContracts).where(eq(completionContracts.companyId, companyId)); + await db.delete(issueDocuments).where(eq(issueDocuments.companyId, companyId)); + await db.delete(documentRevisions).where(eq(documentRevisions.companyId, companyId)); + await db.delete(documents).where(eq(documents.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(agents).where(eq(agents.companyId, companyId)); + await db.delete(companyMemberships).where(eq(companyMemberships.companyId, companyId)); + await db.delete(companies).where(eq(companies.id, companyId)); + } + for (const userId of fixtureUsers.splice(0)) await db.delete(authUsers).where(eq(authUsers.id, userId)); + } + return { create, cleanup }; +} diff --git a/server/src/__tests__/helpers/github-credentials-fixture.ts b/server/src/__tests__/helpers/github-credentials-fixture.ts index cd18ca64c72..ee43dd18da0 100644 --- a/server/src/__tests__/helpers/github-credentials-fixture.ts +++ b/server/src/__tests__/helpers/github-credentials-fixture.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { agents, companies, companyMemberships, companySecrets, connectionGrants, heartbeatRuns, issueComments, issues, toolApplications, toolConnectionInstalls, toolConnections, userSecretDefinitions, type Db } from "@paperclipai/db"; -import { initializeRunIdentity, reserveSteeredIdentity, acceptSteeredIdentity } from "../../services/run-identity.js"; +import { initializeRunIdentity, reserveSteeredIdentity, reconcileSteeredIdentity } from "../../services/run-identity.js"; export function githubCredentialsFixture(getDb: () => Db) { async function seed() { @@ -145,7 +145,7 @@ export function githubCredentialsFixture(getDb: () => Db) { ...input, messageId: id, }); - await acceptSteeredIdentity(db, context!); + await reconcileSteeredIdentity(db, context!); } return { seed, grant, switchTo }; } diff --git a/server/src/__tests__/issue-recovery-actions.test.ts b/server/src/__tests__/issue-recovery-actions.test.ts index e1151c9c899..11f05b1ca7e 100644 --- a/server/src/__tests__/issue-recovery-actions.test.ts +++ b/server/src/__tests__/issue-recovery-actions.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; @@ -48,7 +49,8 @@ import { instanceSettingsService } from "../services/instance-settings.js"; import { withNativeWorkspaceFinalizationOwnership } from "../services/native-runtime/native-workspace-finalization-ownership.js"; import { noticeMetadataReferencesRecoveryAction } from "../services/recovery/successful-run-handoff.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RECOVERY_ACTION_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; function makeRecoveryActionRow(overrides: Record = {}) { @@ -154,8 +156,8 @@ describeEmbeddedPostgres("issue recovery actions", () => { beforeAll(async () => { operationLogRoot = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-recovery-operation-logs-")); process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = operationLogRoot; - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-recovery-actions-"); + db = createDb(external ?? tempDb!.connectionString); }, 30_000); afterEach(async () => { @@ -182,6 +184,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); if (priorOperationLogRoot === undefined) delete process.env.WORKSPACE_OPERATION_LOG_BASE_PATH; else process.env.WORKSPACE_OPERATION_LOG_BASE_PATH = priorOperationLogRoot; @@ -616,8 +619,8 @@ describeEmbeddedPostgres("issue recovery actions", () => { await db.update(agents).set({ status: "paused" }).where(eq(agents.id, coderId)); const enqueueWakeup = vi.fn(async () => null); const recovery = recoveryService(db, { enqueueWakeup }); - const operatorDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); - const probeDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const operatorDb = createDb(external ?? tempDb!.connectionString, { maxConnections: 1 }); + const probeDb = createDb(external ?? tempDb!.connectionString, { maxConnections: 1 }); let release!: () => void; let ready!: () => void; let holderPid = 0; @@ -766,7 +769,7 @@ describeEmbeddedPostgres("issue recovery actions", () => { .where(eq(issues.id, sourceIssue.id)); } const [observed] = await db.select().from(issues).where(eq(issues.id, sourceIssue.id)); - const singleConnectionDb = createDb(tempDb!.connectionString, { maxConnections: 1 }); + const singleConnectionDb = createDb(external ?? tempDb!.connectionString, { maxConnections: 1 }); let timer: ReturnType | undefined; try { const work = recoveryService(singleConnectionDb, { enqueueWakeup: async () => null }) diff --git a/server/src/__tests__/project-git-authorization.test.ts b/server/src/__tests__/project-git-authorization.test.ts index 940b8a5bc78..9bd67aa5a3b 100644 --- a/server/src/__tests__/project-git-authorization.test.ts +++ b/server/src/__tests__/project-git-authorization.test.ts @@ -1,3 +1,4 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import express from "express"; import request from "supertest"; import { eq } from "drizzle-orm"; @@ -20,18 +21,19 @@ vi.mock("../services/github-fetch.js", async importOriginal => ({ ...await importOriginal(), ghFetch: vi.fn(async () => new Response(null, { status: 404 })), })); -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_PROJECT_GIT_TEST_DATABASE_URL"); +const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("explicit project Git authorization", () => { let db: ReturnType, database: Awaited>; const { seed, grant, switchTo } = githubCredentialsFixture(() => db); beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("paperclip-project-git-authorization-"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-project-git-authorization-"); + db = createDb(external ?? database!.connectionString); await db.insert(authUsers).values(["A", "B"].map(id => ({ id, name: id, email: `${id}@example.invalid`, createdAt: new Date(), updatedAt: new Date(), }))); }, 30_000); - afterAll(async () => { await database?.cleanup(); }, 60_000); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, 60_000); async function projectAuthorizationFixture() { const input = await seed(); const selected = await grant(input, "A"); diff --git a/server/src/__tests__/question-response-delivery.test.ts b/server/src/__tests__/question-response-delivery.test.ts index ad3b603ec75..9b8a013c481 100644 --- a/server/src/__tests__/question-response-delivery.test.ts +++ b/server/src/__tests__/question-response-delivery.test.ts @@ -26,6 +26,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { issueThreadInteractionService } from "../services/issue-thread-interactions.js"; import { buildQuestionResponseDeliveryEnvelope, @@ -33,7 +34,8 @@ import { questionResponseDeliveryService, } from "../services/question-response-delivery.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_QUESTION_DELIVERY_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; @@ -64,10 +66,10 @@ describeEmbeddedPostgres("question response delivery", () => { > | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase( - "paperclip-question-delivery-", - ); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-question-delivery-"); + } + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString); }, 20_000); afterEach(async () => { @@ -84,6 +86,7 @@ describeEmbeddedPostgres("question response delivery", () => { }); afterAll(async () => { + await db?.$client.end(); await tempDb?.cleanup(); }); diff --git a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts index a6168189b0b..1e259d81e63 100644 --- a/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts +++ b/server/src/__tests__/recovery-stale-issue-lock-sweep.test.ts @@ -1,7 +1,9 @@ +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { + subscribeDatabaseWork, costEvents, activityLog, agents, @@ -25,9 +27,11 @@ const mockTelemetryClient = vi.hoisted(() => ({ track: vi.fn() })); vi.mock("../telemetry.ts", () => ({ getTelemetryClient: () => mockTelemetryClient })); import { heartbeatService } from "../services/heartbeat.ts"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { recoveryService } from "../services/recovery/service.ts"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STALE_LOCK_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -43,8 +47,8 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { let tempDb: Awaited> | null = null; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-stale-lock-sweep-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-stale-lock-sweep-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); afterEach(async () => { @@ -64,6 +68,7 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { }); afterAll(async () => { + await db?.$client.end({ timeout: 1 }); await tempDb?.cleanup(); }); @@ -585,8 +590,15 @@ describeEmbeddedPostgres("recovery sweepStaleIssueLocks", () => { it.each(["observed", "workspace_finalizing", "ready_for_assessment", "arbitrating", "retryable_failure"])( "preserves a completed native result in %s after its provider exits", async phase => { const { runningRunId, issueId } = await seedPendingNativeResult(phase); - const result = await recoveryService(db, { enqueueWakeup: vi.fn() }).sweepStaleIssueLocks(); - expect(result).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] }); + const events: string[] = []; + const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.chatRunMilestones, event => events.push(event)); + try { + const recovery = recoveryService(db, { enqueueWakeup: vi.fn() }); + for (let pass = 0; pass < 3; pass++) { + expect(await recovery.sweepStaleIssueLocks()).toEqual({ cleared: 0, issueIds: [], terminalizedRunIds: [] }); + } + expect(events).toEqual([]); + } finally { unsubscribe(); } expect(await db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }).from(heartbeatRuns) .where(eq(heartbeatRuns.id, runningRunId))).toEqual([{ status: "running", errorCode: null }]); expect(await db.select({ executionRunId: issues.executionRunId }).from(issues) diff --git a/server/src/__tests__/run-identity.test.ts b/server/src/__tests__/run-identity.test.ts index 1cc8ed20f7a..9f6dcb24c05 100644 --- a/server/src/__tests__/run-identity.test.ts +++ b/server/src/__tests__/run-identity.test.ts @@ -3,7 +3,7 @@ import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agentWakeupRequests, agents, companies, applyPendingMigrations, createDb, closeRegisteredClients, heartbeatRuns, heartbeatRunEvents, issueComments, issueThreadInteractions, issues, secretAccessEvents } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; -import { acceptSteeredIdentity, captureRunIdentity, initializeRunIdentity, listRunIdentityContexts, rejectSteeredIdentity, reserveSteeredIdentity } from "../services/run-identity.js"; +import { acceptSteeredIdentity, reconcileSteeredIdentity, captureRunIdentity, initializeRunIdentity, listRunIdentityContexts, rejectSteeredIdentity, reserveSteeredIdentity } from "../services/run-identity.js"; const externalDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); @@ -143,8 +143,8 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos expect((await captureRunIdentity(db, input, options)).context?.responsibleUserId).toBe("B"); expect(startedUnderA.context?.responsibleUserId).toBe("A"); const a = await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[2] }); - await acceptSteeredIdentity(db, a!); - await acceptSteeredIdentity(db, b!); + await reconcileSteeredIdentity(db, a!); + await reconcileSteeredIdentity(db, b!); expect((await captureRunIdentity(db, input, options)).context?.responsibleUserId).toBe("A"); }); it("keeps rejected steering unchanged and preserves the originating identity on retry", async () => { @@ -203,7 +203,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos sourceIdentityContextId: origin.id, payload: { prompt: "Continue?" } as never, }); const b = await reserveSteeredIdentity(db, { ...input, messageId: input.messageIds[1] }); - await acceptSteeredIdentity(db, b!); + await reconcileSteeredIdentity(db, b!); for (const approval of [true, false]) { const runId = randomUUID(); await db.insert(heartbeatRuns).values({ id: runId, companyId: input.companyId, agentId: input.agentId, status: "running" }); diff --git a/server/src/__tests__/voice-session-store.integration.test.ts b/server/src/__tests__/voice-session-store.integration.test.ts index 4ad00c8157d..7ba06dcb5d4 100644 --- a/server/src/__tests__/voice-session-store.integration.test.ts +++ b/server/src/__tests__/voice-session-store.integration.test.ts @@ -270,7 +270,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await db.update(chatVoiceInboundCalls).set({ expiresAt: new Date(0), updatedAt: new Date(0) }).where(eq(chatVoiceInboundCalls.id, f.call.id)); const onError = vi.fn(); const maintenance = createChatReconciliationCoordinator({ - processPendingDeliveries: async () => undefined, projectRunMilestones: async () => 0, + processPendingDeliveries: async () => undefined, projectNativeProgress: async () => 0, flushPublications: async () => undefined, onError, }); try { diff --git a/server/src/app.ts b/server/src/app.ts index 6da6cb70474..96aba422f03 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -195,7 +195,8 @@ import { createToolGatewayService } from "./services/tool-gateway.js"; import { toolAccessService } from "./services/tool-access.js"; import { chatChannelService } from "./services/chat-channels.js"; import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js"; -import { enqueueChatRunMilestones } from "./services/chat-run-publications.js"; +import { enqueueSafeNativeChatProgress } from "./services/chat-run-publications.js"; +import { chatRunMilestoneWork } from "./services/chat-run-milestone-work.js"; import { createCoalescedAsyncTrigger, isChatPublicationCommitSignal, @@ -323,7 +324,7 @@ export function shouldEnablePrivateHostnameGuard(opts: { type ChatReconciliationLane = | "deliveries" | "GitHub webhook recovery" - | "run milestones" + | "native progress" | "publications" | "Slack file receipts" | "Slack session status"; @@ -336,7 +337,7 @@ type ChatReconciliationLane = export function createChatReconciliationCoordinator(input: { processPendingDeliveries: () => Promise; processFailedGitHubWebhookDeliveries?: () => Promise; - projectRunMilestones: () => Promise; + projectNativeProgress: () => Promise; flushPublications: () => Promise; processPendingSlackFileUploadReceipts?: () => Promise; processPendingSlackSessionSyncs?: () => Promise; @@ -348,15 +349,15 @@ export function createChatReconciliationCoordinator(input: { run: input.flushPublications, onError: (error) => input.onError("publications", error), }); - const milestoneReconciliation = createCoalescedAsyncTrigger({ + const progressReconciliation = createCoalescedAsyncTrigger({ run: async () => { - const inserted = await input.projectRunMilestones(); + const inserted = await input.projectNativeProgress(); // Existing final/question publications never wait on this optional - // projection. Newly committed milestones get a bounded dispatch wake; + // projection. Newly committed progress gets a bounded dispatch wake; // an empty/contended pass does not create a self-sustaining loop. if (inserted > 0) publicationReconciliation.notify(); }, - onError: (error) => input.onError("run milestones", error), + onError: (error) => input.onError("native progress", error), }); const start = ( lane: ChatReconciliationLane, @@ -382,24 +383,24 @@ export function createChatReconciliationCoordinator(input: { input.processFailedGitHubWebhookDeliveries, ); } - milestoneReconciliation.poll(); + progressReconciliation.poll(); publicationReconciliation.poll(); if (input.processPendingSlackFileUploadReceipts) start("Slack file receipts", input.processPendingSlackFileUploadReceipts); if (input.processPendingSlackSessionSyncs) start("Slack session status", input.processPendingSlackSessionSyncs); }, notifyPublications() { - milestoneReconciliation.notify(); + progressReconciliation.notify(); publicationReconciliation.notify(); }, stop() { stopped = true; - milestoneReconciliation.stop(); + progressReconciliation.stop(); publicationReconciliation.stop(); }, async drain() { await Promise.allSettled([ ...inFlight.values(), - milestoneReconciliation.drain(), + progressReconciliation.drain(), ]); // Projecting the final batch can notify dispatch after an earlier drain // would have returned. Join dispatch only after its producer has drained. @@ -1304,6 +1305,9 @@ export async function createApp( chatChannels.registerTeamsConsentExpiryWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive()); chatChannels.registerTeamsAttemptRecoveryWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive()); chatChannels.registerCommandWork(deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive()); + chatRunMilestoneWork(db, opts.authPublicBaseUrl).register( + deliveryWork, () => !isWarmStandby() && !isIdleTaskDrainActive(), + ); emailChannels.start(); const reconcileChatPublicationMaintenance = async () => { await chatChannels.processPublicationMaintenance(); @@ -1314,10 +1318,7 @@ export async function createApp( processPendingDeliveries: async () => undefined, processFailedGitHubWebhookDeliveries: () => chatChannels.processFailedGitHubWebhookDeliveries(), - projectRunMilestones: () => - enqueueChatRunMilestones(db, { - publicBaseUrl: opts.authPublicBaseUrl, - }), + projectNativeProgress: () => enqueueSafeNativeChatProgress(db), flushPublications: () => reconcileChatPublicationMaintenance(), onError: (lane, err) => { logger.error({ err, lane }, `Failed to reconcile chat ${lane}`); @@ -1329,7 +1330,8 @@ export async function createApp( chatReconciliation.notifyPublications(); }, ); - // Provider/action maintenance and milestone projection still use this cadence. + // Native progress and remaining publication maintenance still use this cadence. + // Lifecycle milestones use committed changes and their own recovery worker. // Inbox, outbox, and file receipt queues have independent commit/deadline wakes. let chatPublicationTimer: ReturnType | null = setInterval( () => { diff --git a/server/src/board-claim.ts b/server/src/board-claim.ts index c4784406646..275dfdf0180 100644 --- a/server/src/board-claim.ts +++ b/server/src/board-claim.ts @@ -4,6 +4,7 @@ import type { Db } from "@paperclipai/db"; import { companies, companyMemberships, instanceUserRoles } from "@paperclipai/db"; import type { DeploymentMode } from "@paperclipai/shared"; import { ensureHumanRoleDefaultGrants } from "./services/principal-access-compatibility.js"; +import { notifyChatRunMilestones } from "./services/chat-run-milestone-notifications.js"; const LOCAL_BOARD_USER_ID = "local-board"; const CLAIM_TTL_MS = 1000 * 60 * 60 * 24; @@ -124,6 +125,7 @@ export async function claimBoardOwnership( .then((rows) => rows[0] ?? null); if (!existing) { + await notifyChatRunMilestones(tx, { companyId: company.id }); await tx.insert(companyMemberships).values({ companyId: company.id, principalType: "user", @@ -135,6 +137,7 @@ export async function claimBoardOwnership( } if (existing.status !== "active") { + await notifyChatRunMilestones(tx, { companyId: company.id }); await tx .update(companyMemberships) .set({ status: "active", membershipRole: "owner", updatedAt: new Date() }) diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index 069ecc5d221..aa9e3c13a37 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -30,6 +30,7 @@ import { captureRunIdentity } from "../services/run-identity.js"; import { boardAuthService } from "../services/board-auth.js"; import { retryIdempotentDatabaseOperation } from "../database-retry.js"; import { beginIdleTrackedWork } from "../services/task-admission.js"; +import { notifyChatRunMilestones } from "../services/chat-run-milestone-notifications.js"; export { isTransientDbConnectionError, @@ -652,19 +653,34 @@ async function resolveCloudTenantActorOnce( // exact legacy-marker guards and the operator's own edits. await repairCloudTenantCompanyProvisionDefaults(db, { companyId, stackId, now }); - effectiveMembership = await db - .insert(companyMemberships) - .values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole, updatedAt: now }) - .onConflictDoUpdate({ - target: [companyMemberships.companyId, companyMemberships.principalType, companyMemberships.principalId], - set: { status: "active", membershipRole, updatedAt: now }, - setWhere: sql`${companyMemberships.status} is distinct from 'active' - or ${companyMemberships.membershipRole} is distinct from ${membershipRole}`, - }) - .returning({ companyId: companyMemberships.companyId, membershipRole: companyMemberships.membershipRole, status: companyMemberships.status }) - // PostgreSQL returns no row for an unchanged ON CONFLICT ... WHERE. - // That condition itself proves the existing role/status match the input. - .then((rows) => rows[0] ?? { companyId, membershipRole, status: "active" }); + effectiveMembership = await db.transaction(async tx => { + const [existing] = await tx.select({ + status: companyMemberships.status, + membershipRole: companyMemberships.membershipRole, + }).from(companyMemberships).where(and( + eq(companyMemberships.companyId, companyId), + eq(companyMemberships.principalType, "user"), + eq(companyMemberships.principalId, userId), + )).for("update"); + // Serialize authority restoration and its notice while retaining the + // guarded upsert that leaves unchanged memberships untouched. + if (!existing || existing.status !== "active" || existing.membershipRole === "viewer") { + await notifyChatRunMilestones(tx, { companyId }); + } + return tx + .insert(companyMemberships) + .values({ companyId, principalType: "user", principalId: userId, status: "active", membershipRole, updatedAt: now }) + .onConflictDoUpdate({ + target: [companyMemberships.companyId, companyMemberships.principalType, companyMemberships.principalId], + set: { status: "active", membershipRole, updatedAt: now }, + setWhere: sql`${companyMemberships.status} is distinct from 'active' + or ${companyMemberships.membershipRole} is distinct from ${membershipRole}`, + }) + .returning({ companyId: companyMemberships.companyId, membershipRole: companyMemberships.membershipRole, status: companyMemberships.status }) + // PostgreSQL returns no row for an unchanged ON CONFLICT ... WHERE. + // That condition itself proves the existing role/status match the input. + .then((rows) => rows[0] ?? { companyId, membershipRole, status: "active" }); + }); // An unchanged membership can still lack default grants after a partial // sync or an older deployment; keep this repair inside every actual sync. diff --git a/server/src/middleware/cloud-tenant-actor.test.ts b/server/src/middleware/cloud-tenant-actor.test.ts index 87fd4ae8627..4f9e12c97ad 100644 --- a/server/src/middleware/cloud-tenant-actor.test.ts +++ b/server/src/middleware/cloud-tenant-actor.test.ts @@ -46,7 +46,7 @@ function createFakeDb(options: { : options.settingsRow; const insertedTables: unknown[] = []; const deletedTables: unknown[] = []; - const selectWheres: Array<{ table: unknown; condition: unknown }> = []; + const selectWheres: Array<{ table: unknown; condition: unknown; locked?: boolean }> = []; function writeChain(table: unknown) { let values: unknown; const chain: Record = {}; @@ -63,6 +63,7 @@ function createFakeDb(options: { return chain; } const db = { + transaction: async (work: (tx: Db) => Promise) => work(db), insert: (table: unknown) => { insertedTables.push(table); return writeChain(table); @@ -75,21 +76,32 @@ function createFakeDb(options: { return { from: (table: unknown) => ({ where: (condition: unknown) => { - // Role-default grant seeding reads existing grants first; keep that - // read working so selectThrows exercises only the degradable reads. - if (options.selectThrows && table !== principalPermissionGrants) { + // Only settings lookup is degradable. Membership locking and + // permission seeding must still work in the fail-closed flag test. + if (options.selectThrows && table === instanceSettings) { throw new Error("select unavailable"); } - selectWheres.push({ table, condition }); + const selected: (typeof selectWheres)[number] = { table, condition }; + selectWheres.push(selected); const rows = table === instanceSettings && settingsRow ? [settingsRow] : table === companyMemberships ? (options.membershipQueryRows ?? []) : []; - return { - then: (resolve: (v: unknown) => unknown) => Promise.resolve(rows).then(resolve), + let locked = false; + const result = { + for: () => { locked = true; selected.locked = true; return result; }, + then: (resolve: (v: unknown) => unknown, reject?: (error: unknown) => unknown) => { + // The optional cross-company read can fail independently of + // the primary membership transaction, which must succeed. + const query = options.selectThrows && table === companyMemberships && !locked + ? Promise.reject(new Error("select unavailable")) + : Promise.resolve(rows); + return query.then(resolve, reject); + }, }; + return result; }, }), }; @@ -258,7 +270,8 @@ describe("resolveCloudTenantActor (shared-pool hardening)", () => { expect(insertedTables.filter(table => table === companyMemberships)).toHaveLength(1); expect(deletedTables.filter(table => table === instanceUserRoles)).toHaveLength(20); expect(selectWheres.filter(entry => entry.table === instanceSettings)).toHaveLength(20); - expect(selectWheres.filter(entry => entry.table === companyMemberships)).toHaveLength(20); + expect(selectWheres.filter(entry => entry.table === companyMemberships && !entry.locked)).toHaveLength(20); + expect(selectWheres.filter(entry => entry.table === companyMemberships && entry.locked)).toHaveLength(1); expect(actors.every(actor => actor?.memberships?.[0]?.membershipRole === "owner")).toBe(true); }); @@ -461,11 +474,13 @@ describe("resolveCloudTenantActor (shared-pool hardening)", () => { const { db, selectWheres } = createFakeDb(); await resolveCloudTenantActor(db, fakeReq(VALID_HEADERS)); const membershipSelects = selectWheres.filter((entry) => entry.table === companyMemberships); - expect(membershipSelects).toHaveLength(1); + expect(membershipSelects).toHaveLength(2); + // The first read locks the primary membership before its upsert; the + // second loads the user's active memberships for request authorization. // Rows for other users and rows in any non-active status are excluded // by the query itself: the filter binds exactly this user id and the // "active" status — the same condition the session path applies. - expect(membershipSelects[0]!.condition).toEqual( + expect(membershipSelects[1]!.condition).toEqual( and( eq(companyMemberships.principalType, "user"), eq(companyMemberships.principalId, "user-123"), diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index e45cd770bba..558b353534e 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -26,6 +26,7 @@ import { budgetService, budgetServiceInTransaction, recheckAdmittedRunBudgetRead import { lockRunBudgetRecoveryOwnerInTransaction, recheckRunBudgetReservationInTransaction } from "../../../services/budget-reservations.js"; import { isCompletedOnboardingHandoffWake } from "../../../services/chat-completion-delivery.js"; import { isHeartbeatWakeOnDemandEnabled } from "../../../services/heartbeat-policy.js"; +import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { connectionContinuationPendingResponse } from "../../../services/satisfied-connection-intents.js"; import { collectDispositionRepairSourceState } from "../../../services/recovery/disposition-repair.js"; @@ -660,10 +661,12 @@ export function createPostgresRunDispatchAdapter( async function promoteDueRetryInTx( tx: Db, input: PromoteOrCancelDueRetryInput, + issueId: string | null, ): Promise< | { applied: true; run: HeartbeatRun; postCommitEffects: PostCommitEffect[] } | { applied: false } > { + await notifyChatRunMilestones(tx, { companyId: input.companyId, issueId: issueId ?? undefined }); const [row] = await tx .update(heartbeatRuns) .set({ @@ -705,6 +708,7 @@ export function createPostgresRunDispatchAdapter( tx: Db, input: CancelSuppressedRetryInput, ): Promise { + await notifyChatRunMilestones(tx, { companyId: input.companyId, issueId: input.issueId ?? undefined }); const [row] = await tx .update(heartbeatRuns) .set({ @@ -873,7 +877,7 @@ export function createPostgresRunDispatchAdapter( runId: input.runId, companyId: input.companyId, now, - }); + }, issueId); return promoted.applied ? { outcome: { @@ -911,6 +915,7 @@ export function createPostgresRunDispatchAdapter( providerDispatchNotStarted = false, ): Promise { await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [row] = await tx .update(heartbeatRuns) .set({ diff --git a/server/src/modules/wake-queue/adapters/postgres.ts b/server/src/modules/wake-queue/adapters/postgres.ts index cd2f1aa7a48..90c24c75a57 100644 --- a/server/src/modules/wake-queue/adapters/postgres.ts +++ b/server/src/modules/wake-queue/adapters/postgres.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js"; import { isAcknowledgedNativeReassignmentStop, isAcknowledgedNativeStop } from "../../../services/acknowledged-native-stop.js"; import { readRunCancellation } from "../../../services/run-cancellation.js"; @@ -472,6 +473,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru }, async finalizePromotedWake(input) { + await notifyChatRunMilestones(tx, { companyId: input.companyId, issueId: input.issue.id }); const newRun = await tx .insert(heartbeatRuns) .values({ @@ -688,6 +690,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru sessionBefore, now, }) { + await notifyChatRunMilestones(tx, { companyId, issueId: issue.id }); const wakeupRequest = await tx .insert(agentWakeupRequests) .values({ @@ -949,6 +952,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter { async coalesceIntoActiveExecutionRun(scope, input) { const tx = requireAdmissionTx(scope, input.companyId); const now = new Date(); + await notifyChatRunMilestones(tx, { companyId: input.companyId }); const mergedRun = await tx .update(heartbeatRuns) .set({ contextSnapshot: input.mergedContextSnapshot, updatedAt: now }) diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.test.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.test.ts index d8e30032d73..82741d5d548 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.test.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.test.ts @@ -7,6 +7,7 @@ import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "../../../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; import { createQueuedCommentIssueLockWriter } from "./queued-comment-postgres.js"; import type { QueuedCommentQueuePostgresAdapterDeps } from "./queued-comment-postgres.js"; import { QueuedCommentMutationError } from "../application/queued-comment-use-cases.js"; @@ -19,7 +20,8 @@ import { QueuedCommentMutationError } from "../application/queued-comment-use-ca // leaving a partial write. The decision branching itself is proven against // plain facts in `domain/policy.test.ts`; the use-case orchestration is // proven against a mocked port in `application/queued-comment-use-cases.test.ts`. -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_QUEUED_COMMENT_TEST_DATABASE_URL"); +const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; if (!embeddedPostgresSupport.supported) { @@ -39,8 +41,8 @@ describeEmbeddedPostgres("queued-comment postgres adapter", () => { }; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-queued-comment-postgres-adapter-"); - db = createDb(tempDb.connectionString); + if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-queued-comment-postgres-adapter-"); + db = createDb(external ?? tempDb!.connectionString); }, 20_000); afterEach(async () => { @@ -56,6 +58,7 @@ describeEmbeddedPostgres("queued-comment postgres adapter", () => { }); afterAll(async () => { + await db?.$client.end(); await tempDb?.cleanup(); }); diff --git a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts index 2588636e4bc..e2c0e2f740f 100644 --- a/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts +++ b/server/src/modules/wake-queue/adapters/queued-comment-postgres.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../../../services/chat-run-milestone-notifications.js"; import { and, eq, inArray } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { agentWakeupRequests, agents, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; @@ -45,7 +46,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = { syncCommentExternalObjectsSafely(commentId: string, tx: Db): Promise; }; -function buildTransaction(tx: Db, companyId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { +function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction { return { async updateCommentBody({ issueId, commentId, body, updatedAt }) { const updated = await tx @@ -72,6 +73,7 @@ function buildTransaction(tx: Db, companyId: string, deps: QueuedCommentQueuePos }, async updateQueueRunCommentIds({ queueRunId, contextSnapshot, ids, updatedAt }) { + await notifyChatRunMilestones(tx, { companyId, issueId }); const row = await tx .update(heartbeatRuns) .set({ contextSnapshot: withQueuedCommentIdsInRunContext(contextSnapshot, ids), updatedAt }) @@ -82,6 +84,7 @@ function buildTransaction(tx: Db, companyId: string, deps: QueuedCommentQueuePos }, async deleteComment({ issueId, commentId }) { + await notifyChatRunMilestones(tx, { companyId, issueId }); const row = await tx .delete(issueComments) .where(and(eq(issueComments.id, commentId), eq(issueComments.issueId, issueId), eq(issueComments.companyId, companyId))) @@ -98,6 +101,7 @@ function buildTransaction(tx: Db, companyId: string, deps: QueuedCommentQueuePos }, async cancelQueueRun({ queueRunId, reason, now }) { + await notifyChatRunMilestones(tx, { companyId, issueId }); const row = await tx .update(heartbeatRuns) .set({ status: "cancelled", finishedAt: now, error: reason, errorCode: "queued_comment_discarded", updatedAt: now }) @@ -291,7 +295,7 @@ export function createQueuedCommentIssueLockWriter(db: Db, deps: QueuedCommentQu .then((rows) => rows[0] ?? null) : null; - const transaction = buildTransaction(tx, companyId, deps); + const transaction = buildTransaction(tx, companyId, input.issue.id, deps); const wake = toWakeRow(wakeRow); const queueRun = queueRunRow ? toRunRow(queueRunRow) : null; const activeRun = activeRunRow ? toRunRow(activeRunRow) : null; diff --git a/server/src/services/access.ts b/server/src/services/access.ts index 4c7090fc1d0..42af549b480 100644 --- a/server/src/services/access.ts +++ b/server/src/services/access.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyVoiceChange } from "./voice/voice-recovery-notifications.js"; import { notifyChatEndpointWork } from "./chat-work-notifications.js"; import { notifyToolConnectionHealth } from "./tool-connection-health-notifications.js"; @@ -328,7 +329,7 @@ export function accessService(db: Db) { && grant.credentialSecretRefs.length > 0 && grant.credentialSecretRefs.every((ref) => !connectionSecretIdsToRemove.has(ref.secretId)), ); - if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id); + if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id, companyId); if (connection.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: connection.id }); if (!hasUnaffectedActiveGrant) await notifyToolConnectionHealth(tx, { companyId, id: connection.id }); await tx.update(toolConnections).set({ @@ -554,6 +555,11 @@ export function accessService(db: Db) { data.membershipRole !== undefined ? data.membershipRole : existing.membershipRole; const nextStatus = data.status ?? existing.status; + if (existing.principalType === "user" && nextStatus === "active" && nextMembershipRole !== "viewer" + && (existing.status !== "active" || existing.membershipRole === "viewer")) { + await notifyChatRunMilestones(tx, { companyId }); + } + if ( existing.principalType === "user" && existing.status === "active" && @@ -886,6 +892,7 @@ export function accessService(db: Db) { const existingMembership = existingByCompany.get(companyId); if (existingMembership) { if (existingMembership.status !== "active") { + if (existingMembership.membershipRole !== "viewer") await notifyChatRunMilestones(tx, { companyId }); await tx .update(companyMemberships) .set({ @@ -897,6 +904,7 @@ export function accessService(db: Db) { } continue; } + await notifyChatRunMilestones(tx, { companyId }); await tx.insert(companyMemberships).values({ companyId, principalType: "user", @@ -917,31 +925,43 @@ export function accessService(db: Db) { membershipRole: string | null = "member", status: "pending" | "active" | "suspended" = "active", ) { - const existing = await getMembership(companyId, principalType, principalId); - if (existing) { - if (existing.status !== status || existing.membershipRole !== membershipRole) { - const updated = await db - .update(companyMemberships) - .set({ status, membershipRole, updatedAt: new Date() }) - .where(eq(companyMemberships.id, existing.id)) - .returning() - .then((rows) => rows[0] ?? null); - return updated ?? existing; + return db.transaction(async tx => { + const existing = await tx.select().from(companyMemberships).where(and( + eq(companyMemberships.companyId, companyId), eq(companyMemberships.principalType, principalType), + eq(companyMemberships.principalId, principalId), + )).for("update").then(rows => rows[0] ?? null); + if (existing) { + if (existing.status !== status || existing.membershipRole !== membershipRole) { + if (principalType === "user" && status === "active" && membershipRole !== "viewer" + && (existing.status !== "active" || existing.membershipRole === "viewer")) { + await notifyChatRunMilestones(tx, { companyId }); + } + const updated = await tx + .update(companyMemberships) + .set({ status, membershipRole, updatedAt: new Date() }) + .where(eq(companyMemberships.id, existing.id)) + .returning() + .then((rows) => rows[0] ?? null); + return updated ?? existing; + } + return existing; } - return existing; - } - return db - .insert(companyMemberships) - .values({ - companyId, - principalType, - principalId, - status, - membershipRole, - }) - .returning() - .then((rows) => rows[0]); + if (principalType === "user" && status === "active" && membershipRole !== "viewer") { + await notifyChatRunMilestones(tx, { companyId }); + } + return tx + .insert(companyMemberships) + .values({ + companyId, + principalType, + principalId, + status, + membershipRole, + }) + .returning() + .then((rows) => rows[0]); + }); } async function setPrincipalGrants( @@ -1122,6 +1142,11 @@ export function accessService(db: Db) { data.membershipRole !== undefined ? data.membershipRole : existing.membershipRole; const nextStatus = data.status ?? existing.status; + if (existing.principalType === "user" && nextStatus === "active" && nextMembershipRole !== "viewer" + && (existing.status !== "active" || existing.membershipRole === "viewer")) { + await notifyChatRunMilestones(tx, { companyId }); + } + if ( existing.principalType === "user" && existing.status === "active" && diff --git a/server/src/services/agent-conversations.ts b/server/src/services/agent-conversations.ts index 40ce286d1f9..dce09c886b9 100644 --- a/server/src/services/agent-conversations.ts +++ b/server/src/services/agent-conversations.ts @@ -17,6 +17,7 @@ import { type Db, } from "@paperclipai/db"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { sanitizeQuarantinedCommentForHigherTrust } from "./source-trust.js"; export type ConversationIdentity = { @@ -166,6 +167,7 @@ export async function prepareConversationTurn( } // The boundary lives on the command comment. A crash/retry reuses it instead of resetting twice. if (reset && comment && comment.conversationSessionGeneration == null) { + await notifyChatRunMilestones(tx, { companyId: issue.companyId, issueId: issue.id }); generation += 1; await tx .update(issues) diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index 009193f7cfc..8589877dce4 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -1,6 +1,7 @@ import { leasedChatProvider } from "./chat-channels/runtime/receiver-policy.js"; import { chatReceiverSchedule } from "./chat-channels/runtime/receiver-schedule.js"; import { registerChatReceiverWork } from "./chat-channels/runtime/receiver-work.js"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyChatEndpointConfiguration } from "./chat-channels/configuration-work-notifications.js"; import { registerDiscordCommandWork } from "./chat-channels/discord/command-work.js"; import { discordCommandSchedule } from "./chat-channels/discord/command-schedule.js"; @@ -2121,6 +2122,9 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { taskUserId: string | null, ) => { let conversation = existingConversation; + if (conversation) { + await notifyChatRunMilestones(taskTx, { companyId: taskEndpoint.companyId, issueId: conversation.issueId }); + } if (!conversation) { const [taskResource] = await taskTx.select({ executionDefaults: chatEndpointResources.executionDefaults }) .from(chatEndpointResources).where(and(eq(chatEndpointResources.id, resource.id), eq(chatEndpointResources.endpointId, taskEndpoint.id))); @@ -2162,6 +2166,7 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { }, taskTx, ); + await notifyChatRunMilestones(taskTx, { companyId: taskEndpoint.companyId, issueId: issue.id }); await taskTx .insert(chatConversations) .values({ diff --git a/server/src/services/chat-channels/configuration.ts b/server/src/services/chat-channels/configuration.ts index b7384cc0a4a..fa1b1d5b1ce 100644 --- a/server/src/services/chat-channels/configuration.ts +++ b/server/src/services/chat-channels/configuration.ts @@ -196,7 +196,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_not_active", }); } - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) .set({ @@ -334,7 +334,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption inArray(chatDeliveries.state, ["received", "retry"]), ), ); - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); [activatedEndpoint] = await tx .update(chatEndpoints) .set({ @@ -381,9 +381,9 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption const failure = redactError(error); const failedAt = new Date(); await managementContext?.reauthorize(); - await db.transaction(async (tx) => { + await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) .set({ @@ -487,7 +487,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption code: "chat_endpoint_already_removed", }); } - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) .set({ @@ -721,7 +721,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); const claimed = await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); const rows = await tx .update(chatEndpoints) .set({ @@ -800,7 +800,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await installSpekoVoiceTools(tx, endpoint, actorUserId); } const preserveSlackVerification = preservesProvisionedSlackVerification(current.setup, credentials.signingSecret ?? "", `${getWebhookPublicBaseUrl()}/api/chat-webhooks/${endpoint.publicId}/slack`); - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) .set({ @@ -887,7 +887,7 @@ export function createChatConfiguration(db: Db, options: ChatConfigurationOption await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); await tx .update(chatEndpoints) .set({ diff --git a/server/src/services/chat-channels/credentials.ts b/server/src/services/chat-channels/credentials.ts index 3d20be11237..e2ed3e1ce2c 100644 --- a/server/src/services/chat-channels/credentials.ts +++ b/server/src/services/chat-channels/credentials.ts @@ -383,7 +383,7 @@ export function createChatCredentials(db: Db, options: ChatCredentialsOptions) { .set({ credentialSecretRefs: refs, updatedAt: new Date() }) .where(eq(toolConnections.id, endpoint.connectionId)); if (changedFields.length) { - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); const [currentEndpoint] = await tx.select({ setup: chatEndpoints.setup }).from(chatEndpoints) .where(and(eq(chatEndpoints.id, endpoint.id), eq(chatEndpoints.companyId, endpoint.companyId))).for("update"); if (!currentEndpoint) throw notFound("Chat endpoint not found"); diff --git a/server/src/services/chat-channels/discord/gateway-events.ts b/server/src/services/chat-channels/discord/gateway-events.ts index 3ed17aa6aa3..f59e1c709ed 100644 --- a/server/src/services/chat-channels/discord/gateway-events.ts +++ b/server/src/services/chat-channels/discord/gateway-events.ts @@ -165,7 +165,7 @@ export function createDiscordGatewayEvents(db: Db, options: DiscordGatewayEvents current.setup.step === "complete" ? "active" : "verifying"; await notifyChatReceiver(tx, current); await notifyDiscordCommandRegistration(tx, { kind: "endpoint", id: current.id }); - if (status !== current.status) await notifyChatEndpointWork(tx, current.connectionId); + if (status !== current.status) await notifyChatEndpointWork(tx, current.connectionId, current.companyId); await tx .update(chatEndpoints) .set({ diff --git a/server/src/services/chat-channels/endpoints.ts b/server/src/services/chat-channels/endpoints.ts index 581cc356f92..86b788cea74 100644 --- a/server/src/services/chat-channels/endpoints.ts +++ b/server/src/services/chat-channels/endpoints.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { chatConnectionTransport } from "./provider-catalog.js"; import type { ConnectionManagementContext } from "../connection-management.js"; import { slackSetupState } from "./slack/setup.js"; @@ -454,6 +455,11 @@ function providerSetupState( await managementContext?.reauthorize(); await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); + if ((input.allowDirectMessages === true && !existing.endpoint.allowDirectMessages) + || (input.allowGroupChats === true && !existing.endpoint.allowGroupChats) + || (input.allowUnlinkedPeople === true && !existing.endpoint.allowUnlinkedPeople)) { + await notifyChatRunMilestones(tx, { companyId: existing.endpoint.companyId }); + } if (input.executionDefaults !== undefined) { await assertChatExecutionDefaultsAccess(db, { companyId: existing.endpoint.companyId, diff --git a/server/src/services/chat-channels/github/resources.ts b/server/src/services/chat-channels/github/resources.ts index 5abe2384cd2..f8599487835 100644 --- a/server/src/services/chat-channels/github/resources.ts +++ b/server/src/services/chat-channels/github/resources.ts @@ -201,11 +201,12 @@ export async function reconcileGitHubRepositoryCoordinates(tx: DbTransaction, en type GitHubResourceOptions = { endpointRecord: (endpointId: string) => Promise<{ endpoint: EndpointRow } | null>; upsertProviderResourceRow: (tx: DbTransaction, endpoint: EndpointRow, item: ChatProviderResourceInventoryItem) => Promise<{ id: string } | null>; + restoreResourceConversations: (tx: DbTransaction, endpoint: EndpointRow, resourceId: string) => Promise; updateResourceSelection: (endpointId: string, updates: Array<{ id: string; enabled: boolean }>, actorUserId?: string | null, options?: { allGitHubRepositoriesEnabled?: boolean }) => Promise; }; export function createGitHubResources(db: Db, options: GitHubResourceOptions) { - const { endpointRecord, upsertProviderResourceRow, updateResourceSelection } = options; + const { endpointRecord, upsertProviderResourceRow, restoreResourceConversations, updateResourceSelection } = options; async function reconcileGitHubWebhookRepository( endpoint: EndpointRow, payload: unknown, @@ -220,17 +221,7 @@ export function createGitHubResources(db: Db, options: GitHubResourceOptions) { // A correctly signed repository callback is current provider proof. Keep // Paperclip's enabled choice, and reopen only conversations previously // quarantined for provider availability loss. - await tx - .update(chatConversations) - .set({ state: "active", updatedAt: new Date() }) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.resourceId, resource.id), - eq(chatConversations.state, "unavailable"), - ), - ); + await restoreResourceConversations(tx, endpoint, resource.id); await credentialLease.assertOwned(tx); }); } diff --git a/server/src/services/chat-channels/identity-links.ts b/server/src/services/chat-channels/identity-links.ts index b45ee22db66..abbd2340c94 100644 --- a/server/src/services/chat-channels/identity-links.ts +++ b/server/src/services/chat-channels/identity-links.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { githubIdentityFields } from "./github/identity-links.js"; import { createSlackIdentityLinks, type SlackIdentityEffectStager } from "./slack/identity-links.js"; import { createHash, randomBytes } from "node:crypto"; @@ -319,6 +320,7 @@ export function createChatIdentityLinks(db: Db, options: ChatIdentityLinkOptions }, ); } + await notifyChatRunMilestones(tx, { companyId: currentLink.companyId }); const confirmedLink = await tx .update(chatIdentityLinks) .set({ diff --git a/server/src/services/chat-channels/imessage-photon/runtime-callbacks.ts b/server/src/services/chat-channels/imessage-photon/runtime-callbacks.ts index 7b7e5557fae..414e070a844 100644 --- a/server/src/services/chat-channels/imessage-photon/runtime-callbacks.ts +++ b/server/src/services/chat-channels/imessage-photon/runtime-callbacks.ts @@ -32,7 +32,7 @@ export function createPhotonRuntimeCallbacks(db: Db, options: PhotonRuntimeOptio if (!current) return; const connection = await tx.select().from(toolConnections).where(eq(toolConnections.id, current.connectionId)).then((rows) => rows[0]); if (!connection?.enabled || connection.healthStatus === "failed") throw new Error("Photon endpoint requires operator recovery"); - await notifyChatEndpointWork(tx, connectionId); + await notifyChatEndpointWork(tx, connectionId, companyId); await tx.update(chatEndpoints).set({ status: current.setup.step === "complete" ? "active" : "verifying", healthMessage: "Photon receiver connected", lastError: null, updatedAt: new Date() }).where(eq(chatEndpoints.id, endpointId)); }); }, diff --git a/server/src/services/chat-channels/inbound/wakeups.ts b/server/src/services/chat-channels/inbound/wakeups.ts index 9b9f9738310..2f7b52d2994 100644 --- a/server/src/services/chat-channels/inbound/wakeups.ts +++ b/server/src/services/chat-channels/inbound/wakeups.ts @@ -1,5 +1,6 @@ import { DELIVERY_PROCESSING_STALE_MS } from "./delivery-ledger.js"; import type { EndpointRow, DbOrTransaction } from "../runtime-types.js"; +import { notifyChatRunMilestones } from "../../chat-run-milestone-notifications.js"; import { notifyChatDeliveryWork } from "../../chat-work-notifications.js"; import { enqueueFastResponse } from "../../fast-responses.js"; import { and, eq, sql } from "drizzle-orm"; @@ -110,6 +111,9 @@ export function createInboundWakeups(db: Db, options: WakeupOptions) { ); } await notifyChatDeliveryWork(tx); + if (action.status === "processed") { + await notifyChatRunMilestones(tx, { companyId: action.companyId, issueId: String(action.payload.issueId) }); + } const acceptedAt = new Date(); const accepted = await tx .update(chatDeliveries) @@ -249,6 +253,9 @@ export function createInboundWakeups(db: Db, options: WakeupOptions) { const settle = async (status: string, result: Record) => { const changed = await db.transaction(async (tx) => { await notifyChatDeliveryWork(tx); + if (status === "processed") { + await notifyChatRunMilestones(tx, { companyId: claimed.companyId, issueId: request!.issueId }); + } const changed = await tx .update(chatActions) .set({ status, result, updatedAt: new Date() }) diff --git a/server/src/services/chat-channels/interactions/callbacks.ts b/server/src/services/chat-channels/interactions/callbacks.ts index d2723672dc7..333b3df379c 100644 --- a/server/src/services/chat-channels/interactions/callbacks.ts +++ b/server/src/services/chat-channels/interactions/callbacks.ts @@ -787,7 +787,7 @@ export function createChatInteractionCallbacks(db: Db, options: InteractionCallb .then((rows) => rows[0] ?? null); if (occupied && occupied.publicationId !== current.id) return null; const reconciledAt = new Date(); - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, current.state === "delivery_unknown" ? current : undefined); const [reconciled] = await tx .update(chatPublications) .set({ diff --git a/server/src/services/chat-channels/publications/failures.ts b/server/src/services/chat-channels/publications/failures.ts index b5f6648b646..8a4c8f7ed7a 100644 --- a/server/src/services/chat-channels/publications/failures.ts +++ b/server/src/services/chat-channels/publications/failures.ts @@ -120,7 +120,7 @@ export function createChatPublicationFailures(db: Db, options: PublicationFailur : disposition.kind === "resource_unavailable" ? "cancelled" : "failed"; - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, state === "failed" || state === "cancelled" ? publication : undefined); const [ownedPublication] = await tx .update(chatPublications) .set({ diff --git a/server/src/services/chat-channels/publications/resolution.ts b/server/src/services/chat-channels/publications/resolution.ts index c48c4fed129..74bc9ff9f0c 100644 --- a/server/src/services/chat-channels/publications/resolution.ts +++ b/server/src/services/chat-channels/publications/resolution.ts @@ -218,7 +218,7 @@ export function createChatPublicationResolution(db: Db, options: ResolutionOptio action === "retry_anyway" ) { await recordPhotonPublicationRetry(tx, endpointId, publication, userId); } const now = new Date(); - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, action === "retry_anyway" ? undefined : publication); await notifyVoicePublication(tx, initialRecord.endpoint, publication.conversationId); await tx .update(chatPublications) diff --git a/server/src/services/chat-channels/publications/worker.ts b/server/src/services/chat-channels/publications/worker.ts index 0987b9fb612..9f4f67c39ca 100644 --- a/server/src/services/chat-channels/publications/worker.ts +++ b/server/src/services/chat-channels/publications/worker.ts @@ -154,7 +154,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp ? classified.retryAfterMs : Math.min(60_000, 2 ** Math.max(0, attempts) * 1_000); const result = await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, terminal ? publication : undefined); const [settled] = await tx .update(chatPublications) .set({ @@ -380,7 +380,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp } if (explicitSlackDelivery === "delivered") { await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx.update(chatPublications).set({ state: "cancelled", redactedError: "Identical response already delivered by Slack tool", updatedAt: new Date() }).where(and(eq(chatPublications.id, publication.id), inArray(chatPublications.state, ["pending", "retry"]))) ; }); @@ -507,7 +507,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp ) { await publicationLease.assertOwned(); await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx .update(chatPublications) .set({ @@ -531,7 +531,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp if (!endpoint.allowDirectMessages) { await publicationLease.assertOwned(); await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx .update(chatPublications) .set({ @@ -566,7 +566,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp if (!nonDirectDestinationAllowed(endpoint, resource)) { await publicationLease.assertOwned(); await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx .update(chatPublications) .set({ @@ -605,7 +605,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp if (!currentInteraction || currentInteraction.status !== "pending") { await publicationLease.assertOwned(); await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx .update(chatPublications) .set({ @@ -638,7 +638,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp if (progressSupersessionReason) { await publicationLease.assertOwned(); await db.transaction(async (tx) => { - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); return tx .update(chatPublications) .set({ @@ -710,7 +710,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp if (!authorizationClaim) { await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); await tx .update(chatPublications) .set({ @@ -858,7 +858,7 @@ export function createChatPublicationWorker(db: Db, options: PublicationWorkerOp await db.transaction(async (tx) => { await credentialLease.assertOwned(tx); const committedAt = new Date(); - await notifyChatPublicationWork(tx); + await notifyChatPublicationWork(tx, publication); await notifyVoicePublication(tx, authorizationClaim.endpoint, publication.conversationId); const [completedPublication] = await tx .update(chatPublications) diff --git a/server/src/services/chat-channels/resource-reconciliation.ts b/server/src/services/chat-channels/resource-reconciliation.ts index b189c1cc168..5cf5f570266 100644 --- a/server/src/services/chat-channels/resource-reconciliation.ts +++ b/server/src/services/chat-channels/resource-reconciliation.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { notifyChatEndpointConfiguration } from "./configuration-work-notifications.js"; import { notifyChatConnectionHealth } from "./connection-health-notifications.js"; import type { ConnectionManagementContext } from "../connection-management.js"; @@ -80,6 +81,32 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc return resource ?? null; } + async function setResourceConversationAvailability( + tx: DbTransaction, + endpoint: Pick, + resourceId: string, + available: boolean, + now: Date, + ) { + const affected = and( + eq(chatConversations.companyId, endpoint.companyId), + eq(chatConversations.endpointId, endpoint.id), + eq(chatConversations.resourceId, resourceId), + available ? eq(chatConversations.state, "unavailable") : inArray(chatConversations.state, ["active", "waiting"]), + ); + if (available) { + // Only a reopened binding creates projection work. Repeated inventory + // refreshes and completed history must not wake the milestone scanner. + const reopened = await tx.select({ issueId: chatConversations.issueId }) + .from(chatConversations).where(affected).for("update"); + for (const issueId of new Set(reopened.map(row => row.issueId))) { + await notifyChatRunMilestones(tx, { companyId: endpoint.companyId, issueId }); + } + } + await tx.update(chatConversations) + .set({ state: available ? "active" : "unavailable", updatedAt: now }).where(affected); + } + async function reconcileProviderResourceRows( endpoint: EndpointRow, inventory: ChatProviderInventoryResult, @@ -99,17 +126,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc if (resource) { // Resource recovery reopens only bindings that Paperclip marked // unavailable. Completed historical tasks stay completed. - await tx - .update(chatConversations) - .set({ state: "active", updatedAt: new Date() }) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.resourceId, resource.id), - eq(chatConversations.state, "unavailable"), - ), - ); + await setResourceConversationAvailability(tx, endpoint, resource.id, true, new Date()); } } @@ -132,17 +149,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc .update(chatEndpointResources) .set({ availability: absentAvailability, updatedAt: new Date() }) .where(eq(chatEndpointResources.id, resource.id)); - await tx - .update(chatConversations) - .set({ state: "unavailable", updatedAt: new Date() }) - .where( - and( - eq(chatConversations.companyId, endpoint.companyId), - eq(chatConversations.endpointId, endpoint.id), - eq(chatConversations.resourceId, resource.id), - inArray(chatConversations.state, ["active", "waiting"]), - ), - ); + await setResourceConversationAvailability(tx, endpoint, resource.id, false, new Date()); } await credentialLease.assertOwned(tx); }); @@ -279,6 +286,9 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc before: { enabled: row.enabled, ...(JSON.stringify(row.executionDefaults) !== JSON.stringify(finalDefaults.get(row.id)) ? { executionDefaults: row.executionDefaults } : {}) }, after: { enabled: finalEnabled.get(row.id)!, ...(JSON.stringify(row.executionDefaults) !== JSON.stringify(finalDefaults.get(row.id)) ? { executionDefaults: finalDefaults.get(row.id) } : {}) }, })); + if (changes.some(change => !change.before.enabled && change.after.enabled)) { + await notifyChatRunMilestones(tx, { companyId: endpoint.companyId }); + } if (bulk) { // One atomic database update, independent of pagination/search and // serialized with refresh, reconnect, and individual row changes. @@ -471,25 +481,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc await migrateTelegramResourceConversations(tx, currentEndpoint, previousResource, resource, previousProviderResourceId, providerResourceId, migratedLabel, now); } if (resource) { - await tx - .update(chatConversations) - .set({ - state: - effect.availability === "available" - ? "active" - : "unavailable", - updatedAt: now, - }) - .where( - and( - eq(chatConversations.companyId, currentEndpoint.companyId), - eq(chatConversations.endpointId, currentEndpoint.id), - eq(chatConversations.resourceId, resource.id), - effect.availability === "available" - ? eq(chatConversations.state, "unavailable") - : inArray(chatConversations.state, ["active", "waiting"]), - ), - ); + await setResourceConversationAvailability(tx, currentEndpoint, resource.id, effect.availability === "available", now); } await tx .update(chatEndpoints) @@ -512,7 +504,7 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc ? "verifying" : currentEndpoint.status; await notifyChatEndpointConfiguration(tx, currentEndpoint); - if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx, currentEndpoint.connectionId); + if (status !== currentEndpoint.status) await notifyChatEndpointWork(tx, currentEndpoint.connectionId, currentEndpoint.companyId); await tx .update(chatEndpoints) .set({ @@ -616,7 +608,10 @@ export function createChatResourceReconciliation(db: Db, options: ResourceReconc }); } - const { reconcileGitHubWebhookRepository, listGitHubRepositories, toggleAllGitHubRepositories } = createGitHubResources(db, { endpointRecord, upsertProviderResourceRow, updateResourceSelection }); + const { reconcileGitHubWebhookRepository, listGitHubRepositories, toggleAllGitHubRepositories } = createGitHubResources(db, { + endpointRecord, upsertProviderResourceRow, updateResourceSelection, + restoreResourceConversations: (tx, endpoint, resourceId) => setResourceConversationAvailability(tx, endpoint, resourceId, true, new Date()), + }); return { reconcileGitHubWebhookRepository, reconcileProviderResourceRows, listResources, listGitHubRepositories, replaceResources, toggleAllGitHubRepositories, hasNewerProcessedProviderLifecycleEffect, applyProviderAvailability }; } diff --git a/server/src/services/chat-channels/setup-probes.ts b/server/src/services/chat-channels/setup-probes.ts index 784bf2d38bf..31cacdb7ed2 100644 --- a/server/src/services/chat-channels/setup-probes.ts +++ b/server/src/services/chat-channels/setup-probes.ts @@ -300,7 +300,7 @@ export function createSetupProbes(db: Db, options: SetupProbesOptions) { )).limit(1).then((rows) => rows[0]); if (!linked || !(await lockCurrentPrincipalAuthorization(tx, endpoint, linked.principalId)).allowed) throw forbidden("Your connected account is no longer authorized"); } - await notifyChatEndpointWork(tx, endpoint.connectionId); + await notifyChatEndpointWork(tx, endpoint.connectionId, endpoint.companyId); const [activated] = await tx .update(chatEndpoints) .set({ diff --git a/server/src/services/chat-channels/slack/setup-services.ts b/server/src/services/chat-channels/slack/setup-services.ts index d0fb3a9e47f..45f8d3bc59c 100644 --- a/server/src/services/chat-channels/slack/setup-services.ts +++ b/server/src/services/chat-channels/slack/setup-services.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../../chat-run-milestone-notifications.js"; import { chatIdentityLinks, chatExternalPrincipals, companyMemberships, toolConnections, type Db } from "@paperclipai/db"; import { and, eq, ne, sql } from "drizzle-orm"; import { conflict, forbidden, notFound } from "../../../errors.js"; @@ -46,6 +47,7 @@ export function createSlackSetupServices(db: Db, options: SetupServicesOptions) const links = await tx.select().from(chatIdentityLinks).where(and(eq(chatIdentityLinks.companyId, record.endpoint.companyId), eq(chatIdentityLinks.principalId, principal.id))).for("update"); if (links.some(link => link.status === "linked" && link.paperclipUserId !== actor.userId) || links.some(link => link.endpointId === endpointId && link.status === "revoked")) throw conflict("This Slack account has an existing link", { code: "chat_identity_link_conflict" }); if (links.some(link => link.endpointId === endpointId && link.status === "linked")) return; + await notifyChatRunMilestones(tx, { companyId: record.endpoint.companyId }); await tx.insert(chatIdentityLinks).values({ companyId: record.endpoint.companyId, endpointId, principalId: principal.id, paperclipUserId: actor.userId, status: "linked", confirmedAt: new Date() }) .onConflictDoUpdate({ target: [chatIdentityLinks.endpointId, chatIdentityLinks.principalId], set: { diff --git a/server/src/services/chat-channels/telegram/resources.ts b/server/src/services/chat-channels/telegram/resources.ts index 17bc3b39b52..1e8174afe78 100644 --- a/server/src/services/chat-channels/telegram/resources.ts +++ b/server/src/services/chat-channels/telegram/resources.ts @@ -1,5 +1,6 @@ import { and, eq, sql } from "drizzle-orm"; import { chatConversations, chatEndpointResources, chatEndpoints, type Db } from "@paperclipai/db"; +import { notifyChatRunMilestones } from "../../chat-run-milestone-notifications.js"; import type { ChatProviderLifecycleEffect } from "../../chat-provider-lifecycle.js"; type DbTransaction = Parameters[0]>[0]; type EndpointRow = typeof chatEndpoints.$inferSelect; @@ -70,6 +71,18 @@ export async function migrateTelegramResourceConversations( resource: { id: string }, previousProviderResourceId: string | undefined, providerResourceId: string, migratedLabel: string, now: Date, ) { + const affected = and( + eq(chatConversations.companyId, currentEndpoint.companyId), + eq(chatConversations.endpointId, currentEndpoint.id), + eq(chatConversations.resourceId, previousResource.id), + ); + const migrated = await tx.select({ issueId: chatConversations.issueId }) + .from(chatConversations).where(affected).for("update"); + // Migration restores the binding before generic resource availability runs. + // Register the wake here, within the same owning transaction as that change. + for (const issueId of new Set(migrated.map(row => row.issueId))) { + await notifyChatRunMilestones(tx, { companyId: currentEndpoint.companyId, issueId }); + } const previousThreadPrefix = `telegram:${previousProviderResourceId}`; const migratedThreadPrefix = `telegram:${providerResourceId}`; await tx @@ -94,11 +107,5 @@ export async function migrateTelegramResourceConversations( state: "active", updatedAt: now, }) - .where( - and( - eq(chatConversations.companyId, currentEndpoint.companyId), - eq(chatConversations.endpointId, currentEndpoint.id), - eq(chatConversations.resourceId, previousResource.id), - ), - ); + .where(affected); } diff --git a/server/src/services/chat-github-management.ts b/server/src/services/chat-github-management.ts index 9d30b793ab1..3f1bc34a6b4 100644 --- a/server/src/services/chat-github-management.ts +++ b/server/src/services/chat-github-management.ts @@ -24,6 +24,7 @@ import { import { HttpError, badRequest, conflict, forbidden, notFound, unprocessable } from "../errors.js"; import { toolAccessService } from "./tool-access.js"; import { logActivity } from "./activity-log.js"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { githubBotRequest } from "./chat-github-client.js"; import { environmentService } from "./environments.js"; import { instanceSettingsService } from "./instance-settings.js"; @@ -145,11 +146,12 @@ export function githubChatManagementService(db: Db, fetchImpl = fetch) { "Repository overrides must refer to this App installation", ); await db.transaction(async (tx) => { - await tx - .select({ id: chatEndpoints.id }) + const [currentEndpoint] = await tx + .select({ allowUnlinkedPeople: chatEndpoints.allowUnlinkedPeople, sponsorUserId: chatEndpoints.sponsorUserId }) .from(chatEndpoints) .where(eq(chatEndpoints.id, endpointId)) .for("update"); + if (!currentEndpoint) throw notFound("GitHub bot not found"); await tx.execute( sql`select pg_advisory_xact_lock(hashtextextended(${`github-config:${endpointId}`}, 0))`, ); @@ -160,6 +162,11 @@ export function githubChatManagementService(db: Db, fetchImpl = fetch) { .for("update"); if ((current?.revision ?? 0) !== parsed.expectedRevision) throw conflict("This configuration changed. Refresh before saving."); + const allowUnlinkedPeople = config.people.some((p) => p.kind === "guest"); + if ((!currentEndpoint.allowUnlinkedPeople && allowUnlinkedPeople) || + (allowUnlinkedPeople && currentEndpoint.sponsorUserId !== config.responsibleUserId)) { + await notifyChatRunMilestones(tx, { companyId: bot.companyId }); + } const values = { companyId: bot.companyId, endpointId, @@ -180,7 +187,7 @@ export function githubChatManagementService(db: Db, fetchImpl = fetch) { await tx .update(chatEndpoints) .set({ - allowUnlinkedPeople: config.people.some((p) => p.kind === "guest"), + allowUnlinkedPeople, sponsorUserId: config.responsibleUserId, updatedAt: new Date(), }) @@ -327,6 +334,9 @@ export function githubChatManagementService(db: Db, fetchImpl = fetch) { expiresAt: null, updatedAt: new Date(), }; + if (prior?.status !== "linked") { + await notifyChatRunMilestones(tx, { companyId: bot.companyId }); + } await tx .insert(chatIdentityLinks) .values(values) diff --git a/server/src/services/chat-interaction-arbitration.test.ts b/server/src/services/chat-interaction-arbitration.test.ts index 4addb3b453e..3e32fb04317 100644 --- a/server/src/services/chat-interaction-arbitration.test.ts +++ b/server/src/services/chat-interaction-arbitration.test.ts @@ -15,19 +15,20 @@ import { toolConnections, } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { hasChatRunOwnedProviderInteraction } from "./chat-interaction-arbitration.js"; +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ARBITRATION_TEST_DATABASE_URL"); + describe("provider-owned interaction arbitration", () => { - let temporary: Awaited>; + let temporary: Awaited> | undefined; let db: ReturnType; - let companyCount = 0; beforeAll(async () => { - temporary = await startEmbeddedPostgresTestDatabase( - "chat-interaction-owner-", - ); - db = createDb(temporary.connectionString); + if (!externalDatabaseUrl) temporary = await startEmbeddedPostgresTestDatabase("chat-interaction-owner-"); + db = createDb(externalDatabaseUrl ?? temporary!.connectionString); }, 30_000); afterAll(async () => { + await db?.$client.end(); await temporary?.cleanup(); }); @@ -43,7 +44,7 @@ describe("provider-owned interaction arbitration", () => { .values({ id: companyId, name: "Arbitration", - issuePrefix: `CI${String.fromCharCode(65 + companyCount++)}`, + issuePrefix: `C${companyId.slice(0, 7)}`, }); await db.insert(agents).values( [agentId, otherAgentId].map((id) => ({ diff --git a/server/src/services/chat-run-milestone-notifications.ts b/server/src/services/chat-run-milestone-notifications.ts new file mode 100644 index 00000000000..743a7dd992f --- /dev/null +++ b/server/src/services/chat-run-milestone-notifications.ts @@ -0,0 +1,9 @@ +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; + +/** Lineage changes can authorize descendants, so invalidate the issue rather + * than just the run whose row changed. Configuration may invalidate a company. */ +export type ChatRunMilestoneScope = { companyId: string; issueId?: string }; + +export function notifyChatRunMilestones(tx: object, scope: ChatRunMilestoneScope) { + return notifyDeliveryWork(tx, DELIVERY_QUEUES.chatRunMilestones, JSON.stringify({ companyId: scope.companyId, issueId: scope.issueId })); +} diff --git a/server/src/services/chat-run-milestone-work.ts b/server/src/services/chat-run-milestone-work.ts new file mode 100644 index 00000000000..c02bdd2378d --- /dev/null +++ b/server/src/services/chat-run-milestone-work.ts @@ -0,0 +1,38 @@ +import type { Db } from "@paperclipai/db"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; +import type { ChatRunMilestoneScope } from "./chat-run-milestone-notifications.js"; +import { enqueueChatRunLifecycleMilestones, loadChatRunLifecycleMilestoneScopes } from "./chat-run-publications.js"; + +type Entry = { id: string; companyId: string; issueId: string; dueAt: number }; +const batchSize = 100; + +/** No periodic discovery: reconstruct on startup/committed changes, drain + * current notices, and retain a retry only for failure or an unfinished batch. */ +export function chatRunMilestoneWork(db: Db, publicBaseUrl?: string | null) { + async function load(scope?: ChatRunMilestoneScope, signal?: AbortSignal): Promise { + return (await loadChatRunLifecycleMilestoneScopes(db, scope, signal)).map(row => ({ + ...row, id: `${row.companyId}:${row.issueId}`, dueAt: 0, + })); + } + async function check(entry: Entry, signal?: AbortSignal): Promise { + const inserted = await enqueueChatRunLifecycleMilestones(db, { + scope: { companyId: entry.companyId, issueId: entry.issueId }, + publicBaseUrl, limit: batchSize, signal, + }); + // A short batch exhausted the selector, including currently suppressed + // candidates. Their dependencies must signal a later eligibility change. + return inserted === batchSize ? { ...entry, dueAt: Date.now() } : null; + } + function register(coordinator: ReturnType, canRun: () => boolean) { + return registerScopedDeadlineWork(coordinator, { + queue: DELIVERY_QUEUES.chatRunMilestones, + decodeScope: (hint: string): ChatRunMilestoneScope | undefined => hint === "*" ? undefined : JSON.parse(hint), + matches: (entry: Entry, scope?: ChatRunMilestoneScope) => !scope || + (entry.companyId === scope.companyId && (!scope.issueId || entry.issueId === scope.issueId)), + load, check, + }, canRun); + } + return { load, check, register }; +} diff --git a/server/src/services/chat-run-publications.ts b/server/src/services/chat-run-publications.ts index 93fe111ec2d..e6a1ac317a4 100644 --- a/server/src/services/chat-run-publications.ts +++ b/server/src/services/chat-run-publications.ts @@ -1,3 +1,4 @@ +import type { ChatRunMilestoneScope } from "./chat-run-milestone-notifications.js"; import { notifyChatPublicationWork } from "./chat-work-notifications.js"; import { and, @@ -218,7 +219,7 @@ type ChatRunMilestoneCandidate = { runId: string; runStatus: string; runErrorCode: string | null; - runUpdatedAt: Date; + runUpdatedAt: string; issueId: string; companyId: string; endpointId: string; @@ -297,10 +298,11 @@ export function safeMilestoneText(input: { * lane. The selector intentionally reads only event identity, type, sequence, * and time; native messages and payloads stay inside Paperclip. */ -async function enqueueSafeNativeChatProgress( +export async function enqueueSafeNativeChatProgress( db: Db, - input: { since: Date; limit: number }, + options: { since?: Date; limit?: number } = {}, ): Promise { + const input = chatRunProjectionWindow(options); const basePublication = alias( chatPublications, "safe_native_progress_base_publication", @@ -670,26 +672,19 @@ async function enqueueSafeNativeChatProgress( return inserted; } -/** - * Project only coarse run lifecycle into bound external conversations. Raw - * output, errors, tool events, and reasoning stay in Paperclip. Idempotency is - * keyed by run, milestone, and endpoint so polling and restarts are harmless. - */ -export async function enqueueChatRunMilestones( - db: Db, - input: { - publicBaseUrl?: string | null; - since?: Date; - limit?: number; - } = {}, -): Promise { - const since = input.since ?? new Date(Date.now() - 24 * 60 * 60_000); - const limit = Math.max(1, Math.min(input.limit ?? 200, 1_000)); - const issueIdFromContext = boundRunIssueIdSql(); - const explicitlyAuthoredCommentReason = sql`( +type ChatRunMilestoneCursor = { + updatedAt: string; + runId: string; + conversationId: string; +}; + +const explicitlyAuthoredCommentReason = sql`( ${issueComments.metadata} ->> 'authorizationReason' = 'paperclip_runner_protocol' or left(coalesce(${issueComments.metadata} ->> 'authorizationReason', ''), 6) = 'allow_' )`; + +function lifecycleMilestonePages(db: Db, since: Date, scope?: ChatRunMilestoneScope) { + const issueIdFromContext = boundRunIssueIdSql(); const milestoneFromStatus = sql`case when ${heartbeatRuns.status} = 'queued' then 'queued' when ${heartbeatRuns.status} = 'running' @@ -801,26 +796,15 @@ export async function enqueueChatRunMilestones( and continuation_wake.payload->>'issueId' = ${issueIdFromContext} ) )`; - let inserted = 0; - let cursor: { - updatedAt: Date; - runId: string; - conversationId: string; - } | null = null; - const bindingsCache = new Map< - string, - Awaited> - >(); - const providerInteractionCache = new Map(); - while (inserted < limit) { - const pageSize = Math.max(25, Math.min(200, limit - inserted)); - const pageCursor: typeof cursor = cursor; - const rows: ChatRunMilestoneCandidate[] = await db + return (pageSize: number, pageCursor: ChatRunMilestoneCursor | null): Promise => { + return db .select({ runId: heartbeatRuns.id, runStatus: heartbeatRuns.status, runErrorCode: heartbeatRuns.errorCode, - runUpdatedAt: heartbeatRuns.updatedAt, + // Keep PostgreSQL precision: a Date cursor can repeatedly select a + // rejected page whose updated_at has submillisecond digits. + runUpdatedAt: sql`${heartbeatRuns.updatedAt}::text`, issueId: chatConversations.issueId, companyId: chatConversations.companyId, endpointId: chatConversations.endpointId, @@ -979,22 +963,13 @@ export async function enqueueChatRunMilestones( and target_delivery.status in ('delivered', 'fallback_queued') )`, ), + scope ? eq(heartbeatRuns.companyId, scope.companyId) : undefined, + scope?.issueId ? eq(chatConversations.issueId, scope.issueId) : undefined, gte(heartbeatRuns.updatedAt, since), isNull(chatPublications.id), pageCursor - ? or( - gt(heartbeatRuns.updatedAt, pageCursor.updatedAt), - and( - eq(heartbeatRuns.updatedAt, pageCursor.updatedAt), - or( - gt(heartbeatRuns.id, pageCursor.runId), - and( - eq(heartbeatRuns.id, pageCursor.runId), - gt(chatConversations.id, pageCursor.conversationId), - ), - ), - ), - ) + ? sql`(${heartbeatRuns.updatedAt}, ${heartbeatRuns.id}, ${chatConversations.id}) > + (${pageCursor.updatedAt}::timestamptz, ${pageCursor.runId}::uuid, ${pageCursor.conversationId}::uuid)` : undefined, ), ) @@ -1004,15 +979,59 @@ export async function enqueueChatRunMilestones( asc(chatConversations.id), ) .limit(pageSize); + }; +} + +function milestoneCursor(row: ChatRunMilestoneCandidate): ChatRunMilestoneCursor { + return { updatedAt: row.runUpdatedAt, runId: row.runId, conversationId: row.conversationId }; +} + +/** Reconstruct issue-scoped projection work. Candidate rows are only hints; + * projection still applies current lineage and provider authority. */ +export async function loadChatRunLifecycleMilestoneScopes( + db: Db, + scope?: ChatRunMilestoneScope, + signal?: AbortSignal, +): Promise> { + const pages = lifecycleMilestonePages(db, chatRunProjectionWindow({}).since, scope); + const scopes = new Map(); + let cursor: ChatRunMilestoneCursor | null = null; + for (;;) { + signal?.throwIfAborted(); + const rows = await pages(100, cursor); + for (const row of rows) scopes.set(`${row.companyId}:${row.issueId}`, { companyId: row.companyId, issueId: row.issueId }); + if (rows.length < 100) return [...scopes.values()]; + cursor = milestoneCursor(rows[rows.length - 1]); + } +} + +/** + * Project only coarse run lifecycle into bound external conversations. Raw + * output, errors, tool events, and reasoning stay in Paperclip. Idempotency is + * keyed by run, milestone, and endpoint so polling and restarts are harmless. + */ +export async function enqueueChatRunLifecycleMilestones( + db: Db, + input: ChatRunProjectionInput & { scope?: ChatRunMilestoneScope; signal?: AbortSignal } = {}, +): Promise { + const { since, limit } = chatRunProjectionWindow(input); + let inserted = 0; + const pages = lifecycleMilestonePages(db, since, input.scope); + let cursor: ChatRunMilestoneCursor | null = null; + const bindingsCache = new Map< + string, + Awaited> + >(); + const providerInteractionCache = new Map(); + while (inserted < limit) { + const pageSize = Math.max(25, Math.min(200, limit - inserted)); + input.signal?.throwIfAborted(); + const rows = await pages(pageSize, cursor); if (rows.length === 0) break; - const lastRow = rows.at(-1)!; - cursor = { - updatedAt: lastRow.runUpdatedAt, - runId: lastRow.runId, - conversationId: lastRow.conversationId, - }; + cursor = milestoneCursor(rows[rows.length - 1]); for (const row of rows) { + input.signal?.throwIfAborted(); if (inserted >= limit) break; const milestone = milestoneForStatus(row.runStatus, row.runErrorCode); if (!milestone) continue; @@ -1107,11 +1126,30 @@ export async function enqueueChatRunMilestones( } if (rows.length < pageSize) break; } - if (inserted < limit) { - inserted += await enqueueSafeNativeChatProgress(db, { - since, - limit: limit - inserted, - }); - } return inserted; } + +type ChatRunProjectionInput = { + publicBaseUrl?: string | null; + since?: Date; + limit?: number; +}; + +function chatRunProjectionWindow(input: ChatRunProjectionInput) { + return { + since: input.since ?? new Date(Date.now() - 24 * 60 * 60_000), + limit: Math.max(1, Math.min(input.limit ?? 200, 1_000)), + }; +} + +/** Explicit combined reconciliation retains lifecycle priority and one shared + * publication budget. Background owners can reconcile the two duties separately. */ +export async function enqueueChatRunMilestones( + db: Db, + input: ChatRunProjectionInput = {}, +): Promise { + const { since, limit } = chatRunProjectionWindow(input); + const inserted = await enqueueChatRunLifecycleMilestones(db, { ...input, since, limit }); + if (inserted >= limit) return inserted; + return inserted + await enqueueSafeNativeChatProgress(db, { since, limit: limit - inserted }); +} diff --git a/server/src/services/chat-work-notifications.ts b/server/src/services/chat-work-notifications.ts index 71b00c50cf0..5e0d05262c9 100644 --- a/server/src/services/chat-work-notifications.ts +++ b/server/src/services/chat-work-notifications.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyChatConnectionConfiguration } from "./chat-channels/configuration-work-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; @@ -5,10 +6,19 @@ export async function notifyChatDeliveryWork(tx: object): Promise { await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatInbound); } -export async function notifyChatPublicationWork(tx: object): Promise { +export async function notifyChatPublicationWork( + tx: object, + milestoneScope?: { companyId: string; issueId: string | null }, +): Promise { await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatPublications); // Receipt recovery waits for the publication owner to release its claim. await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatReceipts); + // Published prompts and terminal delivery outcomes can change run lineage or + // release its provider response slot. Callers supply the affected issue only + // for those changes; transport claim/lease bookkeeping has no projection debt. + if (milestoneScope?.issueId) { + await notifyChatRunMilestones(tx, { companyId: milestoneScope.companyId, issueId: milestoneScope.issueId }); + } } export const CHAT_ACTION_QUEUES = { @@ -32,7 +42,10 @@ export async function notifyChatVerificationWork(tx: object): Promise { await notifyDeliveryWork(tx, DELIVERY_QUEUES.chatVerification); } -export async function notifyChatEndpointWork(tx: object, connectionId: string): Promise { +export async function notifyChatEndpointWork(tx: object, connectionId: string, companyId: string): Promise { + // Configuration/status transitions can restore post-control publication + // authority. Ordinary health checks use the narrower configuration signals. + await notifyChatRunMilestones(tx, { companyId }); await notifyChatDeliveryWork(tx); await notifyChatPublicationWork(tx); for (const queue of Object.values(CHAT_ACTION_QUEUES)) await notifyDeliveryWork(tx, queue); diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 16586747cde..e4434d9136f 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -57,6 +57,7 @@ export const DELIVERY_QUEUES = { chatVerification: "chat-verification", chatInbound: "chat-inbound", chatPublications: "chat-publications", + chatRunMilestones: "chat-run-milestones", chatReceipts: "chat-slack-receipts", browser: "browser-use", email: "email-channels", diff --git a/server/src/services/heartbeat/queue.test.ts b/server/src/services/heartbeat/queue.test.ts index ef889747669..7f9e2f7a127 100644 --- a/server/src/services/heartbeat/queue.test.ts +++ b/server/src/services/heartbeat/queue.test.ts @@ -3,6 +3,7 @@ import type { Db } from "@paperclipai/db"; import { agents, agentWakeupRequests, caseHandoffs, cases, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createPostgresWakeQueueAdapter } from "../../modules/wake-queue/adapters/postgres.js"; import { createReleaseIssueExecution } from "../../modules/wake-queue/application/use-cases.js"; @@ -74,14 +75,15 @@ describe("heartbeat queue module boundary", () => { }); }); -const support = await getEmbeddedPostgresTestSupport(); +const external = resolveExternalTestDatabaseUrl("PAPERCLIP_QUEUE_TEST_DATABASE_URL"); +const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport(); if (!support.supported) console.warn(`Skipping queue module database tests: ${support.reason}`); describe.skipIf(!support.supported)("heartbeat queue database wiring", () => { - let database: Awaited>; + let database: Awaited> | undefined; let db: Db; beforeAll(async () => { - database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-queue-module"); - db = createDb(database.connectionString); + if (!external) database = await startEmbeddedPostgresTestDatabase("paperclip-heartbeat-queue-module"); + db = createDb(external ?? database!.connectionString); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { await db?.$client.end(); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 5293009a369..7778b17ac8b 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { assertTaskRetryConversationAllowed, readTaskRetryIntent, readTaskRetryBoardMessageOrigin, retireSupersededTaskRetry, taskRetryMutationAllowed } from "../task-retry.js"; import { insertIssueComment } from "../issue-comment-writes.js"; import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modules/external-agents/index.js"; @@ -845,6 +846,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const error = stopped ? "Automatic continuation stopped by the committed chat conversation close. Send a new request in chat or on the Board to start fresh work." : "Automatic continuation source could not be verified before provider admission. Review the task and send a fresh request; this attempt will not automatically retry."; + await notifyChatRunMilestones(tx, { companyId: current.companyId, issueId }); [terminal] = await tx .update(heartbeatRuns) .set({ @@ -960,9 +962,17 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (!["pending_dispatch", "delivered"].includes(caseHandoffReceipt.state) || caseHandoffReceipt.toAgentId !== run.agentId || !current || current.version !== caseHandoffReceipt.version || current.issueId !== caseHandoffReceipt.issueId || current.agentId !== run.agentId || (!run.startedAt && current.statusVersion !== caseHandoffReceipt.appliedIssueStatusVersion)) { - const [cancelled] = await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), - errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() }) - .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))).returning(); + const cancelled = await db.transaction(async tx => { + const [currentRun] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))) + .for("update"); + if (!currentRun) return null; + await notifyChatRunMilestones(tx, { companyId: run.companyId }); + const [row] = await tx.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + errorCode: "case_handoff_superseded", error: "This case handoff was cancelled or superseded.", updatedAt: new Date() }) + .where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.status, "queued"))).returning(); + return row; + }); if (cancelled) await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: new Date(), error: cancelled.error }); return null; } @@ -1317,6 +1327,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende if (proof.kind === "stopped") { const error = "Automatic continuation stopped by the committed chat conversation close. Send a fresh request in chat or on the Board."; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [cancelled] = await tx .update(heartbeatRuns) .set({ @@ -1395,6 +1406,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // explicitly bound queued-message envelope is subject to the // live-comment discard gate below. await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [claimedRun] = await tx .update(heartbeatRuns) .set({ @@ -1436,6 +1448,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende }); if (liveIds.length === 0) { const reason = "Queued messages were discarded before dispatch"; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [cancelled] = await tx .update(heartbeatRuns) .set({ @@ -1483,6 +1496,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende } await withExternalAdmissionGuard(tx, run.companyId, run.agentId, () => assertNoExternalOverlap(tx, run.companyId, run.agentId)); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); await tx .update(agentWakeupRequests) .set({ @@ -1593,6 +1607,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende )).for("update"); if (!lockedRun || lockedRun.status !== "queued") return null; await withExternalAdmissionGuard(claimTx, run.companyId, run.agentId, () => assertNoExternalOverlap(claimTx, run.companyId, run.agentId)); + await notifyChatRunMilestones(claimTx, { companyId: run.companyId, issueId: issueId ?? undefined }); const claimedRun = await claimTx.update(heartbeatRuns).set(claimValues).where(and( eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "queued"), )).returning().then((rows) => rows[0] ?? null); @@ -1690,6 +1705,11 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende async function releaseRunClaimedJustBeforeSuppression(runId: string) { const now = new Date(); await db.transaction(async (tx) => { + const [candidate] = await tx.select({ companyId: heartbeatRuns.companyId, contextSnapshot: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.status, "running"))).for("update"); + if (!candidate) return; + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, + issueId: readNonEmptyString(parseObject(candidate.contextSnapshot).issueId) ?? undefined }); const released = await tx .update(heartbeatRuns) .set({ @@ -3454,6 +3474,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende const reason = issueCancelled ? "Cancelled because the issue was cancelled before the scheduled retry became due" : "Cancelled because the issue was reassigned before the scheduled retry became due"; + await notifyChatRunMilestones(tx, { companyId: scheduledRun.companyId, issueId: issue.id }); const cancelled = await tx .update(heartbeatRuns) .set({ @@ -3590,6 +3611,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende issue.assigneeAgentId && activeExecutionRun.agentId !== issue.assigneeAgentId ) { + await notifyChatRunMilestones(tx, { companyId: activeExecutionRun.companyId, issueId: issue.id }); const cancelled = await tx .update(heartbeatRuns) .set({ @@ -4309,6 +4331,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende adoptedCommentIds = await undeliveredLegacyUserCommentIds(tx as unknown as Db, agent.companyId, issueId, agentId, adoptedCommentIds); } + await notifyChatRunMilestones(tx, { companyId: agent.companyId, issueId: issueId ?? undefined }); const newRun = await tx .insert(heartbeatRuns) .values({ @@ -4701,6 +4724,13 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende >(); for (const candidate of candidates) { + const payload = parseObject(candidate.payload); + const wakeContext = parseObject(payload._paperclipWakeContext); + const issueId = + readNonEmptyString(payload.issueId) ?? + readNonEmptyString(payload.taskId) ?? + readNonEmptyString(wakeContext.issueId) ?? + null; const dispatchActorId = `native-status-wake-dispatch:${candidate.id}`; const existingDispatch = await db .select() @@ -4722,24 +4752,27 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende : existingDispatch.status === "deferred_issue_execution" ? "coalesced" : existingDispatch.status; - await db - .update(agentWakeupRequests) - .set({ - status: recoveredStatus, - runId: existingDispatch.runId, - finishedAt: - (existingDispatch.runId ?? existingDispatch.finishedAt) - ? (existingDispatch.finishedAt ?? now) - : null, - error: existingDispatch.error, - updatedAt: now, - }) - .where( - and( - eq(agentWakeupRequests.id, candidate.id), - isNull(agentWakeupRequests.runId), - ), - ); + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: recoveredStatus, + runId: existingDispatch.runId, + finishedAt: + (existingDispatch.runId ?? existingDispatch.finishedAt) + ? (existingDispatch.finishedAt ?? now) + : null, + error: existingDispatch.error, + updatedAt: now, + }) + .where( + and( + eq(agentWakeupRequests.id, candidate.id), + isNull(agentWakeupRequests.runId), + ), + ); + }); recovered += 1; continue; } @@ -4773,13 +4806,6 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende .returning({ id: agentWakeupRequests.id }); if (claimed.length === 0) continue; - const payload = parseObject(candidate.payload); - const wakeContext = parseObject(payload._paperclipWakeContext); - const issueId = - readNonEmptyString(payload.issueId) ?? - readNonEmptyString(payload.taskId) ?? - readNonEmptyString(wakeContext.issueId) ?? - null; const scopeKey = issueId ? `${candidate.companyId}:${candidate.agentId}:${issueId}:${readNativeReviewAssignmentContext(wakeContext)?.nativeReviewInteractionId ?? ""}` : null; @@ -4787,16 +4813,19 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende ? deliveredByIssueScope.get(scopeKey) : null; if (priorDelivery) { - await db - .update(agentWakeupRequests) - .set({ - status: "coalesced", - runId: priorDelivery.runId, - finishedAt: new Date(), - error: null, - updatedAt: new Date(), - }) - .where(eq(agentWakeupRequests.id, candidate.id)); + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: "coalesced", + runId: priorDelivery.runId, + finishedAt: new Date(), + error: null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); recovered += 1; continue; } @@ -4895,21 +4924,24 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // promote both rows (the original has no nativeStatusWakeIntentId // provenance and would otherwise run once before the dispatch receipt). const deliveredStatus = delivered?.status ?? "queued"; - await db - .update(agentWakeupRequests) - .set({ - status: wakeRun || deliveredStatus === "deferred_issue_execution" - ? "coalesced" - : deliveredStatus, - runId: wakeRun?.id ?? delivered?.runId ?? null, - finishedAt: - wakeRun || delivered?.finishedAt - ? (delivered?.finishedAt ?? new Date()) - : null, - error: delivered?.error ?? null, - updatedAt: new Date(), - }) - .where(eq(agentWakeupRequests.id, candidate.id)); + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: candidate.companyId, issueId: issueId ?? undefined }); + await tx + .update(agentWakeupRequests) + .set({ + status: wakeRun || deliveredStatus === "deferred_issue_execution" + ? "coalesced" + : deliveredStatus, + runId: wakeRun?.id ?? delivered?.runId ?? null, + finishedAt: + wakeRun || delivered?.finishedAt + ? (delivered?.finishedAt ?? new Date()) + : null, + error: delivered?.error ?? null, + updatedAt: new Date(), + }) + .where(eq(agentWakeupRequests.id, candidate.id)); + }); if (scopeKey) { deliveredByIssueScope.set(scopeKey, { diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 95026919171..62a0396e2e1 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js"; import { pendingCaseHandoffForRun } from "./case-handoff-lifecycle.js"; @@ -891,13 +892,13 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC }; } - await db - .update(heartbeatRuns) - .set({ + await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: livenessRun.companyId, issueId: issueId ?? undefined }); + await tx.update(heartbeatRuns).set({ resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ presentationDecision })}::jsonb`, updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, livenessRun.id)); + }).where(eq(heartbeatRuns.id, livenessRun.id)); + }); await appendRunEvent(livenessRun, { eventType: "run.presentation.resolved", stream: "system", diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 3bbb6f30b40..7fedb4e117d 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import os from "node:os"; import { @@ -1019,6 +1020,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl const tx = transaction as unknown as Db; if (isHeartbeatRunTerminalStatus(status)) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, companyId); await notifyExecutionRunWork(tx); + if (companyId) await notifyChatRunMilestones(tx, { companyId }); return write(tx); }); } diff --git a/server/src/services/issue-thread-interactions.ts b/server/src/services/issue-thread-interactions.ts index 44eb4391055..363d9f2a6f5 100644 --- a/server/src/services/issue-thread-interactions.ts +++ b/server/src/services/issue-thread-interactions.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyToolReviewChange } from "./tool-review-work-notifications.js"; import { normalizeEscapedLineBreaks } from "@paperclipai/shared/validators/text"; import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; @@ -1912,6 +1913,7 @@ async function expireStaleRequestConfirmationTarget( snapshot, }); const expireInTransaction = async (tx: Db | any) => { + if (args.row.kind === "request_confirmation") await notifyChatRunMilestones(tx, args.row); const [updated] = await tx .update(issueThreadInteractions) .set({ @@ -2189,6 +2191,7 @@ export function issueThreadInteractionService( const postCommitActivityPublications: ActivityPublication[] = []; const result = await db.transaction(async (tx) => { await assertInteractionRunWriteAllowed(tx as unknown as Db, args.issue, args.actor); + if (args.current.kind === "request_confirmation") await notifyChatRunMilestones(tx, args.current); await args.mutationOptions?.beforeResolveInTransaction?.(tx); // Policy mutations and review transitions use the same issue-row lock, // so the authoritative review policy and requester are stable through @@ -2472,6 +2475,7 @@ export function issueThreadInteractionService( const now = new Date(); const updated = await db.transaction(async (tx) => { await assertInteractionRunWriteAllowed(tx as unknown as Db, args.issue, args.actor); + if (args.current.kind === "request_confirmation") await notifyChatRunMilestones(tx, args.current); await args.mutationOptions?.beforeResolveInTransaction?.(tx); const issueContext = await tx .select({ @@ -3290,6 +3294,7 @@ export function issueThreadInteractionService( const cancelled: IssueThreadInteraction[] = []; for (const row of rows) { const updated = await db.transaction(async (tx) => { + if (["ask_user_questions", "request_confirmation"].includes(row.kind)) await notifyChatRunMilestones(tx, row); if (row.kind === "request_confirmation") await notifyToolProjectionForInteractions(tx, row.companyId, [row.id]); const [resolved] = await tx .update(issueThreadInteractions) @@ -3375,6 +3380,7 @@ export function issueThreadInteractionService( const expired: IssueThreadInteraction[] = []; for (const { row, replacementInteractionId } of supersededRows) { const updated = await db.transaction(async (tx) => { + if (["ask_user_questions", "request_confirmation"].includes(row.kind)) await notifyChatRunMilestones(tx, row); if (row.kind === "request_confirmation") await notifyToolProjectionForInteractions(tx, row.companyId, [row.id]); const [updatedRow] = await tx .update(issueThreadInteractions) @@ -3736,6 +3742,9 @@ export function issueThreadInteractionService( eq(chatVoiceSessions.callerAuthority, "guest_intake"))).limit(1); if (guest) throw unprocessable("Ask this unverified caller a clarification in a task comment; submit_request will deliver their spoken follow-up. Protected human-input questions require authenticated access.", {code: "voice_guest_use_task_comment"}); } + if (data.kind === "ask_user_questions" || data.kind === "request_confirmation") { + await notifyChatRunMilestones(tx, { companyId: issue.companyId, issueId: issue.id }); + } const [row] = await tx .insert(issueThreadInteractions) .values({ @@ -4459,6 +4468,7 @@ export function issueThreadInteractionService( const expired: IssueThreadInteraction[] = []; for (const row of superseded) { const updated = await db.transaction(async (tx) => { + if (["ask_user_questions", "request_confirmation"].includes(row.kind)) await notifyChatRunMilestones(tx, row); if (row.kind === "connection_intent") { await tx .delete(toolOauthStates) @@ -4595,6 +4605,7 @@ export function issueThreadInteractionService( ); if (!sampleQuestionRow) continue; const updatedInteractions = await db.transaction(async (tx) => { + await notifyChatRunMilestones(tx, sampleQuestionRow); const updatedRows = await tx .update(issueThreadInteractions) .set({ @@ -4633,6 +4644,7 @@ export function issueThreadInteractionService( ); if (!sampleConfirmationRow) continue; const updatedInteractions = await db.transaction(async (tx) => { + if (commentRows.some(row => row.kind === "request_confirmation")) await notifyChatRunMilestones(tx, sampleConfirmationRow); await notifyToolProjectionForInteractions(tx, issue.companyId, confirmationRowIds); const updatedRows = await tx .update(issueThreadInteractions) @@ -4778,6 +4790,7 @@ export function issueThreadInteractionService( document, }); const updated = await db.transaction(async (tx) => { + if (row.kind === "request_confirmation") await notifyChatRunMilestones(tx, row); const [resolved] = await tx .update(issueThreadInteractions) .set({ @@ -4898,6 +4911,7 @@ export function issueThreadInteractionService( // lifecycle reflection. const updated = await db .transaction(async (tx) => { + if (["ask_user_questions", "request_confirmation"].includes(row.kind)) await notifyChatRunMilestones(tx, row); if (row.kind === "connection_intent") { await tx .delete(toolOauthStates) @@ -4988,6 +5002,7 @@ export function issueThreadInteractionService( // request must not outlive a withdrawn card. const updated = await db.transaction(async (tx) => { await assertInteractionRunWriteAllowed(tx as unknown as Db, issue, actor); + if (["ask_user_questions", "request_confirmation"].includes(current.kind)) await notifyChatRunMilestones(tx, current); await resolveLinkedToolActionRequests(tx, current, { status: "cancelled", fromStatuses: ["pending", "approved"], @@ -5123,6 +5138,7 @@ export function issueThreadInteractionService( if (!historicalAnswer) throw interactionIssueClosedError(); } await assertInteractionRunWriteAllowed(tx as unknown as Db, issue, actor); + await notifyChatRunMilestones(tx, current); await mutationOptions.beforeResolveInTransaction?.(tx); const resolvedAt = new Date(); const [row] = await tx @@ -5247,6 +5263,7 @@ export function issueThreadInteractionService( const now = new Date(); const updated = await db.transaction(async (tx) => { await assertInteractionRunWriteAllowed(tx as unknown as Db, issue, actor); + if (["ask_user_questions", "request_confirmation"].includes(current.kind)) await notifyChatRunMilestones(tx, current); await resolveLinkedToolActionRequests(tx, current, { status: "cancelled", fromStatuses: ["pending", "approved"], @@ -5344,6 +5361,7 @@ export function issueThreadInteractionService( const reason = data.reason?.trim() || null; const updated = await db.transaction(async (tx) => { await assertInteractionRunWriteAllowed(tx as unknown as Db, issue, actor); + await notifyChatRunMilestones(tx, current); const resolvedAt = new Date(); const [row] = await tx .update(issueThreadInteractions) diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index 26ed76d447b..e36f1d20a28 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyApprovalWork } from "./approval-work-notifications.js"; import { notifyDecisionTargetWork, notifyDecisionTargetIssueRemoval } from "./decision-targets.js"; import { assertOnboardingFirstTaskCreationAllowed } from "./onboarding-first-task-admission.js"; @@ -11413,6 +11414,9 @@ export function issueService(db: Db) { if (patch.status === "done" || patch.status === "cancelled") { await notifyDecisionTargetWork(tx, `issue:${id}`); } + if (patch.hiddenAt === null && receiptExisting.hiddenAt !== null) { + await notifyChatRunMilestones(tx, { companyId: receiptExisting.companyId, issueId: id }); + } if (patch.hiddenAt !== undefined || patch.harnessKind !== undefined || patch.conversationAgentId !== undefined) { await notifyApprovalWork(tx, receiptExisting.companyId); } diff --git a/server/src/services/native-runtime/automatic-completion-reviews.ts b/server/src/services/native-runtime/automatic-completion-reviews.ts index 1e08e33f1d3..6f12c75b7dc 100644 --- a/server/src/services/native-runtime/automatic-completion-reviews.ts +++ b/server/src/services/native-runtime/automatic-completion-reviews.ts @@ -15,6 +15,7 @@ import { type ActivityPublication, } from "../activity-log.js"; import { enqueueTerminalIssueInteractionChatPublications } from "../chat-interaction-publications.js"; +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; import { logger } from "../../middleware/logger.js"; @@ -126,6 +127,7 @@ export async function dismissAutomaticCompletionReviews( ) .for("update"); if (!issue) return; + await notifyChatRunMilestones(tx, interaction); const now = new Date(); const [retired] = await tx .update(issueThreadInteractions) diff --git a/server/src/services/native-runtime/native-chat-review-presentation.ts b/server/src/services/native-runtime/native-chat-review-presentation.ts index 74cdd87c4fb..366e564acb6 100644 --- a/server/src/services/native-runtime/native-chat-review-presentation.ts +++ b/server/src/services/native-runtime/native-chat-review-presentation.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { and, eq, inArray, ne, sql } from "drizzle-orm"; import { agents, @@ -573,20 +574,28 @@ export async function restoreNativeChatReviewPresentationInTransaction( assessmentId: input.assessmentId, externalChatReviewPresentation: { ...proof, decisionId: input.decisionId }, }; + const runOwner = and( + eq(heartbeatRuns.id, input.runId), + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.nativeIssueId, input.issueId), + eq(heartbeatRuns.runtimeMode, "native"), + ); + const projectedResult = sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(projection)}::jsonb`; + const [run] = await db.select({ + needsUpdate: sql`${projectedResult} is distinct from ${heartbeatRuns.resultJson}`, + }).from(heartbeatRuns).where(runOwner).for("update"); + if (!run) return false; + // A valid existing projection still permits the caller to materialize its + // response. Replaying recovery must not rewrite it or signal another scan. + if (!run.needsUpdate) return true; + await notifyChatRunMilestones(db, input); const [updated] = await db .update(heartbeatRuns) .set({ - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(projection)}::jsonb`, + resultJson: projectedResult, updatedAt: new Date(), }) - .where( - and( - eq(heartbeatRuns.id, input.runId), - eq(heartbeatRuns.companyId, input.companyId), - eq(heartbeatRuns.nativeIssueId, input.issueId), - eq(heartbeatRuns.runtimeMode, "native"), - ), - ) + .where(runOwner) .returning({ id: heartbeatRuns.id }); return Boolean(updated); } diff --git a/server/src/services/native-runtime/native-review-dispatch.ts b/server/src/services/native-runtime/native-review-dispatch.ts index f51389d53e8..76799a7ebbd 100644 --- a/server/src/services/native-runtime/native-review-dispatch.ts +++ b/server/src/services/native-runtime/native-review-dispatch.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { withExternalAdmissionGuard, assertNoExternalOverlap } from "../../modules/external-agents/index.js"; import { agentWakeupRequests, heartbeatRuns, issues, type Db } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; @@ -40,6 +41,7 @@ export async function claimQueuedNativeReviewRun(db: Db, input: { contextSnapshot: context, agentNameKey: input.agentNameKey, claimedAt: input.claimedAt, }); if (!locked) return null; + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId }); const [claimed] = await tx.update(heartbeatRuns).set({ ...input.claimValues, status: "running", startedAt: run.startedAt ?? input.claimedAt, updatedAt: input.claimedAt, diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 88efc195b86..04d1dbef81b 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -1,4 +1,5 @@ import { pendingNativeGovernance } from "./native-governance.js"; +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { onboardingFirstTaskCompletionAllowed } from "../onboarding-first-task-admission.js"; import { notifyExecutionRunWork } from "../execution-control-notifications.js"; import { reconcileMuseNativeTerminalRun } from "./muse-terminal-reconciliation.js"; @@ -377,6 +378,7 @@ async function recordRetryableFailure(input: { : input.failureCode; if (phase === "terminal_failure") await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId); await notifyExecutionRunWork(tx); + await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); await tx .update(nativeRunFinalizations) .set({ @@ -666,6 +668,7 @@ async function projectCommittedRun(input: { const tx = transaction as unknown as Db; await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, input.run.companyId); await notifyExecutionRunWork(tx); + await notifyChatRunMilestones(tx, { companyId: input.run.companyId, issueId: input.coordinator.issueId }); return tx .update(heartbeatRuns) .set({ @@ -965,6 +968,7 @@ export async function repairCommittedNativeChatResponse( const authorizationReason = await resolveChatRunPresentationAuthorizationReason(tx, input); const resolved = resolveHeartbeatRunResponse({ resultJson: monitorResult }); if (!resolved.text) return false; + await notifyChatRunMilestones(tx, input); const comment = await issueService(db).addComment(input.issueId, resolved.text, { agentId: run.agentId, runId: run.id }, { authorizationReason, completionReply: true }, tx); await tx.update(heartbeatRuns).set({ @@ -1019,6 +1023,7 @@ export async function repairCommittedNativeChatResponse( finalAgentMessage, }); if (!resolved.text || resolved.decision.commentAction !== "create") return false; + await notifyChatRunMilestones(tx, input); const comment = await issueService(db).addComment(input.issueId, resolved.text, { agentId: run.agentId, runId: run.id }, { authorizationReason: "internal_agent_write", completionReply: true }, tx); @@ -1062,6 +1067,7 @@ export async function repairCommittedNativeChatResponse( }); if (!resolved.text || resolved.decision.commentAction !== "create") return false; + await notifyChatRunMilestones(tx, input); const comment = await issueService(db).addComment( input.issueId, resolved.text, @@ -1678,6 +1684,7 @@ export async function finalizeNativeRun(input: { const tx = transaction as unknown as Db; if (input.projectRunStatus) await notifyDeliveryWork(tx, DELIVERY_QUEUES.accounting, run.companyId); await notifyExecutionRunWork(tx); + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); return projectRun(tx); }); if ( diff --git a/server/src/services/native-runtime/obsolete-policy-reviews.ts b/server/src/services/native-runtime/obsolete-policy-reviews.ts index e43262875ef..4330f3d8f0f 100644 --- a/server/src/services/native-runtime/obsolete-policy-reviews.ts +++ b/server/src/services/native-runtime/obsolete-policy-reviews.ts @@ -8,6 +8,7 @@ import { logger } from "../../middleware/logger.js"; import { issueService } from "../issues.js"; import { issueThreadInteractionService } from "../issue-thread-interactions.js"; import { enqueueTerminalIssueInteractionChatPublications } from "../chat-interaction-publications.js"; +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { persistActivity, publishActivity, type ActivityPublication } from "../activity-log.js"; const obsoletePrompt = "Review the superseding native policy assessment."; @@ -68,6 +69,7 @@ export async function dismissObsoleteNativePolicyReviews(db: Db, runIds?: string eq(issues.id, decision.issueId), eq(issues.companyId, decision.companyId), )).for("update").then((rows) => rows[0]); if (!issue) return; + await notifyChatRunMilestones(tx, interaction); const now = new Date(); const [cancelled] = await tx.update(issueThreadInteractions).set({ status: "cancelled", diff --git a/server/src/services/native-runtime/status-decision-committer.ts b/server/src/services/native-runtime/status-decision-committer.ts index f19bb8cc244..9c74f4d765c 100644 --- a/server/src/services/native-runtime/status-decision-committer.ts +++ b/server/src/services/native-runtime/status-decision-committer.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { scheduledIssueMonitorWait } from "../issue-monitors.js"; import { onboardingFirstTaskCompletionAllowed } from "../onboarding-first-task-admission.js"; import { hasPublishedGitHubReview, readInheritedGitHubReviewWait } from "../chat-channels/github/review-wait.js"; @@ -127,6 +128,7 @@ export async function dispatchPendingNativeStatusEffects(input: { targetId: effect.targetId, }); } + await notifyChatRunMilestones(tx, input); await tx .update(statusDecisionEffects) .set({ @@ -1973,6 +1975,7 @@ export async function commitNativeStatusDecision(input: { await recordCoordinatorDecision(decisionRow.id); return { decision: decisionRow, issue, replayed: true }; } + await notifyChatRunMilestones(tx, input); if (!decisionRow) { const latestDecisionVersion = await tx .select({ diff --git a/server/src/services/question-response-delivery.ts b/server/src/services/question-response-delivery.ts index dad35c77d7e..6e85adee7ce 100644 --- a/server/src/services/question-response-delivery.ts +++ b/server/src/services/question-response-delivery.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { and, asc, desc, eq, inArray, isNull, lte, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { @@ -454,6 +455,9 @@ export function questionResponseDeliveryService( current.lastAttemptAt.getTime() > claimAt.getTime() - claimStaleMs ) return null; + if (current.status !== "delivering") { + await notifyChatRunMilestones(tx, { companyId: current.companyId, issueId: current.issueId }); + } return tx .update(issueQuestionResponseDeliveries) .set({ @@ -504,6 +508,7 @@ export function questionResponseDeliveryService( }): Promise { const at = now(); const updated = await db.transaction(async (tx) => { + await notifyChatRunMilestones(tx, { companyId: input.delivery.companyId, issueId: input.delivery.issueId }); const row = await tx .update(issueQuestionResponseDeliveries) .set({ @@ -591,25 +596,28 @@ export function questionResponseDeliveryService( const nextErrorCount = delivery.errorCount + (options.bounded ? 1 : 0); const exhausted = options.bounded && nextErrorCount >= MAX_DELIVERY_ATTEMPTS; - await db - .update(issueQuestionResponseDeliveries) - .set({ - // Keep an exhausted claim owned until recordTerminal commits its outcome. - status: exhausted ? "delivering" : "pending", - ...(options.bounded ? { errorCount: nextErrorCount } : {}), - lastErrorCode: errorCode, - updatedAt: at, - }) - .where( - and( - eq(issueQuestionResponseDeliveries.id, delivery.id), - eq(issueQuestionResponseDeliveries.status, "delivering"), - eq( - issueQuestionResponseDeliveries.attemptCount, - delivery.attemptCount, - ), - ), + await db.transaction(async (tx) => { + const ownedClaim = and( + eq(issueQuestionResponseDeliveries.id, delivery.id), + eq(issueQuestionResponseDeliveries.status, "delivering"), + eq(issueQuestionResponseDeliveries.attemptCount, delivery.attemptCount), ); + const [current] = await tx.select({ id: issueQuestionResponseDeliveries.id }) + .from(issueQuestionResponseDeliveries).where(ownedClaim).for("update"); + if (!current) return; + if (!exhausted) { + await notifyChatRunMilestones(tx, { companyId: delivery.companyId, issueId: delivery.issueId }); + } + await tx.update(issueQuestionResponseDeliveries) + .set({ + // Keep an exhausted claim owned until recordTerminal commits its outcome. + status: exhausted ? "delivering" : "pending", + ...(options.bounded ? { errorCount: nextErrorCount } : {}), + lastErrorCode: errorCode, + updatedAt: at, + }) + .where(ownedClaim); + }); return exhausted; } @@ -1301,21 +1309,32 @@ export function questionResponseDeliveryService( async function sweepPending(limit = 50) { const sweepAt = now(); const staleAt = new Date(sweepAt.getTime() - claimStaleMs); - await db - .update(issueQuestionResponseDeliveries) - .set({ - status: "pending", - updatedAt: sweepAt, - }) - .where( - and( - eq(issueQuestionResponseDeliveries.status, "delivering"), - or( - isNull(issueQuestionResponseDeliveries.lastAttemptAt), - lte(issueQuestionResponseDeliveries.lastAttemptAt, staleAt), - ), + await db.transaction(async (tx) => { + const staleClaims = await tx.select({ + id: issueQuestionResponseDeliveries.id, + companyId: issueQuestionResponseDeliveries.companyId, + issueId: issueQuestionResponseDeliveries.issueId, + }).from(issueQuestionResponseDeliveries).where(and( + eq(issueQuestionResponseDeliveries.status, "delivering"), + or( + isNull(issueQuestionResponseDeliveries.lastAttemptAt), + lte(issueQuestionResponseDeliveries.lastAttemptAt, staleAt), ), - ); + )).orderBy(asc(issueQuestionResponseDeliveries.id)).for("update"); + if (staleClaims.length === 0) return; + const notified = new Set(); + for (const claim of staleClaims) { + const scope = `${claim.companyId}:${claim.issueId}`; + if (notified.has(scope)) continue; + await notifyChatRunMilestones(tx, claim); + notified.add(scope); + } + for (let offset = 0; offset < staleClaims.length; offset += 500) { + await tx.update(issueQuestionResponseDeliveries) + .set({ status: "pending", updatedAt: sweepAt }) + .where(inArray(issueQuestionResponseDeliveries.id, staleClaims.slice(offset, offset + 500).map(claim => claim.id))); + } + }); const ids = await db .select({ interactionId: issueQuestionResponseDeliveries.interactionId }) .from(issueQuestionResponseDeliveries) diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 4b21e406c4e..ae7048a1534 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -1,3 +1,5 @@ +import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; +import { boundRunIssueId } from "../run-task-binding.js"; import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js"; import { recoveryRunResultColumn } from "./run-result-read.js"; import { readLegacyDispositionSource } from "./legacy-disposition-read.js"; @@ -4340,6 +4342,7 @@ export function recoveryService( } async function persistAdapterFailureRecoveryClassification( + issue: Pick, latestRun: NonNullable, classification: NonNullable, ): Promise> { @@ -4351,17 +4354,20 @@ export function recoveryService( const classificationMetadata = withAdapterFailureRecoveryClassification( { ...latestRun, resultJson: {} }, classification, ).resultJson; - const [updated] = await db - .update(heartbeatRuns) - .set({ - errorCode: classifiedRun.errorCode, - // Classification owns retry metadata, not workspace repair receipts - // that may have committed after this run was selected. - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(classificationMetadata)}::jsonb`, - updatedAt: new Date(), - }) - .where(eq(heartbeatRuns.id, latestRun.id)) - .returning({ resultJson: heartbeatRuns.resultJson, errorCode: heartbeatRuns.errorCode }); + const [updated] = await db.transaction(async tx => { + await notifyChatRunMilestones(tx, { companyId: issue.companyId, issueId: issue.id }); + return tx + .update(heartbeatRuns) + .set({ + errorCode: classifiedRun.errorCode, + // Classification owns retry metadata, not workspace repair receipts + // that may have committed after this run was selected. + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify(classificationMetadata)}::jsonb`, + updatedAt: new Date(), + }) + .where(and(eq(heartbeatRuns.id, latestRun.id), eq(heartbeatRuns.companyId, issue.companyId))) + .returning({ resultJson: heartbeatRuns.resultJson, errorCode: heartbeatRuns.errorCode }); + }); return updated ? { ...classifiedRun, ...updated } : classifiedRun; } @@ -4960,6 +4966,7 @@ export function recoveryService( }); if (monitored) { latestRun = await persistAdapterFailureRecoveryClassification( + issue, latestRun, adapterFailureClassification, ); @@ -4981,6 +4988,7 @@ export function recoveryService( }); if (updated) { latestRun = await persistAdapterFailureRecoveryClassification( + issue, latestRun, adapterFailureClassification, ); @@ -5184,6 +5192,7 @@ export function recoveryService( }); if (monitored) { latestRun = await persistAdapterFailureRecoveryClassification( + issue, participantLatestRun, participantAdapterFailureClassification, ); @@ -5210,6 +5219,7 @@ export function recoveryService( }); if (updated) { latestRun = await persistAdapterFailureRecoveryClassification( + issue, participantLatestRun, participantAdapterFailureClassification, ); @@ -6179,70 +6189,80 @@ export function recoveryService( await deps.beforeOrphanedRunTerminalWrite?.(run.id); const now = new Date(); - const updated = await db - .update(heartbeatRuns) - .set({ - status: terminalStatus, - finishedAt: run.finishedAt ?? now, - error: run.error ?? (terminalStatus === "interrupted" ? message : null), - errorCode: - run.errorCode ?? - (terminalStatus === "interrupted" ? errorCode : null), - updatedAt: now, - }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - eq(heartbeatRuns.runtimeMode, run.runtimeMode), - nativeRunnerOwnershipNotHeldCondition(), - // Provider exit is expected while the native coordinator resumes a - // session or copies its completed workspace back. The coordinator - // owns those retries, including expired leases and future attempts. - // Check at the write so a newly recorded result cannot be orphaned - // using the earlier liveness snapshot. Terminal issue status remains - // the stronger authority. - !issueTerminalStatus && run.runtimeMode === "native" - ? notExists(db.select({ runId: nativeRunFinalizations.runId }) - .from(nativeRunFinalizations).where(and( - eq(nativeRunFinalizations.runId, heartbeatRuns.id), - eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), - or( - and( - isNotNull(nativeRunFinalizations.resultId), - inArray(nativeRunFinalizations.phase, [ - "observed", "workspace_finalizing", "ready_for_assessment", - "arbitrating", "retryable_failure", - ]), - ), - and( - isNull(nativeRunFinalizations.resultId), - or( - eq(nativeRunFinalizations.phase, "retryable_failure"), - and(eq(nativeRunFinalizations.phase, "observed"), gt(nativeRunFinalizations.attempt, 0)), - ), + const updated = await db.transaction(async tx => { + const eligible = and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + eq(heartbeatRuns.runtimeMode, run.runtimeMode), + nativeRunnerOwnershipNotHeldCondition(), + // Provider exit is expected while the native coordinator resumes a + // session or copies its completed workspace back. The coordinator + // owns those retries, including expired leases and future attempts. + // Check at the write so a newly recorded result cannot be orphaned + // using the earlier liveness snapshot. Terminal issue status remains + // the stronger authority. + !issueTerminalStatus && run.runtimeMode === "native" + ? notExists(tx.select({ runId: nativeRunFinalizations.runId }) + .from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), + or( + and( + isNotNull(nativeRunFinalizations.resultId), + inArray(nativeRunFinalizations.phase, [ + "observed", "workspace_finalizing", "ready_for_assessment", + "arbitrating", "retryable_failure", + ]), + ), + and( + isNull(nativeRunFinalizations.resultId), + or( + eq(nativeRunFinalizations.phase, "retryable_failure"), + and(eq(nativeRunFinalizations.phase, "observed"), gt(nativeRunFinalizations.attempt, 0)), ), ), - ))) - : undefined, - // Recheck ownership in the write: a controller can renew or claim - // the run after the liveness read. An old snapshot cannot end a new - // controller's run, even if that controller's lease later expires. - run.runtimeMode === "legacy" - ? and( - run.controllerBootId - ? eq(heartbeatRuns.controllerBootId, run.controllerBootId) - : isNull(heartbeatRuns.controllerBootId), - or( - isNull(heartbeatRuns.controllerBootId), - sql`${heartbeatRuns.controllerLeaseExpiresAt} <= clock_timestamp()`, ), - ) - : undefined, - ), - ) - .returning() - .then((rows) => rows[0] ?? null); + ))) + : undefined, + // Recheck ownership in the write: a controller can renew or claim + // the run after the liveness read. An old snapshot cannot end a new + // controller's run, even if that controller's lease later expires. + run.runtimeMode === "legacy" + ? and( + run.controllerBootId + ? eq(heartbeatRuns.controllerBootId, run.controllerBootId) + : isNull(heartbeatRuns.controllerBootId), + or( + isNull(heartbeatRuns.controllerBootId), + sql`${heartbeatRuns.controllerLeaseExpiresAt} <= clock_timestamp()`, + ), + ) + : undefined, + ); + // Do not wake projection for a retained native finalization or a run + // whose execution owner changed. Keep the same predicate on the write + // to recheck finalization records committed after this lock was taken. + const [candidate] = await tx.select().from(heartbeatRuns).where(eligible).for("update"); + if (!candidate) return null; + const boundIssueId = boundRunIssueId(candidate); + if (boundIssueId) { + await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: boundIssueId }); + } + return tx + .update(heartbeatRuns) + .set({ + status: terminalStatus, + finishedAt: run.finishedAt ?? now, + error: run.error ?? (terminalStatus === "interrupted" ? message : null), + errorCode: + run.errorCode ?? + (terminalStatus === "interrupted" ? errorCode : null), + updatedAt: now, + }) + .where(eligible) + .returning() + .then((rows) => rows[0] ?? null); + }); if (!updated) { // Another path finalized the run between the read and this write. Keep // that terminal outcome authoritative. diff --git a/server/src/services/run-identity.ts b/server/src/services/run-identity.ts index f72cc974ca7..9b8d845c098 100644 --- a/server/src/services/run-identity.ts +++ b/server/src/services/run-identity.ts @@ -11,7 +11,8 @@ import { type Db, } from "@paperclipai/db"; import { conflict, forbidden } from "../errors.js"; -import { boundRunIssueId, runTaskContextColumn } from "./run-task-binding.js"; +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; +import { boundRunIssueId, boundRunIssueIdSql, runTaskContextColumn } from "./run-task-binding.js"; import { isUuidLike } from "@paperclipai/shared"; import { queuedCommentIdsFromRunContext, queuedCommentIdsFromWakePayload } from "./issue-queued-comment-queue.js"; @@ -138,6 +139,13 @@ async function append( } async function activate(executor: Executor, context: RunIdentityContext) { + const [binding] = await executor.select({ issueId: boundRunIssueIdSql() }) + .from(heartbeatRuns).where(and( + eq(heartbeatRuns.id, context.runId), eq(heartbeatRuns.companyId, context.companyId), + )); + if (binding?.issueId) { + await notifyChatRunMilestones(executor, { companyId: context.companyId, issueId: binding.issueId }); + } const [run] = await executor .update(heartbeatRuns) .set({ @@ -447,6 +455,7 @@ export async function rejectSteeredIdentity( ); } +/** Accept within the caller's task/run transaction; standalone recovery uses reconcileSteeredIdentity. */ export async function acceptSteeredIdentity( executor: Executor, context: RunIdentityContext, diff --git a/server/src/services/secret-proposals.ts b/server/src/services/secret-proposals.ts index 05f5af71d91..870749d1513 100644 --- a/server/src/services/secret-proposals.ts +++ b/server/src/services/secret-proposals.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { withAgentAppearance } from "@paperclipai/shared"; @@ -290,6 +291,7 @@ export function createSecretProposalsService(db: Db) { : outcome === "rejected" ? "rejected" : "withdrawn"; + if (current.status !== status) await notifyChatRunMilestones(txDb, current); await txDb .update(issueThreadInteractions) .set({ diff --git a/server/src/services/tool-access.ts b/server/src/services/tool-access.ts index 45237155713..81b6f646c3c 100644 --- a/server/src/services/tool-access.ts +++ b/server/src/services/tool-access.ts @@ -4665,7 +4665,7 @@ export function toolAccessService( .returning(); const [row] = existing.transport === "chat_sdk" ? await db.transaction(async tx => { - await notifyChatEndpointWork(tx, existing.id); + await notifyChatEndpointWork(tx, existing.id, existing.companyId); const rows = await write(tx as unknown as Db); // Credential resolution must never observe the binding rebuild's // delete/insert gap after a configuration wake. diff --git a/server/src/services/tool-access/connections/removal.ts b/server/src/services/tool-access/connections/removal.ts index 467a67dad62..a9c201f9984 100644 --- a/server/src/services/tool-access/connections/removal.ts +++ b/server/src/services/tool-access/connections/removal.ts @@ -541,7 +541,7 @@ async function removeConnection( .returning({ id: connectionTokenIssuances.id }); const archived = await db.transaction(async (tx) => { - if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id); + if (connection.transport === "chat_sdk") await notifyChatEndpointWork(tx, connection.id, connection.companyId); if (connection.transport === "voice") await notifyVoiceChange(tx, { kind: "connection", id: connection.id }); await notifyToolConnectionHealth(tx, connection); await tx.select({ id: toolApplications.id }).from(toolApplications) diff --git a/server/src/services/tool-action-review.ts b/server/src/services/tool-action-review.ts index 664b4cdc194..59502b42c17 100644 --- a/server/src/services/tool-action-review.ts +++ b/server/src/services/tool-action-review.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { notifyToolReviewChange } from "./tool-review-work-notifications.js"; import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import { and, eq } from "drizzle-orm"; @@ -141,6 +142,7 @@ export async function commitToolActionReview( throw conflict("This review has already been resolved"); } const now = new Date(); + if (interaction && issue) await notifyChatRunMilestones(tx, interaction); await notifyToolReviewChange(tx, current.companyId); const [updated] = await tx .update(toolActionRequests) diff --git a/server/src/services/tool-gateway.ts b/server/src/services/tool-gateway.ts index bdfb0f8b0f4..fb023ba7f6d 100644 --- a/server/src/services/tool-gateway.ts +++ b/server/src/services/tool-gateway.ts @@ -1,3 +1,4 @@ +import { notifyChatRunMilestones } from "./chat-run-milestone-notifications.js"; import { writeToolConnectionHealthChange } from "./tool-connection-health-notifications.js"; import { writeApprovalChange } from "./approval-work-notifications.js"; import { toolReviewProjectionService } from "./tool-review-projection.js"; @@ -2408,6 +2409,9 @@ export function createToolGatewayService( && projection.resultSummary === (input.resultSummary ?? null)) return false; const now = new Date(); + if (linked.issueId && interaction.status === "pending" && ["expired", "cancelled"].includes(input.status)) { + await notifyChatRunMilestones(tx, { companyId: linked.companyId, issueId: linked.issueId }); + } await tx .update(issueThreadInteractions) .set({