mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-01 23:43:16 +01:00
merge: preserve GitHub receipt fences and scoped confirmation effects
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
3e8f00cce2
247 files changed
+11564
-1589
No files matched your search
@@ -11,7 +11,7 @@ const script = job.split(" run: |\n")[1].split("\n").map(line => line.rep
|
||||
const sha = "a".repeat(40);
|
||||
|
||||
test("canary promotion waits for the standard manifest and keeps its serialized channel", () => {
|
||||
assert.match(job, /needs: \[merge-and-push\]/);
|
||||
assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/);
|
||||
assert.match(job, /group: docker-canary-channel-promotion/);
|
||||
assert.match(job, /cancel-in-progress: false/);
|
||||
});
|
||||
|
||||
@@ -33,6 +33,35 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
reuse_canary_image:
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
attestations: read
|
||||
outputs:
|
||||
reused: ${{ steps.reuse.outputs.reused }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Reuse the verified master image when present
|
||||
id: reuse
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: node scripts/reuse-canary-image.mjs
|
||||
|
||||
# Multi-arch by native runner, not QEMU.
|
||||
#
|
||||
# This was one job building linux/amd64,linux/arm64 together on an x86
|
||||
@@ -56,6 +85,8 @@ jobs:
|
||||
# digest; `merge` assembles the manifest list. Both architectures remain
|
||||
# available to self-hosted installations and downstream image composers.
|
||||
build-and-push:
|
||||
needs: [reuse_canary_image]
|
||||
if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }}
|
||||
strategy:
|
||||
# Independent legs: one architecture failing should still publish
|
||||
# nothing, but it must not also hide the other's logs behind a
|
||||
@@ -277,6 +308,7 @@ jobs:
|
||||
# tags. Nothing is publicly tagged until this runs, so a half-published
|
||||
# multi-arch image is not a state anything can pull.
|
||||
merge-and-push:
|
||||
if: ${{ !cancelled() && needs.build-and-push.result == 'success' }}
|
||||
needs: build-and-push
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
@@ -407,11 +439,8 @@ jobs:
|
||||
# promotion is gated on its own completed pushes). Every interleaving
|
||||
# converges the Docker channel onto the npm channel.
|
||||
promote_canary_channel:
|
||||
if: startsWith(github.ref, 'refs/tags/canary/v')
|
||||
# merge-and-push, not build-and-push: the per-arch legs push untagged
|
||||
# digests, and the production `sha-*` tags this promotion retags only
|
||||
# exist once the manifest merge has named them.
|
||||
needs: [merge-and-push]
|
||||
if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }}
|
||||
needs: [reuse_canary_image, merge-and-push]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
|
||||
@@ -202,7 +202,7 @@ jobs:
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs
|
||||
|
||||
- name: Test release verify workflow wiring
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs
|
||||
|
||||
- name: Test standalone package build concurrency
|
||||
run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs
|
||||
|
||||
+19
-4
@@ -86,7 +86,7 @@ internal traceability tag.
|
||||
moves them to the versioned name
|
||||
- canaries, nightlies, and betas never create GitHub Releases
|
||||
- canaries, nightlies, and betas never require changelog generation
|
||||
- Docker `:latest` moves only on stable releases; master builds publish
|
||||
- Docker `:latest` moves only on stable releases; canary dispatches promote
|
||||
`:canary`, nightly builds `:nightly`, and beta builds `:beta`
|
||||
|
||||
## TL;DR
|
||||
@@ -310,12 +310,12 @@ The workflow:
|
||||
|
||||
## Docker Image Tags
|
||||
|
||||
[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted
|
||||
image and the `-cloud` variant with the same lane mapping:
|
||||
[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image:
|
||||
|
||||
| Build ref | Tags |
|
||||
| --- | --- |
|
||||
| `master` push | `:canary`, `:sha-<short>` |
|
||||
| `master` push | `:sha-<short>`, `:sha-<full>` |
|
||||
| `canary/v*` dispatch | `:sha-<short>`, then `:canary` for the current npm canary |
|
||||
| `nightly/v*` tag | `:nightly`, `:sha-<short>` |
|
||||
| `beta/v*` tag | `:beta`, `:sha-<short>` |
|
||||
| `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-<short>` |
|
||||
@@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs
|
||||
therefore dispatch `docker.yml` explicitly at the new tag ref; the tag
|
||||
mapping keys off `github.ref` either way.
|
||||
|
||||
Canary dispatches first look up the full-SHA master image. Reuse requires a
|
||||
valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching
|
||||
registry digest, and signed provenance from this repository's `docker.yml`
|
||||
on `master` at the exact source commit. The image is retagged by immutable
|
||||
digest. This preserves the original build inputs and skips both native builds
|
||||
and the manifest merge. A missing full-SHA image runs the normal build instead;
|
||||
registry errors or invalid provenance fail the run. Rerun after the master
|
||||
image's attestation finishes if the image became visible before its proof.
|
||||
|
||||
Both paths retain the serialized promotion that resolves the current npm
|
||||
`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and
|
||||
master builds retain their existing build paths. Downstream image builds can
|
||||
continue to consume the signed full-SHA master image independently of npm
|
||||
canary publication.
|
||||
|
||||
The Docker build matrix and the release preview image builder configure
|
||||
BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers
|
||||
both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as
|
||||
|
||||
@@ -76,6 +76,12 @@ one bound continuation, and an exact saved confirmation. Its four development
|
||||
cells use the recorded Codex and Claude models. Registration and calibration do
|
||||
not establish Slack, staging authentication, warm lifecycle, or live qualification.
|
||||
|
||||
The explicit `followup-while-running` [Product journey](../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up)
|
||||
checks a saved message during independently observed native work, one attributed
|
||||
reply with the late requirement, and persistence after reload. Its attached and
|
||||
managed targets share the same story and permit one steered run or one normal
|
||||
successor. Registration and calibration are separate from a live staging pass.
|
||||
|
||||
[Reusable Product journeys](../tests/runner-e2e/PRODUCT-JOURNEYS.md) add explicit
|
||||
deployment targets and starting-state recipes. The shared task-properties
|
||||
journey uses no model; Cloud onboarding supplies reusable verified state. Cloud
|
||||
|
||||
@@ -101,6 +101,17 @@ The manager permits at most two outstanding inspection RPCs across workers.
|
||||
Caller timeouts retain their slots until the actual worker reply or exit;
|
||||
timeouts cannot accumulate additional provider reads in later sweeps.
|
||||
|
||||
If a failed ephemeral lease has no usable local inspection path, a managed
|
||||
Daytona instance can ask Cloud for a cleanup-only absence receipt. Cloud checks
|
||||
the stack's historical credential organization and reads the exact allocation
|
||||
with its manager credential. Only confirmed absence can produce a signed receipt
|
||||
bound to the stack, company, environment, lease, provider handle and nullable run.
|
||||
The separate cleanup type and audience cannot authorize run recovery. Older
|
||||
Cloud servers, unsupported providers, invalid signatures, changed organizations
|
||||
and failed reads leave the lease unresolved. Existing bounded cleanup batches
|
||||
and retry deadlines own these reads; no new idle polling is added. An explicit
|
||||
local provider observation is retained rather than replaced by this fallback.
|
||||
|
||||
Host-owned verification metadata binds the company, environment, resource,
|
||||
provider handle, optional run, and exact row version. Concurrent resume,
|
||||
replacement, cleanup, or configuration changes invalidate verification.
|
||||
|
||||
@@ -328,6 +328,14 @@ once known, `paperclip.native_reconciliation.candidates` (a count). A failed
|
||||
pass marks the pass span and the phase that was running with an error status.
|
||||
Faster passes record nothing.
|
||||
|
||||
Heartbeat configuration preparation overlaps independent reads. Skill preparation
|
||||
runs alongside connector assignments; standing connection instructions wait for
|
||||
assignment catalog repairs, but can overlap the skill branch. These spans can
|
||||
finish out of order. Do not add their durations as serial elapsed time. The
|
||||
historical `skills.prepare` span ends when the skill branch settles, rather than
|
||||
when its concurrent branches finish. Each preparation attempt settles its started
|
||||
branches before it returns an error or releases acquired credentials.
|
||||
|
||||
### Detailed execution phases
|
||||
|
||||
The `paperclip.execution` tracer adds these content-free spans beneath the current
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Archive attention sources without repeated feed scans
|
||||
|
||||
Attention retention now reconstructs source deadlines at startup and after committed source changes. It no longer rebuilds every active company's full feed on each heartbeat tick. The archive age remains 90 days. Keep and manual archive state remain authoritative, and automatic archive still commits its notification outbox atomically.
|
||||
|
||||
This document records source ownership, producer coverage, and the evidence needed to review the replacement. It makes no claim about host suspension or external wake alarms.
|
||||
|
||||
## Existing source rows own eligibility and deadlines
|
||||
|
||||
`attention-retention.ts` selects at most 100 unfinished source candidates per reconstruction page. It excludes inactive companies and kept or archived retention states. The scoped coordinator retains eligible deadlines, retries concrete failures, and cancels entries that leave attention. It does not schedule another scan after settlement unless a producer changes a relevant source.
|
||||
|
||||
| Source | Eligible durable state | Activity owner |
|
||||
| --- | --- | --- |
|
||||
| Decision | Open decision with a visible execution issue | Decision updated time |
|
||||
| Interaction | Pending board-visible interaction; newest ordinary confirmation or independent quality gate | Interaction updated time |
|
||||
| Recovery action | Active or escalated action owned by a user or board, with a visible source issue | Recovery updated time |
|
||||
| Blocker attention | Board unblock descriptor or non-live terminal blocker selected by the issue owner | Blocked transition or terminal issue updated time |
|
||||
| Review | Human participant, user assignee, pending approval, or stalled review selected by the issue owner | Issue updated time |
|
||||
| Failed run | Failed or timed-out run with an exhaustion event, eligible agent and issue, and no newer matching run | Finished time, falling back to updated or created time |
|
||||
| Budget alert | Open hard incident, or soft incident meeting the existing 85 percent rounded threshold | Incident updated time |
|
||||
| Agent error | Agent in error status | Agent updated time |
|
||||
|
||||
The feed and worker share confirmation collapse, newer-run probes, issue visibility, review eligibility, and terminal-blocker predicates. Issue graph and agent invokability logic remain with their existing owners. Blocker entries retain both the root used to reconstruct the graph and the terminal subject that owns retention state. PostgreSQL deadlines round submillisecond timestamps upward. The worker locks the source and terminal subject before archival.
|
||||
|
||||
`productivity_review` generates no feed items and owes no retention work. Queue retention-day overrides control shelf presentation, which is computed on reads. They do not change the fixed automatic archive age. Approval retention, join-request retention, and archive notification delivery keep their existing workers.
|
||||
|
||||
## Existing mutation boundaries reactivate work
|
||||
|
||||
Committed hints identify a company, source kind, source ID or changed issue where those values are known. Routine issue writers pass identity to the retention domain helper before their mutation. The helper captures old and new parent paths, reverse blocker ancestors, run-linked issues and liveness-origin references; issue-scoped source queries then select reviews, blockers, interactions, decisions, recovery actions and failed runs by their issue identity. Budget incidents and agent errors remain tied to their own source changes. Ordinary issue writes do not reconstruct the unrelated company feed. Opaque liveness and stranded-recovery origin references resolve through company-scoped issue rows before entering UUID queries, so valid none sentinels do not roll back writes and moved recovery tasks still invalidate their original review. Queue keys explicitly pick supported identity fields. Retention's own writes use exact source hints; blocker retention changes invalidate the company/source because multiple blocked roots can project one terminal subject. Reconstruction pages and due batches remain bounded. Review attention and interaction invokability are computed in company batches per candidate page; a full 100-candidate review page uses at most 20 statements and 100 confirmations use at most six. Decisions, recovery source links, runs, eligible agents, issue visibility, newer-run probes and blocker graphs also load in company batches per page. A 100-source decision or recovery page uses at most three statements; 100 failed runs use at most six; 100 blocker roots use at most 30 graph/source statements plus retention-state reads for their emitted entries. These evidence maps exist only for one load. The blocker owner shares fetched node and edge facts while tracking depth, node count, expanded nodes and live-work reachability independently for each root. The shared graph fetch stores at most 2,000 nodes and 4,000 edges, and bounds each frontier query before materializing rows. Positive live-work evidence from the bounded returned rows is retained independently of node/edge admission, including running status, active runs and pending confirmations. Filling the graph with explicit blockers cannot hide a subsequently returned active child. An internal batch result identifies roots whose evaluation is incomplete; the legacy map API exposes no extra transport fields. Retention schedules only those roots for an immediate fresh single-root check, which disarms or saves the actual source deadline. It never archives a partial intermediary or adds a recurring incomplete-root poll. Fully evaluated roots preserve their exact classification. All later graph queries, including recovery source IDs, agent IDs and workspace-finalize inputs, use bounded chunks. A SQL-generated 70,000-node graph proves PostgreSQL parameter limits are respected at the driver boundary. Archive checks still lock and freshly re-read sources, terminal subjects and related eligibility. Ordinary execution, decision and budget mutations pass their affected company; exact decision sources are included.
|
||||
|
||||
Producer coverage includes decision creation and changes; interaction creation, resolution, expiry and supersession; issue creation, updates, blocker relations, checkout, adoption and release; agent hire, configuration, lifecycle, error clearing and deletion; budget incidents and company pause/resume; exhausted-run events and retry-run creation; native status, review, finalization and reconstruction; external conversation transitions; and imported blocker relations. Existing service transactions retain outer commit ownership. Savepoint release does not publish work before outer commit.
|
||||
|
||||
The writer contract follows the existing delivery workers: serving processes use the transaction-aware service boundaries, and each serving process registers its workers. Independent raw SQL writers must supply equivalent ingress or restart reconstruction. Restores reconstruct work at startup. The change adds no permanent listener, independent timer, table, or Cloud wake mechanism. Existing sleep-safety gates remain in force.
|
||||
|
||||
## Real PostgreSQL evidence proves quiet and reactivation
|
||||
|
||||
The new acceptance suite exercises all eight sources, including terminal blocker identity and date, newest confirmations, independent quality gates, unhealthy reporting chains, conversation visibility, maintained reviews and pending approvals, newer-run suppression, ordinary task and interaction producers, ordinary exhaustion-event ingress, and ordinary error clearing. Additional ordinary producer controls cover child creation with a parent-blocking relation, budget incident creation and resolution, and repeated company-scoped execution-control writes.
|
||||
|
||||
It also checks keep/revive, moved and cancelled deadlines, company activation and isolation, retained terminal history, restart, nested outer commit and rollback, first writes during drain, retry across drain, atomic archive/outbox rollback, and notifications during reconstruction. A 225-item backlog uses three startup pages and source-specific self-invalidations. SQL is counted at the PostgreSQL driver boundary.
|
||||
|
||||
The old aggregate path issued 12 statements per tick in each of three configured-idle samples. The replacement issued zero statements during 300 seconds after settlement. One transaction-status read after an injected rollback is classified as uncertainty reconciliation; the following 300-second drain window must remain quiet.
|
||||
|
||||
The focused suite supports `PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL`, validates its dedicated `_test` database name, applies pending migrations, and prewarms a six-connection pool before advancing fake time. Providers are not involved in retention selection; the existing archive outbox delivery tests verify the separate controlled notification boundary. Fake time proves scheduling and SQL quiet, not database or host suspension.
|
||||
|
||||
## Review and operational limits remain explicit
|
||||
|
||||
Local embedded PostgreSQL could not start on the test host because system semaphores were exhausted. The accepted runs use isolated PostgreSQL 17 databases. A mixed consumer run also exposed fixture contamination from a lifecycle suite that retains native finalization rows; isolated reruns are required and preserve the failed run in the review evidence. Broad Linux CI remains the required full-suite gate.
|
||||
|
||||
Review the source parity fixtures and producer hints together. The intended outcome is unchanged archive state and notifications with no recurring retention SQL when the sources remain unchanged. No merge or deployment is part of this change.
|
||||
|
||||
Issue producer invalidation has its own finite discovery budget. Reverse parent/blocker traversal returns at most 101 distinct UUIDs in one recursive query, with each relation adjacency capped at 101 unique edges before materialization. At most 100 affected issues produce one sorted canonical batch hint; an overflow sentinel produces one company hint and the existing complete paged reconstruction. UNION deduplication terminates cycles. A PostgreSQL EXPLAIN ANALYZE on 2,100 reverse edges proves the outer limit stops at 101 CTE rows while the indexed relation branch returns only 100 rows. Repeated identical writes coalesce by canonical key. The shared scheduler is unchanged: a broader company hint does not cancel already pending narrower scopes; those remain finite batches of at most 100 IDs, processed at the existing 25-scope pass limit. This permits redundant bounded reconstruction instead of discarding invalidations or introducing cursor/subsumption policy across other owners. A 151-node cyclic chain proves overflow still registers the far ancestor's exact archive deadline and settles quietly afterward.
|
||||
@@ -0,0 +1,63 @@
|
||||
# Backup expiration without idle polling
|
||||
|
||||
The backup expiration duty retains its existing minimum age and safety policy.
|
||||
It uses the shared scoped deadline coordinator. The archived runner evidence and
|
||||
checkpoint expiration duties keep their separate registrations.
|
||||
|
||||
The durable source is each controller-owned `failover-backups/current` directory,
|
||||
its control-plane session identity, and the associated run, issue, finalization,
|
||||
and environment lease rows. The next eligible instant is one millisecond after
|
||||
the later of the backup mtime and every related issue's close time, plus the
|
||||
configured retention duration. An open issue, live run or lease, unsettled
|
||||
finalization, or cleanup activation has no time-derived expiry; its producer
|
||||
must invalidate the source when it changes. A busy process or failed I/O leaves
|
||||
real unresolved cleanup and uses the shared retry backoff. Dry-run reports an
|
||||
eligible source once and waits for source invalidation or restart.
|
||||
|
||||
Startup pages existing disk scopes and indexes each backup's actual session,
|
||||
run, coalesced native-task/task, agent and lease references, including protected
|
||||
backups. Ordinary task, run, lease and process-release notifications retain
|
||||
these identities and inspect only affected roots. New run or lease references
|
||||
resolve through the owner's database relationships. Local publication retains
|
||||
its exact digest. Explicit company or instance configuration changes can still
|
||||
reconstruct their whole affected scope. Roots without backup or cleanup tombstones
|
||||
perform no backup SQL. After discovery and all obligations settle, there is no
|
||||
recurring SQL or filesystem scan. Startup reconstructs future work; inspecting a
|
||||
source never changes its business timestamps. Runtime task close/reopen and
|
||||
retention timestamp changes, native environment and retained session writes,
|
||||
local backup publication, and process release invalidate the company or digest.
|
||||
Signals from database producers register before writes inside their owning
|
||||
transaction, including caller-owned transactions and nested savepoints.
|
||||
|
||||
The controller remains the only supported writer of its local backup files.
|
||||
An independent process writing those files must use the same producer ingress;
|
||||
manual disk edits require reconstruction on restart. Database writer ingress
|
||||
continues to use the existing transaction-aware work signals. This change adds
|
||||
no permanent database listener and does not add a polling table or migration.
|
||||
|
||||
Read-only reconstruction does not lock issue rows. Transient identity I/O errors
|
||||
retain a retry obligation; missing or malformed state remains unrecognized.
|
||||
Byte accounting occurs before the retirement transaction. The current backup
|
||||
age is rechecked inside the transaction before the rename.
|
||||
|
||||
The existing issue share locks now remain held through the retirement rename.
|
||||
This fences task reopen while the backup becomes unavailable. A failed database
|
||||
transaction restores `.retiring-*`; a process interruption is recovered from
|
||||
that existing file journal at startup. Journal reconciliation runs before
|
||||
identity or owner lookup, so an agent purge cannot strand expired physical
|
||||
cleanup or prevent restoration of a retiring backup. After database settlement, `.expired-*`
|
||||
records outstanding physical removal so a partial delete retries idempotently.
|
||||
Ownership, session identity, live run, lease and local execution safety checks
|
||||
remain in the backup owner. No host-sleep blocker or expiry-on-use rule changes.
|
||||
An in-process deadline does not register a Cloud wake alarm.
|
||||
|
||||
Acceptance uses isolated real PostgreSQL for service writes, transaction signals,
|
||||
safety checks, resume pointer deletion, and restart reconstruction. A temporary
|
||||
filesystem is the controlled storage boundary. These tests do not claim cloud
|
||||
object storage behavior, deployed sleep, or fleet savings.
|
||||
|
||||
Review validation also covers expired-only directories whose listing fails with
|
||||
EIO or EACCES. Only ENOENT means absent; other listing failures retain retry debt.
|
||||
Both controls failed before the fix; all 104 expiration tests pass afterward.
|
||||
Executor release tests assert both the preserved broad cleanup scope and the
|
||||
precise source identity used for backup invalidation.
|
||||
@@ -0,0 +1,23 @@
|
||||
# Deferred execution receipt ownership
|
||||
|
||||
This slice removes the periodic `resumeQueuedRuns` sweep after queue dispatch migrated separately. It also delegates native status wake intents out of the periodic orphan reaper. Existing queue admission, authorization, task recovery, cancellation, resource and controller checks remain the mutation authority.
|
||||
|
||||
| Duty | Durable pending truth | Committing producers | Deadline and exit |
|
||||
| --- | --- | --- | --- |
|
||||
| Task Retry | deferred task_retry wake with certified actor/generation | requestTaskRetry through execution control; execution, issue, agent, company and budget eligibility | immediate; current admission links canonical intent to one dispatch/run, supersession or terminal policy retires it; blocked input awaits its actual gate mutation |
|
||||
| Saved comments and interaction responses | deferred wake carrying saved comment IDs or interaction mutation | wake admission insert; merge only genuinely new comment IDs or interaction identity; issue/lock/assignment/dependency and execution eligibility | immediate after terminal source; original release owner revalidates comment/card authority, process/cleanup evidence and issue claim |
|
||||
| Execution wait | deferred wake with executionWait | original admission/restored source receipt; actual recovery action, execution and eligibility changes | original updatedAt + 30 seconds, then current owner revalidation; unchanged blocked receipt carries no recurring scan |
|
||||
| Interrupt retry | deferred wake with queuedCommentInterrupt | authorized route persists interrupt marker; cancellation lifecycle and cleanup/eligibility transitions | original updatedAt + 30 seconds; cancelled legacy run naming exact queue bypasses initial grace as before; owner retains explicit interrupt and stop evidence |
|
||||
| Cancelled legacy queue | cancelled legacy run result names exact still-deferred queue | run lifecycle terminal/control commits; original interrupt intent | immediate restart reconstruction; original release/admission owner settles or leaves saved input behind its actual blocker |
|
||||
| Task chat receipt settlement | deferred task_retry with taskRetry.chatActionId | failed chat retry action settlement | immediate settlement-only pass folds linked action result; it cannot restage its own processed-but-deferred chat action. Startup/actual eligibility retains normal re-admission policy |
|
||||
| Native status wake intent | system/native-status-committer wake queued/claimed without run | status committer transaction; actual execution/issue/agent/company/budget eligibility | immediate queued intent; claimedAt + 60 seconds for crash recovery; error updatedAt + 30 seconds. Existing deterministic dispatcher identity/claims prevent duplicate delivery; original intent becomes linked/coalesced/skipped while a separate blocked dispatch receipt owns saved admission |
|
||||
|
||||
Three scoped owners reconstruct only their actual durable receipt predicates, with keyset pages of 100. Normal signals contain normalized company/request identity. Instance scheduler settings alone may request instance-wide reconstruction. Repeated source changes coalesce reconstruction for 30 seconds; already-known stored deadlines preempt that cooldown. Pending checks read memory only. Shutdown uses the existing coordinator and physical execution tracking; task drain reconciles producer transactions and releases the settled intent hold while retaining work for resume. Warm standby and disabled registration do not execute domain work.
|
||||
|
||||
The generic run-dispatch notifier and broad issue review-state notifier deliberately do not fan out deferred eligibility. Queue bookkeeping may write them while processing its own saved input. Separate eligibility commits and settlement-only chat notifications avoid a self-restaging loop.
|
||||
|
||||
C2 remains separate and mandatory: active output silence deadlines, remaining orphan/process/controller/remote authority and resource debts, legacy shutdown settlements, stranded agent status, and final aggregate chain removal. Retained workspace and migrated native finalization workers remain their existing owners.
|
||||
|
||||
Review controls cover queue edits that move an already loaded grace deadline. Each due check re-reads current stored grace or claim time before admission. Subtree hold release, subtree restore, and member archival publish eligibility inside their own transactions. Member archival preserves the cancelled Retry tombstone created by an earlier user reassignment.
|
||||
|
||||
A resume that first records a new execution-wait reason arms its existing 30-second grace once. Writing the same closed-gate reason again does not publish another hint. A clock obligation, an eligibility change, and an infrastructure failure therefore retain separate exits: stored deadline, owning committed producer, and bounded failure backoff. Refreshing a diagnostic timestamp alone does not create permanent polling behind a deliberate pause.
|
||||
@@ -0,0 +1,124 @@
|
||||
# External object refresh deadlines
|
||||
|
||||
External-object refresh uses the delivery coordinator. The server registers the
|
||||
service in both heartbeat-scheduler configurations. It no longer calls the
|
||||
company-wide aggregate sweep on each heartbeat tick.
|
||||
|
||||
`external-objects.ts` owns automatic eligibility, resolver selection, refresh
|
||||
claims, provider results and the next deadline. `index.ts` supplies registration
|
||||
and admission gates. The existing scoped deadline helper owns wake coalescing,
|
||||
page reconciliation, fair company selection, retry backoff and shutdown.
|
||||
|
||||
## Durable obligations and quiet states
|
||||
|
||||
An automatic refresh needs an active company, a retained mention in that company,
|
||||
a non-null `next_refresh_at` and an available resolver. Injected resolvers,
|
||||
including custom `url/link` resolvers, remain supported. A plugin resolver needs a
|
||||
ready plugin with `external.objects.read`, a matching provider/object declaration,
|
||||
and no company setting that disables it. Built-in resolvers remain available
|
||||
independently of optional plugin support.
|
||||
|
||||
A plugin result carries its selected plugin ID inside the host owner. Persistence
|
||||
rechecks that exact plugin's readiness, read capability, object declaration and
|
||||
same-company enabled setting in the result update. Another available resolver
|
||||
cannot authorize an obsolete plugin response. A rejected response releases only
|
||||
its own claim; the due row remains for a fresh allowed resolver. Plugin RPC
|
||||
payloads and public result shapes do not contain this host selection context.
|
||||
|
||||
|
||||
Terminal objects are passive. The existing exception remains: GitHub
|
||||
`not_found` snapshots without `data.probeVerified=true` need a repository-visibility
|
||||
probe before their terminal verdict is definitive. A null deadline retains its
|
||||
existing never-refresh meaning. Registration does not change stored timestamps.
|
||||
|
||||
The worker reconstructs eligible deadlines in pages of at most 250 rows. A
|
||||
company notification reads that company only. Ordinary ingestion uses object IDs
|
||||
or stable provider/object/external identity hints, including concurrent upserts,
|
||||
so one mention write does not reconstruct all company history. A due callback reads only its
|
||||
object. Pages select scheduling columns and retain compact deadlines in memory.
|
||||
Dispatch uses fair batches of at most 50 objects. The deadline is the later of
|
||||
`next_refresh_at` and the current refresh lease expiry. Lease renewals and atomic
|
||||
refresh tokens continue to exclude another claimant.
|
||||
|
||||
Unsupported generic links, orphan objects, disabled plugins, empty configuration
|
||||
and retained terminal history do not schedule fallback discovery. Manual refresh
|
||||
retains its existing `no_resolver` result. Actual provider failures retain their
|
||||
saved retry deadline. Unexpected failures retain scoped coordinator backoff;
|
||||
restart reconstructs the saved object and any unexpired claim.
|
||||
|
||||
The supported-resolver and retained-reference approach follows the related draft
|
||||
[PR #17298](https://github.com/paperclipai/paperclip/pull/17298) by
|
||||
[@kaluli123123](https://github.com/kaluli123123). This change adapts that eligibility
|
||||
pattern to scoped scheduling, adds dynamic plugin/company permission predicates,
|
||||
and fences automatic result persistence. It does not modify that draft PR.
|
||||
|
||||
## Producer and transaction audit
|
||||
|
||||
| Mutation | Owner and activation |
|
||||
| --- | --- |
|
||||
| Issue title/description, comment and document references | Existing sync methods register object intent before mention replacement and stable identity intent before object upsert. Routes, runtime tools, adapters and plugin paths reuse these methods. |
|
||||
| Comment/document disappearance | Sync derives the retained mention owners and registers only those companies before deleting obsolete mentions. |
|
||||
| Last mention removal and reattachment | Mention replacement invalidates company deadlines. Retained objects stay available for reattachment. |
|
||||
| Issue deletion | Issue service registers its company before cascading mention deletion. |
|
||||
| Company status/archive/deletion, including budget pause/resume | Company and budget services register company reconstruction before status or deletion writes. |
|
||||
| Feature enable/disable | Instance settings registers global reconstruction before the experimental-settings write. |
|
||||
| Plugin manifest/status/uninstall | Plugin registry registers reconstruction before lifecycle writes. Installed plugins acquire support through the ordinary ready transition. |
|
||||
| Plugin company configuration and enabled setting | Registry writes register only the affected company. There is no plugin-readiness poll. |
|
||||
| Manual and automatic resolver result | Result persistence registers object intent in the same transaction. Automatic persistence rechecks reference and resolver eligibility. Lost support releases the worker's own claim without publishing the stale result. |
|
||||
| Normalized GitHub ingress | The event service registers company intent with the snapshot write. It clears the old refresh token, so an older provider response cannot overwrite that event. |
|
||||
| Import and restore | No independent object-table importer exists in the source baseline. Supported reference ingestion uses the ordinary sync boundary. Startup reconstructs restored durable objects. |
|
||||
|
||||
Notifications are awaited before writes. Caller-owned and nested transactions
|
||||
retain the outer commit boundary through the existing database-work protocol.
|
||||
Rollback does not publish the write. Notifications received during a scan or an
|
||||
active resolver remain dirty until a successful reconciliation.
|
||||
|
||||
The supported live writer model is one shared `createDb` signal owner, including
|
||||
its transactions and child pools. These process-local signals do not cover
|
||||
independent SQL clients or other replicas. Such writers must invoke the supported
|
||||
service boundary or arrange explicit reconstruction/restart. Existing normalized
|
||||
GitHub ingress uses the supported boundary. No permanent database listener or
|
||||
periodic global safety scan is added.
|
||||
|
||||
## Acceptance evidence
|
||||
|
||||
The focused suite uses real PostgreSQL with a controlled simulated provider.
|
||||
Ordinary sync and registry/settings writes create work; the fixture does not
|
||||
pre-create the tested resolved outcome. Assertions independently read saved
|
||||
objects. Tests measure owner reads and provider calls across former tick windows.
|
||||
|
||||
Coverage includes empty and terminal quiet, unsupported links, orphan removal and
|
||||
reattachment, feature/company eligibility, plugin readiness and permission loss
|
||||
during a held provider call, exact and earlier deadlines, durable provider retry,
|
||||
rollback and outer commit after nested savepoint release, startup and paginated
|
||||
reconstruction, inactive companies, never-refresh rows, first work during drain,
|
||||
warm standby, failed refresh after restart, lease renewal, duplicate exclusion,
|
||||
and shutdown while a provider call remains active. The GitHub ingress suite also
|
||||
checks a real normalized event superseding a held resolver response.
|
||||
|
||||
The conservative external-object host-sleep blocker remains. An in-process
|
||||
deadline cannot wake stopped compute. This change proves worker quiet and
|
||||
reactivation in the tested process; it does not establish Cloud alarm coverage,
|
||||
host/database suspension or deployed savings.
|
||||
|
||||
|
||||
## Host cancellation and registered ownership
|
||||
|
||||
The resolver contract accepts an optional host `AbortSignal`. The worker passes
|
||||
its coordinator signal through refresh to the resolver. This reuses the narrow
|
||||
provider-owned capability introduced with merge-confirmation work: GitHub combines
|
||||
host cancellation with a ten-second request budget for real fetch, response-body
|
||||
reads and visibility probes. Plugin RPC payloads remain unchanged. The host
|
||||
observes cancellation while retaining handlers for any eventual plugin/custom
|
||||
result; it releases only its own claim and cannot publish that late result.
|
||||
Startup does not await provider work. Shutdown closes the active GitHub HTTP
|
||||
request, and its saved due row can resume immediately after restart.
|
||||
|
||||
The old `refreshDueObjects*` batch helpers are removed. Existing selection,
|
||||
lease, concurrent manual refresh, terminal-history and not-found recovery tests
|
||||
now enter through the registered scheduler owner. The removed baseline control
|
||||
recorded six SQL reads over three empty aggregate ticks; the current quiet-work
|
||||
acceptance measures zero recurring SQL after settlement. The scheduled check retains its
|
||||
post-refresh scoped read to revalidate current company, reference and resolver
|
||||
eligibility and recover an authoritative lease after a competing writer; all such
|
||||
reads target one object rather than rebuilding a company.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Issue recovery without periodic scans
|
||||
|
||||
The issue recovery owners reconstruct their current obligations once at startup. Ordinary comment writes invalidate only the affected issue for assignment and lock recovery. Lifecycle, graph, run, configuration and eligibility changes invalidate their company. The coordinator keeps scopes, per-issue deadlines, and per-entry failures in memory. Its pending check does not query the database.
|
||||
|
||||
| Owner | Durable truth and producer boundary | Deadline or event | Exit |
|
||||
| --- | --- | --- | --- |
|
||||
| Stranded assignment | Visible assigned todo/in-progress/review issues; issue create/update/checkout/release; run lifecycle and execution control; agent/company/budget/configuration; interaction, approval and subtree hold changes | A committed eligibility change. A transient continuation uses the latest finish plus the existing exponential delay. | An execution path, durable wait, escalation, or current ineligibility. |
|
||||
| Orphan blocker assignment | Agent-created unassigned todo/blocked blockers with a nonterminal dependent; issue, relation, agent and company changes | Once per changed company, independent of assignment candidates. | Assigned, no longer blocking, or creator cannot execute until its next eligibility change. |
|
||||
| Active recovery actions | Active/escalated action records, including actions on completed sources; action, issue, run and wait changes | Once per changed company, independent of assignment candidates. | Resolved, restored path, explicit hold or a separately owned retry. |
|
||||
| Resolved dependency | Blocked assigned issues and company-scoped blocker relations; blocker status or relation changes through issue service and import; run/control and pause gate changes | The committing change. A failed enqueue retains that issue with error backoff. | Existing idempotent wake, live path, unresolved dependency or deliberate self-block. |
|
||||
| Task watchdog | Active issue watchdogs and their current source subtree; normal watchdog upsert/disable; issue/comment/document/work-product and wait changes | Exact source creation plus the existing 15-second first-run grace. An incomplete scan retains the watchdog with error backoff. | Live path, open/reviewed reusable review, disabled configuration, or ineligible source. |
|
||||
| Stale issue lock | Existing checkout/execution run references; issue and run lifecycle changes | A committing ownership change, or the fresh owning controller's saved lease expiry. | No stale reference, a valid owner, or the existing guarded terminalization and lock clear. |
|
||||
|
||||
Local comment hints do not broaden into an assignment check for every open task. A real PostgreSQL regression starts with1,000 open assigned issues, writes one comment through `issueService.addComment`, and observes exactly one assignment check; a subsequent company eligibility notification still checks all1,000. Restoring the previous comment hint makes that same regression check all1,000 after the comment. The test substitutes the expensive assignment decision only to count its invocations; it uses the real transaction, notification, coordinator and paged SQL. Separate producer acceptance cases exercise real recovery decisions and process execution.
|
||||
|
||||
Parent/descendant disposition repair and shared execution references make status and run changes broader than a comment. Their existing notification boundaries deliberately retain company scope. Orphan blockers, active recovery actions, resolved dependencies and watchdogs also keep their company/subtree duties; an issue comment does not drop those independent obligations. The scope serializer selects known primitive fields and never serializes an entire insert or update object.
|
||||
|
||||
Reconstruction uses keyset pages of at most100rows. Repeated company hints coalesce reconstruction for30seconds. Already-known entry deadlines preempt that spacing without an early probe on each comment. Scheduling suppression is read once per pass. A failed read keeps scoped backoff debt. The two subsidiary company repairs do not run inside the per-issue loop. Legacy direct aggregate API calls retain both repairs.
|
||||
|
||||
The owner methods accept company and issue/watchdog identity. A narrowed lock check still reads other same-company references before using terminal-issue authority. A stale terminal alias cannot end an active issue's shared run. The controller lease owner supplies its fresh expiry; the existing write predicate revalidates the observed boot ID and database-clock expiry.
|
||||
|
||||
Producer coverage uses the existing run-dispatch, execution-control, issue-review-state, native status reassessment and external conversation settlement boundaries. Direct gaps have explicit signals inside their owning transaction: document writes/import, attachment removal, blocker import, recovery-action adoption/mutation, conversation pause release, Telegram failure comment replacement, native artifact title/metadata updates and company deletion. Metadata-only comment classification that does not change timestamp, visibility or execution eligibility needs no additional watchdog activation. Approval links and decisions use their existing issue-review/continuation effects. Work-product create/update/delete/import uses its existing native status boundary.
|
||||
|
||||
The new owners replace only these four startup and periodic calls. Deferred comments, silent active runs and residual orphan/resource recovery retain their existing backstop until the next slice. Every previously migrated native-owner skip remains unchanged.
|
||||
|
||||
A controlled process test prints an execution proof but cannot apply a task disposition. The test operator completes that task through issue service. If the existing continuation policy already saved a disposition-repair retry, the test preserves it until its stored deadline. Only after all provider work has joined does it advance the clock; dispatch then revalidates the completed task and cancels the retry. The final quiet proof requires no registered deadlines, no tracked in-flight work and no owner SQL over twenty former polling minutes. It does not claim a saved future retry is already drained.
|
||||
|
||||
Active action repair has no separate grace or lease countdown. Its delayed disposition repair first writes a scheduled-retry run, then mirrors the date into action wakePolicy.retryAt and timeoutAt. The dispatch owner owns that saved run deadline. Other skipped gates wait for their existing execution, wait, pause, budget, or finalizer producer; the company action worker does not add a polling fallback.
|
||||
@@ -0,0 +1,118 @@
|
||||
# Merged pull request confirmation scheduling
|
||||
|
||||
The merged-PR confirmation duty previously ran from every heartbeat maintenance
|
||||
callback, including when there were no merge requests. It loaded all pending
|
||||
confirmation cards and then applied the merge-only parser in memory.
|
||||
|
||||
The interaction service still owns merge-only eligibility, provider access,
|
||||
automatic acceptance, current resolver authority, audit, and continuation.
|
||||
The worker registers the existing scoped-deadline helper with a dedicated delivery
|
||||
topic. Shared scheduling receives interaction/company identities and deadlines;
|
||||
it receives no GitHub payloads. No database migration is required.
|
||||
|
||||
## Durable truth and producers
|
||||
|
||||
Pending `request_confirmation` interaction rows remain the durable work source.
|
||||
The candidate query pages pending cards in bounded groups of 100 and excludes
|
||||
terminal and hidden issues and linked tool actions. The existing fail-closed
|
||||
parser is the sole text eligibility source: it requires merge intent and a PR
|
||||
reference and excludes additional actions, governed actions, and document/attachment targets.
|
||||
No accepted or cancelled history keeps a deadline alive. A full page without any
|
||||
merge candidates still advances its keyset cursor; later candidates are not lost.
|
||||
|
||||
Ordinary creation registers the topic before inserting a confirmation in its
|
||||
transaction. Existing confirmation-resolution notifications invalidate its issue
|
||||
before acceptance, rejection, withdrawal, supersession, and administrative expiry.
|
||||
Issue status and visibility changes invalidate that issue before their update.
|
||||
The signal helper joins caller-owned outer transactions; nested savepoint release
|
||||
is not a commit. Startup reconstructs existing candidates. Direct tool, secret,
|
||||
and OAuth interaction inserts do not create eligible merge-only work. Company
|
||||
portability does not import pending interaction rows.
|
||||
|
||||
The supported writer model is the instance's ordinary service/API process.
|
||||
Independent SQL writers must use the same transaction-aware notification contract
|
||||
or explicitly invalidate/restart the owning worker. Process-local notifications
|
||||
do not establish a cross-replica ingress guarantee. No permanent database listener
|
||||
or empty recovery scan is added.
|
||||
|
||||
## External observation and retries
|
||||
|
||||
The existing normalized `applyPullRequestEvent` to
|
||||
`sweepMergedPullRequestConfirmations` seam remains available. A normalized merge
|
||||
hint checks matching references in its company; unrelated tenants and PRs do not
|
||||
cause provider reads. Existing provider/binding authorization remains unchanged.
|
||||
|
||||
A pending candidate can reference a PR whose external changes have no dependable
|
||||
webhook coverage. Each such candidate retains a five-minute observation deadline.
|
||||
Resolution, withdrawal, supersession, hiding, or terminal issue state removes the
|
||||
obligation. Provider/read failures and unknown states retain the candidate, using
|
||||
the existing scoped helper's 30-second exponential backoff capped at five minutes.
|
||||
These retries do not scan completed history. Removing candidate reconciliation
|
||||
requires a proven durable delivery/replay guarantee for every referenced PR,
|
||||
including the first event after idle and supported non-Cloud installations.
|
||||
|
||||
This is candidate-backed external observation. It does not eliminate all polling,
|
||||
register a Cloud wake alarm, or prove that the host/database suspends. Existing
|
||||
sleep-safety blockers remain in force.
|
||||
|
||||
Startup registers this observation without waiting for live GitHub requests.
|
||||
The internal resolver invocation accepts an optional host-owned abort signal;
|
||||
that signal is not part of external-object records or plugin/RPC payloads.
|
||||
The GitHub owner combines cancellation with a ten-second request budget and
|
||||
passes it to the actual object fetch and repository-visibility probes, including
|
||||
response body reads. Shutdown aborts the request and waits for its cancellation;
|
||||
no post-abort acceptance is committed. A timeout leaves the durable card pending.
|
||||
|
||||
## Acceptance evidence
|
||||
|
||||
Focused tests use real PostgreSQL, the actual service producers, transaction
|
||||
signals, coordinator, scheduler, acceptance transaction, and persisted audit and
|
||||
chat continuation outbox. GitHub state and the direct heartbeat wake callback are
|
||||
controlled simulation boundaries. The tests do not claim live GitHub delivery or
|
||||
real agent execution.
|
||||
|
||||
Cases cover empty/configured idle and retained terminal history with no recurring
|
||||
SQL/provider work across old cadence windows; the first ordinary creation;
|
||||
outer commit/rollback; exact candidate and failure deadlines; withdrawal;
|
||||
provider failure across drain; standby/resume; restart; tenant-scoped and duplicate
|
||||
merge hints; latest/superseded cards during provider observation; hidden/terminal
|
||||
issues; URL targets; no-continuation policy; and shutdown during an in-flight read.
|
||||
Existing startup and normalized GitHub event tests exercise the changed consumers.
|
||||
Controlled local HTTP tests hold headers, partial response bodies, 404 probes,
|
||||
and public-visibility probes. They observe server-side response close after
|
||||
shutdown within one second, rather than only racing a promise. A shortened
|
||||
request budget times out a real request; the saved card then accepts after
|
||||
restart when that same HTTP fixture serves a merged response. Existing GitHub
|
||||
provider and external-object service tests preserve their authorization behavior.
|
||||
|
||||
## Verification follow-up
|
||||
|
||||
The original head passed full-workspace typecheck and build. A later required
|
||||
Linux CI shard failed the unchanged managed-skill maintenance CLI fixture in
|
||||
`managed-skill-content.test.ts`: it returned `unavailable_or_corrupt_package`
|
||||
where `local_storage` was expected. Both the isolated case and the complete
|
||||
unchanged 17-test file pass against fresh local PostgreSQL. The original Linux cause is
|
||||
not established; that failure is retained and is not called a flake.
|
||||
|
||||
After merging current master, the same fixture failed locally because the new
|
||||
managed-Cloud policy from PR #18350 correctly refuses local disk. A separate
|
||||
test-only prerequisite commit changes that fixture to the real S3 provider with
|
||||
a controlled HTTP endpoint, valid fixture credentials, and a stable test key.
|
||||
It verifies remote reads without seeding or mutating runtime configuration,
|
||||
rejects corrupt and missing objects, and recovers after restoring canonical bytes.
|
||||
The full existing 17-test file passes with these failure controls. Production
|
||||
configuration validation remains unchanged. Current-head required CI remains
|
||||
the completion gate.
|
||||
|
||||
## Existing limitations outside this migration
|
||||
|
||||
Direct non-chat continuation still uses the existing postcommit `wakeup` call.
|
||||
If that call fails after acceptance commits, the previous periodic sweep also
|
||||
skipped the accepted row and did not reconstruct the missing wake. This migration
|
||||
preserves that behavior; a general durable interaction-continuation contract is
|
||||
a separate follow-up. Real chat bindings retain the existing transactional
|
||||
`interaction_wakeup` outbox, verified through ordinary creation and acceptance.
|
||||
|
||||
The separate review-authority issue tracked in public issue #17704 and PR #17711
|
||||
remains owned there. The worker isolates candidate failures and preserves current
|
||||
review authorization; it does not grant a system actor new review authority.
|
||||
@@ -0,0 +1,61 @@
|
||||
# Routine execution status settlement
|
||||
|
||||
The Cloud routine status worker repairs a routine invocation when its linked
|
||||
execution task and saved routine status disagree. The previous heartbeat callback
|
||||
scanned every five minutes, even with no routines or only completed history.
|
||||
|
||||
## Ownership and trigger
|
||||
|
||||
`routine-run-status.ts` owns the projection, custody checks and repair transaction.
|
||||
The worker loads actual mismatches in keyset pages of 50 at startup. It accepts issue hints from issue
|
||||
creation, status/origin changes, checkout/release, routine issue association,
|
||||
execution control writes, review state writes and subtree status restoration.
|
||||
Bulk restoration registers each issue hint in the same transaction before its
|
||||
grouped write. Restored history does not depend on an optional agent wake. It uses the existing
|
||||
transaction-aware work notification protocol. An outer transaction owns the hint;
|
||||
a savepoint release does not wake the repair worker before commit.
|
||||
|
||||
An execution must retain reciprocal task/invocation links in the same company
|
||||
and belong to a routine in that company. This is the existing routine status
|
||||
owner contract. Origin labels can change when a plugin declaration changes or
|
||||
is removed; retained reciprocal links preserve historical execution custody. Repair
|
||||
locks the routine and task and rechecks the linked invocation before writing.
|
||||
Direct route hooks also lock the execution task and linked invocation in their
|
||||
own transaction, nested under caller transactions when present. An older hook
|
||||
cannot write after a newer task status has committed. It preserves skipped/coalesced history and unrelated dispatch failures. A task
|
||||
that resumes clears an execution-task transient failure through the same owner
|
||||
used by native completion and issue routes.
|
||||
|
||||
`activeIssue` remains derived from open tasks and live heartbeat runs. A terminal
|
||||
heartbeat with an open task does not create a completion obligation. The worker
|
||||
does not retry merely because a run is active or a routine exists.
|
||||
|
||||
## Lifecycle and writer contract
|
||||
|
||||
`registerScopedDeadlineWork` owns coalescing, fair batches, failure backoff, drain
|
||||
and shutdown. A failed owner retries with bounded backoff. Successful settlement
|
||||
removes that owner. Settled configuration, healthy active tasks and completed
|
||||
history leave no recurring worker SQL. The existing Cloud and heartbeat scheduler
|
||||
enablement gates remain in place. Ordinary routine scheduling is unchanged.
|
||||
|
||||
Supported app writers register `notifyRoutineExecutionStatus(tx, issueId)` before
|
||||
their mutation. Common execution writers without an issue hint request debt reconstruction only
|
||||
for their recorded company. Projection debt settles even when that company is
|
||||
paused; company activity does not create or cancel this obligation. Backups and independent SQL import/repair tools must restore
|
||||
before startup or call that same transaction contract; process-local hints do
|
||||
not observe independent SQL or another process. No permanent listener or periodic
|
||||
configured-routine scan is added.
|
||||
|
||||
The in-process retry deadline does not establish external Cloud wake coverage.
|
||||
This change preserves the existing host sleep safety checks and does not claim
|
||||
measured host or database suspension.
|
||||
|
||||
## Acceptance
|
||||
|
||||
Tests exercise ordinary issue writes and a real heartbeat lifecycle transition
|
||||
against PostgreSQL with the production coordinator. Provider wakeup is simulated.
|
||||
They inspect durable routine rows and the normal routine detail read separately.
|
||||
Coverage includes empty/configured/history quiet periods, first producer,
|
||||
terminal outcomes and reopen, legacy mismatch reconstruction, skipped/coalesced
|
||||
custody, outer commit/rollback, retry/restart, drain/resume and standby/disable.
|
||||
Focused worker tests exercise in-flight invalidations, retry isolation and stop.
|
||||
@@ -0,0 +1,34 @@
|
||||
# Run dispatch without idle retry scans
|
||||
|
||||
The run-dispatch module owns retry admission, its durable `scheduled_retry_at`
|
||||
deadline, and atomic promotion. The heartbeat queue owns concurrency, current
|
||||
execution authority, and dispatch. The shared coordinator owns notification
|
||||
coalescing, nonoverlap, transaction uncertainty, drain/resume and shutdown.
|
||||
The heartbeat facade and `index.ts` only connect these owners.
|
||||
|
||||
A scheduled retry is a future obligation. Startup reconstructs its next stored
|
||||
deadline through the run-dispatch adapter. Writes that create, move, cancel or
|
||||
invalidate retries notify the retry worker in their transaction. Promotion uses
|
||||
the existing gate and compare-and-set, and its committed write wakes ordinary
|
||||
queued dispatch. A queued run behind occupied capacity waits for the owning
|
||||
run's lifecycle transition. Configuration and authority changes also wake it.
|
||||
An idle queue has no recurring read or timer-driven dispatch attempt. Notifications batch for one second and carry the changed company to queued dispatch. Known retry deadlines still preempt batching. Cancellation-owned retries stay with their cancellation worker and do not retain a promotion deadline.
|
||||
|
||||
The supported writer contract is service writes through the shared database
|
||||
connection scope, including caller-owned and nested transactions. Existing
|
||||
explicit domain notification boundaries cover lifecycle, issue review state,
|
||||
agent configuration/status, company lifecycle and scheduling settings. Direct
|
||||
SQL writers and independent replicas must provide an explicit ingress signal or
|
||||
restart reconstruction; a process-local signal does not observe their commits.
|
||||
This change adds no permanent database listener or write interception.
|
||||
|
||||
The remaining periodic recovery chain still owns deferred comment receipts,
|
||||
execution-wait cancellation recovery, orphan/resource recovery, stranded issue
|
||||
assignments, dependency wake repair, task watchdogs, silent active runs and stale
|
||||
locks. Those duties retain their backstop until their separate producer/deadline
|
||||
migrations are qualified. This slice removes only the periodic scheduled-retry
|
||||
promotion call. Ordinary queued dispatch can still be reached by that backstop;
|
||||
the existing atomic claim and per-agent start lock prevent duplicate execution.
|
||||
|
||||
These workers use internal process deadlines. They do not register Cloud wake
|
||||
alarms and do not relax the durable idle-sleep safety proof.
|
||||
@@ -0,0 +1,157 @@
|
||||
# Status-card evaluation deadlines
|
||||
|
||||
Status-card evaluation uses existing cards, updates and issues as durable truth.
|
||||
`index.ts` registers a typed worker. The status-card owner controls evaluation,
|
||||
generation claims, assignment through the ordinary issue wake API, and failed
|
||||
assignment cancellation. Shared delivery infrastructure controls transaction
|
||||
settlement, coalescing, retries, admission drain, standby and shutdown.
|
||||
|
||||
Manual cards have no automatic obligation. Interval cards retain the requested
|
||||
evaluation cadence. Reactive cards evaluate committed source changes and retain
|
||||
only an actual debounce, hourly rate, daily budget, active-hour or saved-query
|
||||
membership deadline. Unchanged cards disarm. A changed experimental setting
|
||||
invalidates the worker's cached feature gate. Other source writes do not reread
|
||||
settings. Startup reconstructs every disarmed reactive scope together with any
|
||||
concurrent company-scoped hint, without moving saved eligibility/retry deadlines.
|
||||
|
||||
Company search reports the earliest matching `updatedWithin` expiry through an
|
||||
optional internal callback in its existing aggregate query. It includes rows
|
||||
outside the returned page: an expiring priority-ranked row ahead of an offset
|
||||
can change the watched set. The callback exposes only a normalized deadline;
|
||||
ordinary search consumers and API responses retain their existing contract.
|
||||
Overlapping query scopes can require an evaluation when one query expires even
|
||||
if another still retains the same issue; no generation occurs without a filtered
|
||||
card change. No recurring scan replaces the expired query obligation.
|
||||
|
||||
## Claims and recovery
|
||||
|
||||
A feature-owned PostgreSQL transaction advisory lock serializes scheduled
|
||||
per-card evaluation across connections. Its transaction holds no card row lock
|
||||
while existing issue/card services commit through their own transactions. The
|
||||
lock remains held through actual refresh settlement. Existing card deadline CAS,
|
||||
generating-issue ownership and issue idempotency keys remain in force. Manual
|
||||
refresh and older instances continue to use those existing guards. A busy lock
|
||||
retains a company-scoped retry; consuming a hint is not successful evaluation.
|
||||
A failed scheduled refresh retains the existing five-minute durable claim.
|
||||
|
||||
Once a generation exists, the normal issue assignment authority remains its
|
||||
owner. Assignment failure cancels that issue and finalizes the card's failed
|
||||
ledger entry. The user's ordinary refresh action can retry it. Restart recovery
|
||||
must not repeat an already completed generation. No new timer table, automatic
|
||||
write replay, execution-authority exemption or sleep-safety relaxation is added.
|
||||
|
||||
## Supported producer coverage
|
||||
|
||||
All hints are registered before the relevant write within its owning transaction.
|
||||
Nested service writes share outer commit ownership. Company-scoped hints batch
|
||||
reactive reconciliation; ordinary execution writes without known scope retain
|
||||
all existing execution consumers without sending an instance-wide card hint.
|
||||
|
||||
| Producer | Notification boundary and coverage |
|
||||
| --- | --- |
|
||||
| Card create, configuration, archive/restore, delete, compile, query/summary result, evaluation claim | Status-card mutation transactions; stalled issue finalization signals before releasing the card claim |
|
||||
| Experimental status-card setting | Instance experimental-settings transaction; full reconstruction is appropriate for this instance-wide setting |
|
||||
| Company pause/archive/reactivate/delete | Company lifecycle/deletion transactions and budget-owned company hard-stop/resume transactions |
|
||||
| Issue create/update/delete and label changes | Ordinary issue transactions; label deletion signals its known company before cascades; title and prefix changes signal their existing owner transactions |
|
||||
| Issue checkout/adoption/release/reset and environment-selection recency | Existing issue transaction, or a transaction around the previously standalone write; company comes from the existing issue/run read |
|
||||
| Comment create, edit, delete, tombstone, import and recency | Common comment insert; outer root comment transaction; queued-comment and Telegram edit transactions; existing metadata-upgrade path; remove/tombstone transactions |
|
||||
| Issue import, comment import, document import | One notification per affected company in the existing batch transaction; no per-row notification lookup |
|
||||
| Document create/update/restore/delete and search membership | Document owner transactions; only the existing company is passed, without copying search predicates into producers |
|
||||
| Review, interaction and execution transitions | Existing issue-review-state and native-status-reassessment boundaries; scoped execution-control helpers; interaction recency touches own a transaction when called after settlement |
|
||||
| Run admission, retries, monitors, watchdog/recovery and native settlement/reclaim | Existing scoped review/milestone/status/execution-control hooks; explicit notifications at the uncovered issue execution bind, legacy claim, monitor claim/release and review-participant recovery transaction |
|
||||
| Agent deletion/access reassignment and project deletion/privacy cascades | Existing company-scoped owner transactions before foreign-key or recursive issue changes |
|
||||
| Workspace selection/binding, plugin/provider workspace and cleanup/audit touches | Existing owner transactions, including standalone cleanup and audit recency writes now enclosed with their intent |
|
||||
| Skill-test harness expiry and task-watchdog fingerprint touch | Existing owner mutation transaction; no feature policy is moved into the writer |
|
||||
|
||||
The audit includes direct issue/comment mutations, raw recursive privacy updates,
|
||||
bulk insert helpers, label/project/agent cascades, linked document search content
|
||||
and timestamp-dependent search membership. Metadata-only writes that do not
|
||||
change the fingerprint, matching fields or visibility do not create an evaluation
|
||||
obligation: examples include run identity linkage, workspace IDs without a
|
||||
recency change, document locks and source-trust receipts without searched content
|
||||
changes. Routine and status-summary documents that have no issue-document link
|
||||
do not change issue search membership.
|
||||
|
||||
This follows the existing [single-active-writer contract](../idle-sleep-safety.md).
|
||||
Supported services, import/recovery writers, caller-owned transactions and startup
|
||||
reconstruction participate. Independent SQL writers or another active server
|
||||
process require explicit wake ingress; process-local hints do not establish
|
||||
cross-replica notification delivery. The advisory lock supplies competing
|
||||
scheduler exclusion, not cross-process event transport. Internal deadlines do
|
||||
not establish a Cloud wake alarm or prove host/database suspension.
|
||||
|
||||
## Verification
|
||||
|
||||
Real PostgreSQL acceptance uses ordinary card create/compile and authorized
|
||||
summarizer writeback. The assignment boundary is controlled: it persists a wake,
|
||||
and the test explicitly performs the authorized result write; no model runs.
|
||||
Checks cover empty/manual/reactive/disabled quiet SQL, first source change,
|
||||
debounce and durable wake/result, earlier/cancelled deadlines, outer commit and
|
||||
rollback, drain/resume, competing connections, busy locks, restart eligibility,
|
||||
active hours/rate/budget gates, multiple companies, budget-owned company resume,
|
||||
ignored activity, document/comment membership, relative query expiry including
|
||||
an off-page candidate, failed assignment cleanup/manual retry and the persisted
|
||||
five-minute internal-failure retry across restart/drain.
|
||||
|
||||
Shared coordinator regressions also cover warm standby, ambiguous transaction
|
||||
intent, failure entering drain and different domain/coordinator gates. The paused
|
||||
worker releases only settled holds and retains a future in-memory resume wake.
|
||||
An expired saved deadline cannot schedule an immediate skipped-attempt loop.
|
||||
Existing startup, status-card, company-search, execution and monitor suites cover
|
||||
the production wiring and unchanged consumers. Exact commands and results belong
|
||||
in the PR verification record; this document does not claim deployed savings.
|
||||
|
||||
### Review corrections: pool capacity and restart deadline ownership
|
||||
|
||||
Scheduled evaluation holds its advisory transaction on `withDedicatedDbConnection`,
|
||||
not on the ordinary write pool. The helper ends that physical connection in
|
||||
`finally`, including contention and failed evaluation. Its lock still spans the
|
||||
actual refresh settlement. Summary writeback runs watched-query reads on its
|
||||
existing transaction, so a one-connection main pool completes the full flow.
|
||||
|
||||
Migration0419 adds nullable internal `next_eval_reason` to the existing card row.
|
||||
Membership deadlines may be reconstructed at startup after a committed source
|
||||
hint was lost. Policy eligibility and the atomically recorded five-minute retry
|
||||
lease retain their saved deadline. Explicit refresh deadlines remain due work.
|
||||
The HTTP card response removes this internal provenance; the public schema stays
|
||||
unchanged.
|
||||
|
||||
For legacy rows, a pending change hash preserves policy eligibility. An untagged
|
||||
active deadline farther than five minutes away cannot be an outstanding
|
||||
five-minute claim, so startup rechecks it. An ambiguous untagged deadline within
|
||||
five minutes keeps its remaining time without extension. This bounds migration
|
||||
compatibility delay while preserving prior crash/failure recovery. After the
|
||||
first successful evaluation all newly saved deadlines have explicit ownership.
|
||||
No recurring compatibility scan is added.
|
||||
|
||||
Source notifications use the shared deadline helper's fixed one-second batching
|
||||
window. The first hint after idle can reconcile immediately. Later hints merge
|
||||
by company until that window ends. The window does not slide with each new
|
||||
write, so a sustained source stream cannot postpone reconciliation forever.
|
||||
One second keeps reactive cards prompt (the removed heartbeat default was 30s)
|
||||
and bounds burst SQL. No minimum spacing is set: an already-known policy,
|
||||
expiry, or retry deadline can preempt batching. The window alone creates no
|
||||
scheduled work after settlement. Real issue/comment producer acceptance measures
|
||||
due/next queries and actual searches under 100 writes, preserves both companies'
|
||||
hints, dispatches another company's saved deadline inside the batching window,
|
||||
and checks SQL quietness after authorized results.
|
||||
|
||||
Card deletion resolves the card's owning company and emits only that company's
|
||||
deadline-reconstruction hint. Every card mutation requires a company ID; only
|
||||
instance-level settings retain wildcard ingress. Deleting an already-missing
|
||||
card emits no hint. The real two-company regression proves deletion does not
|
||||
re-run unchanged reactive searches or wake either summarizer, then stays quiet.
|
||||
|
||||
Active-hours eligibility keeps the existing exact UTC-minute search across DST
|
||||
changes. A bounded 32-entry timezone formatter cache removes repeated ICU
|
||||
construction. A bounded 256-entry cache reuses the next opening for the exact
|
||||
(timezone,start,end) policy only while the candidate instant stays within its
|
||||
previously searched closed range. The precise instant is checked first, so
|
||||
historical second-based offsets, the opening itself, and millisecond deadlines
|
||||
keep their behavior. Clock rollback and policy changes cannot reuse the wrong
|
||||
range. Empty windows return without formatter or search work. A cold nonempty
|
||||
policy can still perform at most 2881 conversions including its first check; warm
|
||||
closed-range evaluations perform one conversion, not another minute walk.
|
||||
Both caches use bounded LRU retention and hold no company/card data. Tests prove
|
||||
eviction, skipped/repeated hours, half-hour DST, date-line changes, rollback,
|
||||
strict boundaries, and actual twenty-card source delivery followed by quiet.
|
||||
@@ -62,6 +62,14 @@ authority as company-agent API access.
|
||||
Gaps: Instance start/stop and task workspace start/stop are different capabilities.
|
||||
Local service control is not agent company authority.
|
||||
|
||||
Historical Cloud sandbox cleanup remains server-owned. Cleanup-only absence
|
||||
reads authenticate the tenant host with its server capability; they expose no
|
||||
new board or agent endpoint. Their signed scope cannot authorize task recovery.
|
||||
[Runtime and idle-gate regressions](../../server/src/__tests__/historical-cleanup-verification.test.ts)
|
||||
cover reconciliation, while [receipt checks](../../server/src/services/sandbox-absence-proof.test.ts)
|
||||
cover signature, identity, freshness and separation from recovery authority.
|
||||
These checks do not establish a deployed sleep/wake result.
|
||||
|
||||
## `configuration`
|
||||
|
||||
Capability ID: `instance-operations/configuration`.
|
||||
|
||||
@@ -61,6 +61,15 @@ served by the admitted application before it finishes. Repeat with a disconnect
|
||||
before headers arrive. Confirm that neither leaves a permanent request hold. An
|
||||
unfinished application write must still prevent sleep after its client leaves.
|
||||
|
||||
Automated: [historical cleanup](../../server/src/__tests__/historical-cleanup-verification.test.ts)
|
||||
exercises the runtime inspection path for failed Cloud sandbox leases, including
|
||||
verification leases without a run. A signed cleanup-only absence observation
|
||||
can remove this blocker without changing failed outcomes or granting run
|
||||
continuation. An unresolved provider resource still prevents idle sleep.
|
||||
Manual: on an owned Cloud staging workspace, confirm the exact published build,
|
||||
reconcile failed test allocations, then verify actual provider suspension and
|
||||
normal hosted entry after wake. Local tests do not prove that hosted journey.
|
||||
|
||||
### `settings-backup`
|
||||
|
||||
Cloud startup reads nonsecret application choices from a private, versioned
|
||||
|
||||
@@ -60,6 +60,13 @@ inspect the resulting task and run in Paperclip.
|
||||
|
||||
The explicit `stop-redirect` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md) covers `steering/stop-and-pause` only for Stop and one new direction. It requires active work, native cancellation acknowledgement, resource cleanup, and completion without later original-turn activity. It does not qualify pause, interrupt, or all queued-message behavior. Source registration is not a live Cloud pass.
|
||||
|
||||
The explicit `followup-while-running` [Product journey](../../tests/runner-e2e/PRODUCT-JOURNEYS.md#active-follow-up)
|
||||
covers `steering/queue`: one follow-up is saved while a bound native command is
|
||||
active, its late requirement appears in one attributed final reply, the queue
|
||||
empties, and the result survives reload. One steered turn or one normal successor
|
||||
is allowed. Unit calibration and source registration are not live acceptance.
|
||||
|
||||
|
||||
Preconditions: follow the [baseline](./README.md#before-driving-a-journey).
|
||||
Use an agent that stays busy long enough to inspect the queue. Record the adapter,
|
||||
native/legacy mode, and advertised steering capability. Test supported and
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE "status_cards" ADD COLUMN "next_eval_reason" text;
|
||||
+512
-4
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "27c9008c-60e3-449b-9630-9e4977fad444",
|
||||
"prevId": "5690a14a-f9cc-41a6-af51-c27f688f4ef6",
|
||||
"id": "cce00822-b7a4-4acd-9c59-0f6f5822c0cf",
|
||||
"prevId": "c777fa84-85a6-418f-bea1-0d60b56cf071",
|
||||
"version": "7",
|
||||
"dialect": "postgresql",
|
||||
"tables": {
|
||||
@@ -19052,6 +19052,98 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.company_skill_mutation_receipts": {
|
||||
"name": "company_skill_mutation_receipts",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"key": {
|
||||
"name": "key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"fingerprint": {
|
||||
"name": "fingerprint",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"skill_id": {
|
||||
"name": "skill_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"version_id": {
|
||||
"name": "version_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"response": {
|
||||
"name": "response",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"company_skill_mutation_receipts_key_idx": {
|
||||
"name": "company_skill_mutation_receipts_key_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "key",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"company_skill_mutation_receipts_company_id_companies_id_fk": {
|
||||
"name": "company_skill_mutation_receipts_company_id_companies_id_fk",
|
||||
"tableFrom": "company_skill_mutation_receipts",
|
||||
"tableTo": "companies",
|
||||
"columnsFrom": [
|
||||
"company_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "cascade",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.company_skill_repository_snapshots": {
|
||||
"name": "company_skill_repository_snapshots",
|
||||
"schema": "",
|
||||
@@ -19079,7 +19171,46 @@
|
||||
"name": "file_inventory",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
"notNull": true,
|
||||
"default": "'[]'::jsonb"
|
||||
},
|
||||
"storage": {
|
||||
"name": "storage",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"upload": {
|
||||
"name": "upload",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"restore": {
|
||||
"name": "restore",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"migration_at": {
|
||||
"name": "migration_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"default": "now()"
|
||||
},
|
||||
"maintenance_at": {
|
||||
"name": "maintenance_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"default": "now()"
|
||||
},
|
||||
"restore_retry_at": {
|
||||
"name": "restore_retry_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
@@ -19110,6 +19241,38 @@
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"company_skill_snapshots_migration_idx": {
|
||||
"name": "company_skill_snapshots_migration_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "migration_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"company_skill_repository_snapshots\".\"migration_at\" is not null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"company_skill_snapshots_maintenance_idx": {
|
||||
"name": "company_skill_snapshots_maintenance_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "maintenance_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"company_skill_repository_snapshots\".\"maintenance_at\" is not null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
@@ -19999,6 +20162,25 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"package_snapshot_id": {
|
||||
"name": "package_snapshot_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"package_file_map": {
|
||||
"name": "package_file_map",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"migration_at": {
|
||||
"name": "migration_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": false,
|
||||
"default": "now()"
|
||||
},
|
||||
"revision_number": {
|
||||
"name": "revision_number",
|
||||
"type": "integer",
|
||||
@@ -20057,6 +20239,52 @@
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"company_skill_versions_migration_idx": {
|
||||
"name": "company_skill_versions_migration_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "migration_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"company_skill_versions\".\"migration_at\" is not null",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"company_skill_versions_package_snapshot_idx": {
|
||||
"name": "company_skill_versions_package_snapshot_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "package_snapshot_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"company_skill_versions_repository_snapshot_idx": {
|
||||
"name": "company_skill_versions_repository_snapshot_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "repository_snapshot_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"company_skill_versions_skill_revision_idx": {
|
||||
"name": "company_skill_versions_skill_revision_idx",
|
||||
"columns": [
|
||||
@@ -20168,6 +20396,19 @@
|
||||
"onDelete": "restrict",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk": {
|
||||
"name": "company_skill_versions_package_snapshot_id_company_skill_repository_snapshots_id_fk",
|
||||
"tableFrom": "company_skill_versions",
|
||||
"tableTo": "company_skill_repository_snapshots",
|
||||
"columnsFrom": [
|
||||
"package_snapshot_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "restrict",
|
||||
"onUpdate": "no action"
|
||||
},
|
||||
"company_skill_versions_author_agent_id_agents_id_fk": {
|
||||
"name": "company_skill_versions_author_agent_id_agents_id_fk",
|
||||
"tableFrom": "company_skill_versions",
|
||||
@@ -20613,6 +20854,78 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.skill_package_deletions": {
|
||||
"name": "skill_package_deletions",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true,
|
||||
"default": "gen_random_uuid()"
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"provider": {
|
||||
"name": "provider",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"archive_key": {
|
||||
"name": "archive_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"manifest_key": {
|
||||
"name": "manifest_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"delete_after": {
|
||||
"name": "delete_after",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"skill_package_deletions_due_idx": {
|
||||
"name": "skill_package_deletions_due_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "delete_after",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.company_transfer_runs": {
|
||||
"name": "company_transfer_runs",
|
||||
"schema": "",
|
||||
@@ -33849,7 +34162,7 @@
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_failed')) IS NOT TRUE",
|
||||
"where": "\"heartbeat_runs\".\"runtime_mode\" = 'native' AND \"heartbeat_runs\".\"status\" = 'cancelled'\n AND \"heartbeat_runs\".\"runner_profile_json\"->'nativeWorkspaceSync'->>'state' = 'prepared'\n AND \"heartbeat_runs\".\"result_json\"->'nativeCancellation'->>'dispatchState' = 'acknowledged'\n AND (\"heartbeat_runs\".\"result_json\"->'workspaceRestoreRecovery'->>'schema' = 'paperclip.workspace-restore-recovery.v1'\n AND \"heartbeat_runs\".\"result_json\"->>'workspaceRestoreFailure' IN ('restore_permission_denied', 'restore_lock_timeout', 'restore_unsafe_archive', 'restore_invalid_metadata', 'restore_failed')) IS NOT TRUE",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
@@ -51261,6 +51574,195 @@
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.sandbox_creation_operations": {
|
||||
"name": "sandbox_creation_operations",
|
||||
"schema": "",
|
||||
"columns": {
|
||||
"id": {
|
||||
"name": "id",
|
||||
"type": "uuid",
|
||||
"primaryKey": true,
|
||||
"notNull": true
|
||||
},
|
||||
"company_id": {
|
||||
"name": "company_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"environment_id": {
|
||||
"name": "environment_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"original_environment_id": {
|
||||
"name": "original_environment_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"run_id": {
|
||||
"name": "run_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"request_key": {
|
||||
"name": "request_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'run'"
|
||||
},
|
||||
"issue_id": {
|
||||
"name": "issue_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"execution_workspace_id": {
|
||||
"name": "execution_workspace_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"provider": {
|
||||
"name": "provider",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"plugin_id": {
|
||||
"name": "plugin_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"driver_key": {
|
||||
"name": "driver_key",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"state": {
|
||||
"name": "state",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "'pending'"
|
||||
},
|
||||
"config": {
|
||||
"name": "config",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"metadata": {
|
||||
"name": "metadata",
|
||||
"type": "jsonb",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"lease_id": {
|
||||
"name": "lease_id",
|
||||
"type": "uuid",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"next_attempt_at": {
|
||||
"name": "next_attempt_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true
|
||||
},
|
||||
"created_at": {
|
||||
"name": "created_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
},
|
||||
"updated_at": {
|
||||
"name": "updated_at",
|
||||
"type": "timestamp with time zone",
|
||||
"primaryKey": false,
|
||||
"notNull": true,
|
||||
"default": "now()"
|
||||
}
|
||||
},
|
||||
"indexes": {
|
||||
"sandbox_creation_operations_pending_deadline_idx": {
|
||||
"name": "sandbox_creation_operations_pending_deadline_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "next_attempt_at",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": false,
|
||||
"where": "\"sandbox_creation_operations\".\"state\" = 'pending'",
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
},
|
||||
"sandbox_creation_operations_owner_idx": {
|
||||
"name": "sandbox_creation_operations_owner_idx",
|
||||
"columns": [
|
||||
{
|
||||
"expression": "company_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "original_environment_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "run_id",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
},
|
||||
{
|
||||
"expression": "request_key",
|
||||
"isExpression": false,
|
||||
"asc": true,
|
||||
"nulls": "last"
|
||||
}
|
||||
],
|
||||
"isUnique": true,
|
||||
"concurrently": false,
|
||||
"method": "btree",
|
||||
"with": {}
|
||||
}
|
||||
},
|
||||
"foreignKeys": {
|
||||
"sandbox_creation_operations_environment_id_environments_id_fk": {
|
||||
"name": "sandbox_creation_operations_environment_id_environments_id_fk",
|
||||
"tableFrom": "sandbox_creation_operations",
|
||||
"tableTo": "environments",
|
||||
"columnsFrom": [
|
||||
"environment_id"
|
||||
],
|
||||
"columnsTo": [
|
||||
"id"
|
||||
],
|
||||
"onDelete": "set null",
|
||||
"onUpdate": "no action"
|
||||
}
|
||||
},
|
||||
"compositePrimaryKeys": {},
|
||||
"uniqueConstraints": {},
|
||||
"policies": {},
|
||||
"checkConstraints": {},
|
||||
"isRLSEnabled": false
|
||||
},
|
||||
"public.secret_access_events": {
|
||||
"name": "secret_access_events",
|
||||
"schema": "",
|
||||
@@ -52268,6 +52770,12 @@
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"next_eval_reason": {
|
||||
"name": "next_eval_reason",
|
||||
"type": "text",
|
||||
"primaryKey": false,
|
||||
"notNull": false
|
||||
},
|
||||
"next_eval_at": {
|
||||
"name": "next_eval_at",
|
||||
"type": "timestamp with time zone",
|
||||
+7
@@ -2920,6 +2920,13 @@
|
||||
"when": 1793395364937,
|
||||
"tag": "0418_curved_lady_mastermind",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 419,
|
||||
"version": "7",
|
||||
"when": 1793402408013,
|
||||
"tag": "0419_moaning_vulture",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -56,6 +56,9 @@ export const statusCards = pgTable(
|
||||
lastModel: text("last_model"),
|
||||
generatingIssueId: uuid("generating_issue_id").references(() => issues.id, { onDelete: "set null" }),
|
||||
failureReason: text("failure_reason"),
|
||||
// Internal deadline ownership; null denotes a row written before provenance
|
||||
// was recorded. Membership may be rechecked at startup, retry/policy may not.
|
||||
nextEvalReason: text("next_eval_reason").$type<"membership" | "policy" | "retry" | "refresh">(),
|
||||
nextEvalAt: timestamp("next_eval_at", { withTimezone: true }),
|
||||
archivedAt: timestamp("archived_at", { withTimezone: true }),
|
||||
archivedByUserId: text("archived_by_user_id"),
|
||||
|
||||
@@ -98,6 +98,35 @@ digest, and refuse the rest. A profile identity does not establish hosted
|
||||
harness qualification. [`rules/acpx-profile-identity.mdc`](../../../rules/acpx-profile-identity.mdc)
|
||||
summarizes these practices for contributors.
|
||||
|
||||
Qualified ACPX native executables use Runner's persistent digest cache. The cache
|
||||
stores a computed SHA-256 digest under a key made from device, inode, size,
|
||||
nanosecond modification time, and nanosecond change time. Each check compares
|
||||
that digest with the independently selected build-owned pin. A changed file
|
||||
identity, missing entry, or unusable cache causes a full descriptor read and
|
||||
hash. A hit reuses the earlier digest; metadata identity is not a fresh content
|
||||
hash. Claude, Codex, and Grok use the same native executable admission path.
|
||||
|
||||
Cache entries live in the operating-system temporary directory under
|
||||
`paperclip-runner-<uid>/executable-digests-v1`, outside task workspaces such as
|
||||
`/home/agent`. Neither `HOME` nor the task working directory selects this path.
|
||||
Node's `os.tmpdir()` selects the root and honors the platform's absolute
|
||||
`TMPDIR`, `TMP`, or `TEMP` override. Hosts must keep that root outside task
|
||||
workspaces; a relative or unusable root disables caching. Temporary-directory
|
||||
eviction removes the optimization until a later full hash fills it again. The
|
||||
directory and files are private to that user. Writes use a temporary file and
|
||||
atomic rename, so later Runner or sidecar processes can reuse complete entries.
|
||||
Cache errors fall back to hashing. The cache stores no credentials or file paths.
|
||||
|
||||
Every admission still opens the executable without following its final symlink,
|
||||
checks its regular-file type, bounds, execute permission, and before/after file
|
||||
identity, and retains the descriptor for launch. Package qualification and the
|
||||
preparation-to-launch identity check remain. The cache deliberately trusts the
|
||||
operating-system user: a process with that user's access can plant a digest for
|
||||
a chosen fingerprint. It is not authenticated with a secret or signature.
|
||||
An administrator-owned read-only Runner can therefore use a user-writable cache,
|
||||
but this provides weaker protection against that same user than hashing every
|
||||
executable on each admission. Removing the cache restores full hashing.
|
||||
|
||||
## External controller checkpoints
|
||||
|
||||
The optional controller persistence callback stores recovery control state, not
|
||||
|
||||
@@ -45,9 +45,9 @@ reader.on('line', line => {
|
||||
if(request.method==='initialize') {
|
||||
process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{protocolVersion:1,agentInfo:{name:'antigravity-acp',version:'1.4.3'},agentCapabilities:{_meta:{antigravity:{paperclipAdmission:1}}},authMethods:[{id:'oauth-personal'}]}})+'\\n');
|
||||
} else if(request.method==='authenticate') {
|
||||
const url = mode==='bad-url' ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())};
|
||||
const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())};
|
||||
const message='Open the following link to authenticate the ACP server: '+url+'\\n';
|
||||
process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42));
|
||||
if (!mode.endsWith('url-after-result')) {process.stderr.write(message.slice(0,42));process.stderr.write(message.slice(42));}
|
||||
if(mode==='blocked' || mode==='bad-url' || mode==='stubborn' || mode==='retained') return;
|
||||
if(mode==='error') {process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,error:{code:-32000,message:'do not expose fixture-refresh-token'}})+'\\n');return;}
|
||||
fs.mkdirSync(path.join(home,'antigravity-acp'),{mode:0o700});
|
||||
@@ -55,7 +55,14 @@ reader.on('line', line => {
|
||||
process.stdout.write(JSON.stringify({jsonrpc:'2.0',id:request.id,result:{}})+'\\n');
|
||||
} else {throw new Error('inference/session request forbidden');}
|
||||
});
|
||||
reader.on('close',()=>mode==='stubborn'||mode==='retained'?setInterval(()=>{},1000):process.exit(0));
|
||||
reader.on('close',()=> {
|
||||
if (mode.endsWith('url-after-result')) {
|
||||
const url = mode.startsWith('bad-url') ? 'https://example.invalid/steal' : ${JSON.stringify(authorizationUrl())};
|
||||
// EOF proves the client received the authenticate result before this stderr.
|
||||
process.stderr.write('Open the following link to authenticate the ACP server: '+url+'\\n', () => process.exit(0));
|
||||
} else if(mode==='stubborn'||mode==='retained') setInterval(()=>{},1000);
|
||||
else process.exit(0);
|
||||
});
|
||||
`);
|
||||
setup.verify.mockResolvedValue({ openCommand: async () => ({ spawn: setup.spawn, close: setup.close }) });
|
||||
setup.spawn.mockImplementation((_args, options) => {
|
||||
@@ -95,8 +102,8 @@ describe("Antigravity setup-only Google authentication", () => {
|
||||
|
||||
// The login predicate keeps its five-second bound. Allow ten more seconds
|
||||
// for fixture startup and the durable mocked credential checks on a busy CI worker.
|
||||
it("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls", async () => {
|
||||
const home = await fixture(), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch);
|
||||
it.each(["success", "url-after-result"])("performs native authenticate, retires it, verifies identity, and preserves refresh rotation without repeated status network calls (%s)", async mode => {
|
||||
const home = await fixture(mode), fetch = googleIdentity(); vi.stubGlobal("fetch", fetch);
|
||||
const login = startAntigravityBrowserLogin(home);
|
||||
await vi.waitFor(() => expect(login.outcome).toBe("success"), { timeout: 5_000 });
|
||||
expect(login.authorizationUrl).toBe(authorizationUrl());
|
||||
@@ -118,13 +125,13 @@ describe("Antigravity setup-only Google authentication", () => {
|
||||
expect(fetch).toHaveBeenCalledTimes(6);
|
||||
}, 15_000);
|
||||
|
||||
it.each(["error", "bad-url"])("rejects %s without leaking provider text or calling Google", async mode => {
|
||||
it.each(["error", "bad-url", "bad-url-after-result"])("rejects %s without leaking provider text or calling Google", async mode => {
|
||||
const home = await fixture(mode), fetch = vi.fn(); vi.stubGlobal("fetch", fetch);
|
||||
const login = startAntigravityBrowserLogin(home);
|
||||
await vi.waitFor(() => expect(login.outcome).toBe("failure"), { timeout: 5_000 });
|
||||
await vi.waitFor(() => expect(setup.close).toHaveBeenCalledOnce(), { timeout: 5_000 });
|
||||
expect(login.error).not.toContain("fixture");
|
||||
if (mode === "bad-url") expect(login.authorizationUrl).toBeUndefined();
|
||||
if (mode.startsWith("bad-url")) expect(login.authorizationUrl).toBeUndefined();
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rename, rm, symlink, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createExecutableDigestCache, executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js";
|
||||
import { openVerifiedRuntimeExecutable } from "./installation-integrity.js";
|
||||
|
||||
const temporary: string[] = [];
|
||||
const identity: ExecutableFileIdentity = { device: "1", inode: "2", size: "3", modifiedNanoseconds: "4", changedNanoseconds: "5" };
|
||||
const digest = `sha256:${"a".repeat(64)}`;
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await Promise.all(temporary.splice(0).map(path => rm(path, { recursive: true, force: true })));
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(join(tmpdir(), "runner-executable-digest-cache-"));
|
||||
temporary.push(root);
|
||||
const directory = join(root, "cache");
|
||||
const cache = createExecutableDigestCache(directory);
|
||||
return { root, directory, cache };
|
||||
}
|
||||
|
||||
describe("persistent executable digest cache", () => {
|
||||
it("stores the actual digest in a private, versioned entry and survives a new cache instance", async () => {
|
||||
const f = await fixture();
|
||||
expect(await f.cache.read(identity)).toBeNull();
|
||||
await f.cache.write(identity, digest);
|
||||
expect(await createExecutableDigestCache(f.directory).read(identity)).toBe(digest);
|
||||
const [name] = await readdir(f.directory);
|
||||
expect(name).toMatch(/^v1-[a-f0-9]{64}\.json$/);
|
||||
expect(JSON.parse(await readFile(join(f.directory, name), "utf8"))).toEqual({ version: 1, digest });
|
||||
expect((await lstat(f.directory)).mode & 0o777).toBe(0o700);
|
||||
expect((await lstat(join(f.directory, name))).mode & 0o777).toBe(0o600);
|
||||
});
|
||||
|
||||
it.each(Object.keys(identity) as (keyof ExecutableFileIdentity)[])("invalidates a changed %s", async key => {
|
||||
const f = await fixture();
|
||||
await f.cache.write(identity, digest);
|
||||
expect(await f.cache.read({ ...identity, [key]: "changed" })).toBeNull();
|
||||
});
|
||||
|
||||
it.each(["malformed", "version", "digest", "oversized", "permissions", "symlink"] as const)(
|
||||
"treats an unusable %s entry as a miss", async kind => {
|
||||
const f = await fixture();
|
||||
await f.cache.write(identity, digest);
|
||||
const [name] = await readdir(f.directory), file = join(f.directory, name);
|
||||
if (kind === "malformed") await writeFile(file, "{");
|
||||
else if (kind === "version") await writeFile(file, JSON.stringify({ version: 2, digest }));
|
||||
else if (kind === "digest") await writeFile(file, JSON.stringify({ version: 1, digest: "expected" }));
|
||||
else if (kind === "oversized") await writeFile(file, "x".repeat(1025));
|
||||
else if (kind === "permissions") await chmod(file, 0o644);
|
||||
else { await rename(file, `${file}.target`); await symlink(`${file}.target`, file); }
|
||||
expect(await f.cache.read(identity)).toBeNull();
|
||||
},
|
||||
);
|
||||
|
||||
it("falls back when the cache directory cannot be used and leaves existing permissions unchanged", async () => {
|
||||
const f = await fixture();
|
||||
await writeFile(f.directory, "not a directory");
|
||||
await expect(f.cache.write(identity, digest)).resolves.toBeUndefined();
|
||||
expect(await f.cache.read(identity)).toBeNull();
|
||||
await rm(f.directory);
|
||||
await mkdir(f.directory, { mode: 0o755 });
|
||||
await f.cache.write(identity, digest);
|
||||
expect(await f.cache.read(identity)).toBeNull();
|
||||
expect((await lstat(f.directory)).mode & 0o777).toBe(0o755);
|
||||
});
|
||||
|
||||
it("does not resolve a relative cache directory through the task working directory", async () => {
|
||||
const cache = createExecutableDigestCache("relative-task-cache");
|
||||
await cache.write(identity, digest);
|
||||
expect(await cache.read(identity)).toBeNull();
|
||||
});
|
||||
|
||||
it("publishes complete entries under concurrent writers without leftover temporary files", async () => {
|
||||
const f = await fixture();
|
||||
await Promise.all(Array.from({ length: 8 }, () => f.cache.write(identity, digest)));
|
||||
expect(await f.cache.read(identity)).toBe(digest);
|
||||
expect(await readdir(f.directory)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe.skipIf(!["linux", "darwin"].includes(process.platform))("descriptor admission with cached digests", () => {
|
||||
async function executable() {
|
||||
const f = await fixture(), path = join(f.root, "native"), bytes = Buffer.from("qualified native fixture");
|
||||
await writeFile(path, bytes, { mode: 0o700 });
|
||||
vi.spyOn(executableDigestCache, "read").mockImplementation(f.cache.read);
|
||||
vi.spyOn(executableDigestCache, "write").mockImplementation(f.cache.write);
|
||||
return { ...f, path, digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}` };
|
||||
}
|
||||
|
||||
it("hashes a miss, reuses the actual digest, and still rejects a different independent pin", async () => {
|
||||
const f = await executable();
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
expect(executableDigestCache.write).toHaveBeenCalledOnce();
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
expect(executableDigestCache.write).toHaveBeenCalledOnce();
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, digest, "fixture")).rejects.toThrow("digest mismatch");
|
||||
});
|
||||
|
||||
it("rehashes changed contents and rejects corruption despite a previous valid entry", async () => {
|
||||
const f = await executable();
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
await writeFile(f.path, "corrupted native fixture");
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
|
||||
});
|
||||
|
||||
it.each(["permissions", "symlink", "replacement"] as const)("retains admission checks after cached %s changes", async kind => {
|
||||
const f = await executable();
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
if (kind === "permissions") {
|
||||
await chmod(f.path, 0o600);
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("bounded executable file");
|
||||
} else if (kind === "symlink") {
|
||||
await rename(f.path, `${f.path}.original`); await symlink(`${f.path}.original`, f.path);
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("real regular file");
|
||||
} else {
|
||||
await rename(f.path, `${f.path}.original`); await writeFile(f.path, "new invalid executable", { mode: 0o700 });
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects mutation while a cache hit is being read", async () => {
|
||||
const f = await executable();
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
vi.mocked(executableDigestCache.read).mockImplementationOnce(async () => {
|
||||
await writeFile(f.path, "mutation during lookup");
|
||||
return f.digest;
|
||||
});
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified");
|
||||
});
|
||||
|
||||
it("checks executable identity after publishing a cold cache entry", async () => {
|
||||
const f = await executable();
|
||||
vi.mocked(executableDigestCache.write).mockImplementationOnce(async (identity, digest) => {
|
||||
await f.cache.write(identity, digest);
|
||||
await writeFile(f.path, "mutation during cache publication");
|
||||
});
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("changed while it was verified");
|
||||
});
|
||||
|
||||
it("falls back to hashing when the cache is unavailable", async () => {
|
||||
const f = await executable();
|
||||
await writeFile(f.directory, "unavailable cache");
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
await writeFile(f.path, "invalid native fixture");
|
||||
await expect(openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).rejects.toThrow("digest mismatch");
|
||||
});
|
||||
|
||||
it("trusts a digest planted by the same operating-system user for current metadata", async () => {
|
||||
const f = await executable();
|
||||
await writeFile(f.path, "content changed by the same user");
|
||||
const metadata = await lstat(f.path, { bigint: true });
|
||||
await f.cache.write({ device: metadata.dev.toString(), inode: metadata.ino.toString(), size: metadata.size.toString(),
|
||||
modifiedNanoseconds: metadata.mtimeNs.toString(), changedNanoseconds: metadata.ctimeNs.toString() }, f.digest);
|
||||
await (await openVerifiedRuntimeExecutable(f.path, f.digest, "fixture")).handle.close();
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { MAX_ACPX_RUNTIME_EXECUTABLE_BYTES, ACPX_PRIVATE_SNAPSHOT_ENV, createAcp
|
||||
import { createNativeAcpxDistributionSnapshot, NATIVE_ACPX_BOOTSTRAP_NAME, readNativeAcpxDistributionEntries, type NativeAcpxDistributionInput } from "./native-distribution-integrity.js";
|
||||
export type { NativeAcpxDistributionInput } from "./native-distribution-integrity.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { executableDigestCache, type ExecutableFileIdentity } from "../../executable-digest-cache.js";
|
||||
import {
|
||||
spawn as spawnChildProcess,
|
||||
type ChildProcess,
|
||||
@@ -606,13 +607,7 @@ export async function awaitVerifiedAcpxProviderExit(
|
||||
await exitProof;
|
||||
}
|
||||
|
||||
interface VerifiedAcpxCommandIdentity {
|
||||
device: string;
|
||||
inode: string;
|
||||
size: string;
|
||||
modifiedNanoseconds: string;
|
||||
changedNanoseconds: string;
|
||||
}
|
||||
type VerifiedAcpxCommandIdentity = ExecutableFileIdentity;
|
||||
|
||||
interface VerifiedAcpxRuntimeExecutable {
|
||||
path: string;
|
||||
@@ -1204,7 +1199,7 @@ async function inspectCommand(
|
||||
}
|
||||
}
|
||||
|
||||
async function openVerifiedRuntimeExecutable(
|
||||
export async function openVerifiedRuntimeExecutable(
|
||||
executablePath: string,
|
||||
expectedDigest: string,
|
||||
agent: string,
|
||||
@@ -1246,23 +1241,31 @@ async function openVerifiedRuntimeExecutable(
|
||||
`ACPX ${agent} runtime executable must be a bounded executable file`,
|
||||
);
|
||||
}
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.alloc(1024 * 1024);
|
||||
const cachedDigest = await executableDigestCache.read(fileIdentity(before));
|
||||
let digest = cachedDigest;
|
||||
let position = 0;
|
||||
try {
|
||||
while (position < Number(before.size)) {
|
||||
const { bytesRead } = await handle.read(
|
||||
buffer,
|
||||
0,
|
||||
Math.min(buffer.length, Number(before.size) - position),
|
||||
position,
|
||||
);
|
||||
if (bytesRead === 0) break;
|
||||
hash.update(buffer.subarray(0, bytesRead));
|
||||
position += bytesRead;
|
||||
if (digest === null) {
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.alloc(1024 * 1024);
|
||||
try {
|
||||
while (position < Number(before.size)) {
|
||||
const { bytesRead } = await handle.read(
|
||||
buffer,
|
||||
0,
|
||||
Math.min(buffer.length, Number(before.size) - position),
|
||||
position,
|
||||
);
|
||||
if (bytesRead === 0) break;
|
||||
hash.update(buffer.subarray(0, bytesRead));
|
||||
position += bytesRead;
|
||||
}
|
||||
} finally {
|
||||
buffer.fill(0);
|
||||
}
|
||||
} finally {
|
||||
buffer.fill(0);
|
||||
digest = `sha256:${hash.digest("hex")}`;
|
||||
}
|
||||
if (cachedDigest === null && position === Number(before.size) && digest === expectedDigest) {
|
||||
await executableDigestCache.write(fileIdentity(before), digest);
|
||||
}
|
||||
const after = await handle.stat({ bigint: true });
|
||||
const lexicalAfter = await lstat(executablePath, { bigint: true }).catch(
|
||||
@@ -1271,7 +1274,7 @@ async function openVerifiedRuntimeExecutable(
|
||||
const beforeIdentity = fileIdentity(before);
|
||||
const afterIdentity = fileIdentity(after);
|
||||
if (
|
||||
position !== Number(before.size) ||
|
||||
(cachedDigest === null && position !== Number(before.size)) ||
|
||||
lexicalAfter === null ||
|
||||
lexicalAfter.isSymbolicLink() ||
|
||||
!lexicalAfter.isFile() ||
|
||||
@@ -1283,7 +1286,6 @@ async function openVerifiedRuntimeExecutable(
|
||||
`ACPX ${agent} runtime executable changed while it was verified`,
|
||||
);
|
||||
}
|
||||
const digest = `sha256:${hash.digest("hex")}`;
|
||||
if (digest !== expectedDigest) {
|
||||
throw new Error(`ACPX ${agent} runtime executable digest mismatch`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, mkdir, open, rename, unlink } from "node:fs/promises";
|
||||
import { tmpdir, userInfo } from "node:os";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
|
||||
export interface ExecutableFileIdentity {
|
||||
device: string;
|
||||
inode: string;
|
||||
size: string;
|
||||
modifiedNanoseconds: string;
|
||||
changedNanoseconds: string;
|
||||
}
|
||||
|
||||
export function createExecutableDigestCache(directory?: string) {
|
||||
async function location(identity: ExecutableFileIdentity, create: boolean): Promise<string> {
|
||||
const user = userInfo();
|
||||
const temporaryRoot = tmpdir();
|
||||
if (!isAbsolute(directory ?? temporaryRoot)) throw new Error("Executable digest cache directory is unavailable");
|
||||
const base = join(temporaryRoot, `paperclip-runner-${user.uid}`);
|
||||
const directories = directory ? [directory] : [base, join(base, "executable-digests-v1")];
|
||||
for (const [index, path] of directories.entries()) {
|
||||
if (create) await mkdir(path, { mode: 0o700 }).catch(error => {
|
||||
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
|
||||
});
|
||||
const metadata = await lstat(path);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink() || metadata.uid !== user.uid ||
|
||||
(metadata.mode & (index === directories.length - 1 ? 0o077 : 0o022)) !== 0) {
|
||||
throw new Error("Executable digest cache directory is unavailable");
|
||||
}
|
||||
}
|
||||
const key = createHash("sha256").update(JSON.stringify([1, identity.device, identity.inode, identity.size,
|
||||
identity.modifiedNanoseconds, identity.changedNanoseconds])).digest("hex");
|
||||
return join(directories.at(-1)!, `v1-${key}.json`);
|
||||
}
|
||||
|
||||
return {
|
||||
async read(identity: ExecutableFileIdentity): Promise<string | null> {
|
||||
try {
|
||||
const path = await location(identity, false);
|
||||
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
const metadata = await handle.stat();
|
||||
if (!metadata.isFile() || metadata.uid !== userInfo().uid || (metadata.mode & 0o077) !== 0 ||
|
||||
metadata.size < 1 || metadata.size > 1024) return null;
|
||||
const entry: unknown = JSON.parse(await handle.readFile("utf8"));
|
||||
if (typeof entry !== "object" || entry === null || !("version" in entry) || entry.version !== 1 ||
|
||||
!("digest" in entry) || typeof entry.digest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(entry.digest)) return null;
|
||||
return entry.digest;
|
||||
} finally { await handle.close(); }
|
||||
} catch { return null; }
|
||||
},
|
||||
async write(identity: ExecutableFileIdentity, digest: string): Promise<void> {
|
||||
let temporary: string | undefined;
|
||||
try {
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(digest)) return;
|
||||
const path = await location(identity, true);
|
||||
temporary = `${path}.${randomUUID()}.tmp`;
|
||||
const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
||||
try { await handle.writeFile(JSON.stringify({ version: 1, digest })); }
|
||||
finally { await handle.close(); }
|
||||
await rename(temporary, path);
|
||||
} catch { } finally {
|
||||
if (temporary) await unlink(temporary).catch(() => undefined);
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const executableDigestCache = createExecutableDigestCache();
|
||||
@@ -0,0 +1,34 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { resolveStandardImageDigest } from "./standard-image-contract.mjs";
|
||||
|
||||
export async function reuseCanaryImage(sha, {
|
||||
resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }),
|
||||
run = (command, args) => execFileSync(command, args, { stdio: "inherit" }),
|
||||
} = {}) {
|
||||
assert.match(sha ?? "", /^[a-f0-9]{40}$/);
|
||||
const digest = await resolve(sha);
|
||||
if (digest === null) return false;
|
||||
assert.match(digest, /^sha256:[a-f0-9]{64}$/);
|
||||
const image = `ghcr.io/paperclipai/paperclip@${digest}`;
|
||||
await run("gh", ["attestation", "verify", `oci://${image}`,
|
||||
"--repo", "paperclipai/paperclip", "--bundle-from-oci",
|
||||
"--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml",
|
||||
"--source-ref", "refs/heads/master", "--source-digest", sha,
|
||||
"--signer-digest", sha, "--deny-self-hosted-runners"]);
|
||||
await run("docker", ["buildx", "imagetools", "create", "-t",
|
||||
`ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const reused = await reuseCanaryImage(process.env.GITHUB_SHA);
|
||||
if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`);
|
||||
const summary = reused
|
||||
? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge."
|
||||
: "No full-SHA master image exists. Building both platforms for this canary.";
|
||||
console.log(summary);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`);
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import test from "node:test";
|
||||
import { reuseCanaryImage } from "./reuse-canary-image.mjs";
|
||||
|
||||
const sha = "a".repeat(40);
|
||||
const digest = `sha256:${"b".repeat(64)}`;
|
||||
|
||||
test("reuses only an exact-source, master-signed multi-platform image by digest", async () => {
|
||||
const calls = [];
|
||||
const reused = await reuseCanaryImage(sha, {
|
||||
resolve: async source => { assert.equal(source, sha); return digest; },
|
||||
run: async (...args) => calls.push(args),
|
||||
});
|
||||
assert.equal(reused, true);
|
||||
assert.deepEqual(calls, [
|
||||
["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`,
|
||||
"--repo", "paperclipai/paperclip", "--bundle-from-oci",
|
||||
"--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml",
|
||||
"--source-ref", "refs/heads/master", "--source-digest", sha,
|
||||
"--signer-digest", sha, "--deny-self-hosted-runners"]],
|
||||
["docker", ["buildx", "imagetools", "create", "-t",
|
||||
"ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]],
|
||||
]);
|
||||
});
|
||||
|
||||
test("a missing master image enables the fallback without retagging", async () => {
|
||||
assert.equal(await reuseCanaryImage(sha, {
|
||||
resolve: async () => null,
|
||||
run: () => assert.fail("A missing image must not be promoted"),
|
||||
}), false);
|
||||
});
|
||||
|
||||
test("registry and provenance failures stop promotion rather than bypass verification", async () => {
|
||||
await assert.rejects(reuseCanaryImage(sha, {
|
||||
resolve: async () => { throw new Error("HTTP 403"); },
|
||||
run: () => assert.fail("A registry failure must not be promoted"),
|
||||
}), /HTTP 403/);
|
||||
let commands = 0;
|
||||
await assert.rejects(reuseCanaryImage(sha, {
|
||||
resolve: async () => digest,
|
||||
run: command => {
|
||||
assert.equal(command, "gh");
|
||||
commands++;
|
||||
throw new Error("Untrusted source or signature");
|
||||
},
|
||||
}), /Untrusted source/);
|
||||
assert.equal(commands, 1);
|
||||
});
|
||||
|
||||
test("invalid source refs and digests cannot reach external commands", async () => {
|
||||
for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) {
|
||||
await assert.rejects(reuseCanaryImage(invalid, {
|
||||
resolve: () => assert.fail("Invalid source accepted"),
|
||||
}));
|
||||
}
|
||||
await assert.rejects(reuseCanaryImage(sha, {
|
||||
resolve: async () => "mutable:tag",
|
||||
run: () => assert.fail("Invalid digest accepted"),
|
||||
}));
|
||||
});
|
||||
|
||||
test("canary reuse gates both native builds and still runs convergent promotion", () => {
|
||||
const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8");
|
||||
const reuse = workflow.split(" reuse_canary_image:\n")[1].split(" build-and-push:\n")[0];
|
||||
assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/);
|
||||
assert.match(reuse, /node scripts\/reuse-canary-image.mjs/);
|
||||
const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0];
|
||||
assert.match(build, /needs: \[reuse_canary_image\]/);
|
||||
assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/);
|
||||
assert.match(build, /platform: linux\/amd64/);
|
||||
assert.match(build, /platform: linux\/arm64/);
|
||||
const promotion = workflow.split(" promote_canary_channel:\n")[1];
|
||||
assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/);
|
||||
});
|
||||
@@ -21,6 +21,7 @@ export function standardImageDigest(bytes) {
|
||||
/** Read registry bytes directly; CLI stdout formatting is not artifact content. */
|
||||
export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
|
||||
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)),
|
||||
missing = "error",
|
||||
} = {}) {
|
||||
assert.match(sha ?? "", /^[a-f0-9]{40}$/);
|
||||
const request = async (url, headers, consume, allowNotFound = false) => {
|
||||
@@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
|
||||
}
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel();
|
||||
if (allowNotFound && response.status === 404 && missing === "return-null") return null;
|
||||
const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404);
|
||||
assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`);
|
||||
const retryAfter = Number(response.headers.get("retry-after"));
|
||||
@@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
|
||||
};
|
||||
const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json());
|
||||
assert.ok(typeof token === "string" && token, "Missing public pull token");
|
||||
const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, {
|
||||
const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, {
|
||||
authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
}, async response => {
|
||||
assert.ok(response.body, "Standard image index has no body");
|
||||
@@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, {
|
||||
} finally { await reader.cancel(); }
|
||||
return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") };
|
||||
}, true);
|
||||
if (manifest === null) return null;
|
||||
const { raw, expected } = manifest;
|
||||
const digest = standardImageDigest(raw);
|
||||
assert.equal(digest, expected, "Registry digest does not match the index bytes");
|
||||
return digest;
|
||||
|
||||
@@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a
|
||||
assert.equal(calls, expectedCalls);
|
||||
}
|
||||
});
|
||||
|
||||
test("optional lookup falls back only for a missing manifest", async () => {
|
||||
for (const status of [401, 403, 404, 503]) {
|
||||
let calls = 0;
|
||||
const lookup = resolveStandardImageDigest("a".repeat(40), async url => {
|
||||
if (url.includes("/token?")) return Response.json({ token: "public-fixture" });
|
||||
calls++;
|
||||
return new Response(null, { status });
|
||||
}, { missing: "return-null", sleep: async () => {} });
|
||||
if (status === 404) assert.equal(await lookup, null);
|
||||
else await assert.rejects(lookup, /Registry lookup failed/);
|
||||
assert.equal(calls, status === 503 ? 5 : 1);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,897 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql, type SQL } from "drizzle-orm";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { subscribeDatabaseWork, authUsers, companyMemberships, agents, approvals, budgetIncidents, budgetPolicies, companies, createDb, applyPendingMigrations, decisionArchiveNotificationOutbox,
|
||||
decisionRetention, decisions, costEvents, heartbeatRunEvents, heartbeatRuns, issueApprovals, issueRecoveryActions,
|
||||
issueRelations, issueThreadInteractions, issues, type Db } from "@paperclipai/db";
|
||||
import { attentionRetentionService, registerAttentionRetentionWork } from "../services/attention-retention.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { attentionService } from "../services/attention.js";
|
||||
import { companyService } from "../services/companies.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "../services/decision-retention.js";
|
||||
import { appendHeartbeatRunEvent } from "../services/heartbeat-run-events.js";
|
||||
import { buildIssueGraphLivenessIncidentKey, RECOVERY_ORIGIN_KINDS } from "../services/recovery/origins.js";
|
||||
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
|
||||
import { createAgentLifecycle } from "../services/agent-lifecycle.js";
|
||||
import { BLOCKER_ATTENTION_MAX_DEPTH, BLOCKER_ATTENTION_MAX_NODES, issueService } from "../services/issues.js";
|
||||
import { notifyAttentionRetentionWork, notifyIssueAttentionRetentionWork } from "../services/retention-work-notifications.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ATTENTION_RETENTION_TEST_DATABASE_URL");
|
||||
const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const delay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
(support.supported ? describe : describe.skip)("remaining attention retention", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string, agentId: string, draining = false;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
const errors: unknown[] = [];
|
||||
const statements: string[] = [];
|
||||
const parameterCounts: number[] = [];
|
||||
beforeAll(async () => {
|
||||
if (externalUrl) await applyPendingMigrations(externalUrl);
|
||||
else temporary = await startEmbeddedPostgresTestDatabase("attention-retention-");
|
||||
db = createDb(externalUrl ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
db.$client.options.onnotice = () => {};
|
||||
db.$client.options.debug = (_connection, query, parameters) => { statements.push(query); parameterCounts.push(parameters.length); };
|
||||
}, 30_000);
|
||||
beforeEach(async () => {
|
||||
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0;
|
||||
await db.insert(companies).values({ id: companyId, name: "Retention", issuePrefix: `A${companyId.slice(0, 6)}` });
|
||||
await db.insert(authUsers).values({ id: "board", name: "Board", email: "board@retention.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "board", membershipRole: "owner", status: "active" });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Origin", role: "engineer", status: "active" });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true });
|
||||
vi.setSystemTime(new Date("2026-11-01T12:00:00Z"));
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local client_min_messages = warning`);
|
||||
await tx.execute(sql`drop trigger if exists attention_retention_fail on decision_retention`);
|
||||
await tx.execute(sql`drop function if exists attention_retention_fail()`);
|
||||
await tx.execute(sql`truncate companies cascade`);
|
||||
});
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
const old = () => new Date(Date.now() - delay - 600_000);
|
||||
async function insertIssue(status = "todo", values: Partial<typeof issues.$inferInsert> = {}, database: Db = db) {
|
||||
const [row] = await database.insert(issues).values({ companyId, title: randomUUID(), status,
|
||||
createdByAgentId: agentId, createdAt: old(), updatedAt: old(), ...values }).returning();
|
||||
return row;
|
||||
}
|
||||
async function change<T>(write: (tx: Db) => PromiseLike<T>, scope = { companyId }) {
|
||||
return db.transaction(async tx => { await notifyAttentionRetentionWork(tx, scope); return write(tx as unknown as Db); });
|
||||
}
|
||||
const state = (kind: string, id: string) => db.select().from(decisionRetention).where(and(
|
||||
eq(decisionRetention.companyId, companyId), eq(decisionRetention.sourceKind, kind), eq(decisionRetention.sourceId, id)))
|
||||
.then(rows => rows[0] ?? null);
|
||||
async function settle() {
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 15_000 });
|
||||
}
|
||||
async function quiet() {
|
||||
await settle(); expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
|
||||
statements.length = 0;
|
||||
await vi.advanceTimersByTimeAsync(300_000);
|
||||
expect(statements).toEqual([]);
|
||||
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
|
||||
select.mockRestore(); execute.mockRestore(); transaction.mockRestore();
|
||||
}
|
||||
async function start(service = attentionRetentionService(db)) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true, onError: error => errors.push(error) });
|
||||
await registerAttentionRetentionWork(coordinator, service, () => !draining).ready;
|
||||
return service;
|
||||
}
|
||||
async function archived(kind: string, id: string) {
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await state(kind, id))?.archivedAt).toBeInstanceOf(Date); }, { timeout: 15_000 });
|
||||
}
|
||||
async function loadAll(service = attentionRetentionService(db)) {
|
||||
const entries = [];
|
||||
for (;;) { const page = await service.load({ companyId }); entries.push(...page.rows ?? []); if (page.complete) return entries; }
|
||||
}
|
||||
const mutation = (sourceKind: "review", sourceId: string) => ({ companyId, sourceKind, sourceId,
|
||||
authActor: { type: "board" as const, source: "local_implicit" as const, userId: "board", companyIds: [companyId], isInstanceAdmin: true },
|
||||
actor: { actorType: "user" as const, actorId: "board", userId: "board", agentId: null, runId: null, agentApiKeyId: null, responsibleUserId: "board" } });
|
||||
|
||||
it("records the aggregate baseline SQL across old ticks and eliminates it after settlement", async () => {
|
||||
await insertIssue("done");
|
||||
const ticks: number[] = [];
|
||||
for (let tick = 0; tick < 3; tick++) {
|
||||
statements.length = 0;
|
||||
const feed = await attentionService(db).list(companyId, { includeDismissed: true, excludeSources: ["approval", "join_request"], all: true, allowUnscopedAll: true });
|
||||
await decisionRetentionService(db).autoArchive({ companyId, items: feed.items });
|
||||
ticks.push(statements.length);
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
}
|
||||
expect(ticks.every(count => count > 0)).toBe(true);
|
||||
process.stdout.write(JSON.stringify({ baselineStatementsPer30sTick: ticks, treatmentIdle300s: 0 }) + "\n");
|
||||
await start(); await quiet();
|
||||
});
|
||||
|
||||
it("has no recurring SQL when empty or when terminal source history remains", async () => {
|
||||
await insertIssue("done"); await insertIssue("cancelled");
|
||||
await start(); await quiet();
|
||||
});
|
||||
|
||||
it("wakes from the ordinary task producer, waits until due, archives with an outbox, and settles", async () => {
|
||||
await start(); await quiet();
|
||||
const row = await issueService(db).create(companyId, { title: "Review a delivered result", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
|
||||
await settle();
|
||||
const [saved] = await db.select({ at: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000)`.mapWith(Number) }).from(issues).where(eq(issues.id, row.id));
|
||||
const due = saved.at + delay;
|
||||
expect(coordinator!.nextWakeAt()).toBe(due); expect(await state("review", row.id)).toBeNull();
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute");
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1000);
|
||||
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); select.mockRestore(); execute.mockRestore();
|
||||
await vi.advanceTimersByTimeAsync(2000); await archived("review", row.id); await quiet();
|
||||
const outbox = await db.select().from(decisionArchiveNotificationOutbox);
|
||||
expect(outbox).toHaveLength(1); expect(outbox[0]).toMatchObject({ companyId, sourceKind: "review", sourceId: row.id, originAgentId: agentId, originIssueId: row.id });
|
||||
expect((await db.select().from(issues).where(eq(issues.id, row.id)))[0].status).toBe("in_review");
|
||||
await coordinator!.stop(); coordinator = undefined; await start(); await quiet();
|
||||
});
|
||||
|
||||
it("registers the ordinary createChild blocker path inside its relation transaction", async () => {
|
||||
const parent = await issueService(db).create(companyId, { title: "Parent", status: "in_progress", assigneeAgentId: agentId, createdByAgentId: agentId });
|
||||
await start(); await quiet();
|
||||
const { issue: child, parentBlockerAdded } = await issueService(db).createChild(parent.id, {
|
||||
title: "Child needs review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId,
|
||||
blockParentUntilDone: true,
|
||||
});
|
||||
expect(parentBlockerAdded).toBe(true);
|
||||
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.issueId, child.id),
|
||||
eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(delay + 2000); await archived("review", child.id); await quiet();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, child.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("batches a full page of maintained reviews and confirmations by company", async () => {
|
||||
const rows = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "in_review", assigneeAgentId: agentId, updatedAt: old() }))).returning();
|
||||
await db.insert(issueThreadInteractions).values(rows.map(row => ({ companyId, issueId: row.id,
|
||||
kind: "request_confirmation", addresseeAgentId: agentId, payload: { version: 1, prompt: "Agent verdict" } })));
|
||||
statements.length = 0;
|
||||
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "review" });
|
||||
expect(page.rows).toEqual([]); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(20);
|
||||
expect(statements.filter(query => query.includes('from "agents"')).length).toBeLessThanOrEqual(1);
|
||||
await db.update(agents).set({ status: "paused" }).where(eq(agents.id, agentId));
|
||||
statements.length = 0;
|
||||
const interactions = await attentionRetentionService(db).load({ companyId, sourceKind: "issue_thread_interaction" });
|
||||
expect(interactions.rows).toHaveLength(100); expect(interactions.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(6);
|
||||
expect(statements.filter(query => query.includes('from "agents"'))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("batches decision and recovery source links and rechecks a late hidden origin", async () => {
|
||||
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded" }).returning();
|
||||
const decisionRows = await db.insert(decisions).values(origins.map(origin => ({ companyId,
|
||||
originAgentId: agentId, originIssueId: origin.id, originRunId: run.id, title: "Decision", body: "Choose",
|
||||
options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }))).returning();
|
||||
const recoveryRows = await db.insert(issueRecoveryActions).values(origins.map(origin => ({ companyId,
|
||||
sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture",
|
||||
nextAction: "Choose", updatedAt: old() }))).returning();
|
||||
const service = attentionRetentionService(db);
|
||||
const pages = [];
|
||||
for (const sourceKind of ["decision", "recovery_action"] as const) {
|
||||
statements.length = 0;
|
||||
const page = await service.load({ companyId, sourceKind });
|
||||
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(3);
|
||||
pages.push(page.rows!);
|
||||
}
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[99].id));
|
||||
const hiddenDecision = pages[0].find(entry => entry.sourceId === decisionRows[99].id)!;
|
||||
const hiddenRecovery = pages[1].find(entry => entry.sourceId === recoveryRows[99].id)!;
|
||||
expect(await service.check(hiddenDecision)).toBeNull(); expect(await service.check(hiddenRecovery)).toBeNull();
|
||||
expect(await state("decision", hiddenDecision.sourceId)).toBeNull(); expect(await state("recovery_action", hiddenRecovery.sourceId)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("batches 100 failed runs and rechecks late visibility and supersession before archive", async () => {
|
||||
const origins = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: old() }))).returning();
|
||||
const runs = await db.insert(heartbeatRuns).values(origins.map(origin => ({ companyId, agentId,
|
||||
status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }))).returning();
|
||||
await db.insert(heartbeatRunEvents).values(runs.map(run => ({ companyId, agentId, runId: run.id,
|
||||
seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: batch fixture" })));
|
||||
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "failed_run" as const };
|
||||
statements.length = 0;
|
||||
const page = await service.load(scope);
|
||||
expect(page.rows).toHaveLength(100); expect(page.complete).toBe(false);
|
||||
expect(statements.length).toBeLessThanOrEqual(6);
|
||||
const hiddenRun = runs[98], supersededRun = runs[99];
|
||||
const hidden = page.rows!.find(entry => entry.sourceId === hiddenRun.id)!;
|
||||
const superseded = page.rows!.find(entry => entry.sourceId === supersededRun.id)!;
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, origins[98].id));
|
||||
await db.insert(heartbeatRuns).values({ companyId, agentId, status: "succeeded",
|
||||
contextSnapshot: { issueId: origins[99].id }, createdAt: new Date(old().getTime() + 60_000), finishedAt: old() });
|
||||
expect(await service.check(hidden)).toBeNull(); expect(await service.check(superseded)).toBeNull();
|
||||
expect(await state("failed_run", hiddenRun.id)).toBeNull(); expect(await state("failed_run", supersededRun.id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
expect((await service.load(scope)).complete).toBe(true);
|
||||
const refreshed = await service.load(scope);
|
||||
expect(refreshed.rows).toHaveLength(98);
|
||||
expect(refreshed.rows!.some(entry => [hiddenRun.id, supersededRun.id].includes(entry.sourceId))).toBe(false);
|
||||
});
|
||||
|
||||
it("batches 100 blocker graphs with exact terminal timestamps and fresh terminal visibility", async () => {
|
||||
const roots = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "blocked", updatedAt: old(), blockedTransitionAt: old(),
|
||||
unblockDescriptor: { owner: "board", action: "Pick an owner" } }))).returning();
|
||||
const terminals = await db.insert(issues).values(Array.from({ length: 100 }, () => ({ companyId,
|
||||
title: randomUUID(), status: "backlog", assigneeAgentId: agentId, updatedAt: old() }))).returning();
|
||||
await db.insert(issueRelations).values(roots.map((root, index) => ({ companyId,
|
||||
issueId: terminals[index].id, relatedIssueId: root.id, type: "blocks" })));
|
||||
await db.execute(sql`update issues set updated_at = updated_at + interval '0.000321 seconds' where id = ${terminals[99].id}::uuid`);
|
||||
const service = attentionRetentionService(db), scope = { companyId, sourceKind: "blocker_attention" as const };
|
||||
statements.length = 0;
|
||||
const page = await service.load(scope);
|
||||
const queries = [...statements];
|
||||
expect(page.rows).toHaveLength(200); expect(page.complete).toBe(false);
|
||||
const retentionReads = queries.filter(query => query.includes('from "decision_retention"'));
|
||||
expect(retentionReads).toHaveLength(200);
|
||||
expect(queries.length - retentionReads.length).toBeLessThanOrEqual(30);
|
||||
const terminal = page.rows!.find(entry => entry.rootId === roots[99].id && entry.variant === "terminal")!;
|
||||
expect(terminal.sourceId).toBe(terminals[99].id);
|
||||
const [expected] = await db.execute(sql`select updated_at::text as at,
|
||||
ceil(extract(epoch from updated_at) * 1000 + ${delay}) as due from issues where id = ${terminals[99].id}::uuid`);
|
||||
expect(terminal.activityAt).toBe(expected.at); expect(terminal.dueAt).toBe(Number(expected.due));
|
||||
await db.update(issues).set({ hiddenAt: new Date() }).where(eq(issues.id, terminals[99].id));
|
||||
expect(await service.check(terminal)).toBeNull();
|
||||
expect(await state("blocker_attention", terminals[99].id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
expect((await service.load(scope)).complete).toBe(true);
|
||||
const refreshed = await service.load(scope);
|
||||
expect(refreshed.rows!.some(entry => entry.sourceId === terminals[99].id)).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["depth", "nodes"] as const)("preserves shallow terminal selection when an unrelated graph hits its %s limit", async limit => {
|
||||
const root = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const middle = await insertIssue("todo", { updatedAt: new Date() });
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
const heavy = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const count = limit === "depth" ? BLOCKER_ATTENTION_MAX_DEPTH + 1 : BLOCKER_ATTENTION_MAX_NODES;
|
||||
const heavyNodes = await db.insert(issues).values(Array.from({ length: count }, () => ({ companyId,
|
||||
title: randomUUID(), status: "todo", updatedAt: new Date() }))).returning();
|
||||
await db.insert(issueRelations).values([
|
||||
{ companyId, issueId: middle.id, relatedIssueId: root.id, type: "blocks" },
|
||||
{ companyId, issueId: terminal.id, relatedIssueId: middle.id, type: "blocks" },
|
||||
...heavyNodes.map((node, index) => ({ companyId, issueId: node.id,
|
||||
relatedIssueId: limit === "nodes" || index === 0 ? heavy.id : heavyNodes[index - 1].id, type: "blocks" })),
|
||||
]);
|
||||
const owner = issueService(db);
|
||||
const expected = (await owner.listBlockerAttention(companyId, [root])).get(root.id)!;
|
||||
const limited = (await owner.listBlockerAttention(companyId, [heavy])).get(heavy.id)!;
|
||||
expect(expected.terminalBlockerIssueId).toBe(terminal.id);
|
||||
expect(limited.terminalBlockerIssueId).toBe(limited.directBlockerIssueId);
|
||||
for (const input of [[root, heavy], [heavy, root]]) {
|
||||
const batch = await owner.listBlockerAttentionBatch(companyId, input);
|
||||
if (!batch.deferredRootIds.has(root.id)) expect(batch.attention.get(root.id)).toEqual(expected);
|
||||
if (!batch.deferredRootIds.has(heavy.id)) expect(batch.attention.get(heavy.id)).toEqual(limited);
|
||||
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
if (limit === "depth") expect(batch.deferredRootIds.has(root.id)).toBe(false);
|
||||
}
|
||||
const page = await attentionRetentionService(db).load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(entry.dueAt).toBeLessThanOrEqual(Date.now());
|
||||
if (limit === "depth") expect(entry.sourceId).toBe(terminal.id);
|
||||
await attentionRetentionService(db).check(entry);
|
||||
expect((await state("blocker_attention", terminal.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, terminal.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps root-specific reachability and cycle budgets when another root explores farther", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const chain = await db.insert(issues).values(Array.from({ length: BLOCKER_ATTENTION_MAX_DEPTH + 1 }, (_, index) => ({
|
||||
companyId, title: randomUUID(), status: index === BLOCKER_ATTENTION_MAX_DEPTH ? "in_progress"
|
||||
: index === BLOCKER_ATTENTION_MAX_DEPTH - 2 ? "blocked" : "todo", updatedAt: old() }))).returning();
|
||||
const overlapping = chain[BLOCKER_ATTENTION_MAX_DEPTH - 2];
|
||||
const cycle = await insertIssue("blocked");
|
||||
const leaf = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values([
|
||||
...chain.map((node, index) => ({ companyId, issueId: node.id, relatedIssueId: index ? chain[index - 1].id : root.id, type: "blocks" })),
|
||||
{ companyId, issueId: leaf.id, relatedIssueId: cycle.id, type: "blocks" },
|
||||
{ companyId, issueId: cycle.id, relatedIssueId: leaf.id, type: "blocks" },
|
||||
]);
|
||||
const owner = issueService(db), roots = [root, overlapping, cycle];
|
||||
const singles = new Map();
|
||||
for (const item of roots) singles.set(item.id, (await owner.listBlockerAttention(companyId, [item])).get(item.id));
|
||||
expect(singles.get(root.id).blockingTreeLive).toBe(false);
|
||||
expect(singles.get(overlapping.id).blockingTreeLive).toBe(true);
|
||||
for (const input of [roots, [...roots].reverse()]) {
|
||||
const batch = await owner.listBlockerAttention(companyId, input);
|
||||
for (const item of roots) expect(batch.get(item.id)).toEqual(singles.get(item.id));
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds every graph query when independent root trees contain over 65535 unfinished nodes", async () => {
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'parameter-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 70) n`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, root.id, 'parameter-branch-' || n, 'todo', ${old().toISOString()}::timestamptz
|
||||
from issues root cross join generate_series(1, 10) n
|
||||
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, branch.id, 'parameter-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from issues branch cross join generate_series(1, 99) n
|
||||
where branch.company_id = ${companyId}::uuid and branch.title like 'parameter-branch-%'`);
|
||||
const completed = await insertIssue("blocked");
|
||||
const shallow = await insertIssue("blocked", { updatedAt: new Date() });
|
||||
const middle = await insertIssue("todo", { updatedAt: new Date() });
|
||||
const overdue = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values([
|
||||
{ companyId, issueId: middle.id, relatedIssueId: shallow.id, type: "blocks" },
|
||||
{ companyId, issueId: overdue.id, relatedIssueId: middle.id, type: "blocks" },
|
||||
]);
|
||||
const roots = await db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.status, "blocked")));
|
||||
parameterCounts.length = 0;
|
||||
const result = await issueService(db).listBlockerAttentionBatch(companyId, roots).catch(error => {
|
||||
throw new Error(`Large graph failed: ${String(error.cause?.message ?? error.message).slice(0, 300)}`);
|
||||
});
|
||||
expect(result.attention.size).toBe(72);
|
||||
expect(result.deferredRootIds.has(completed.id)).toBe(false);
|
||||
expect(result.attention.get(completed.id)?.terminalBlockerIssueId).toBe(completed.id);
|
||||
expect([...result.attention.values()].every(item => item.state === "needs_attention" && item.terminalBlockerIssueId)).toBe(true);
|
||||
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
|
||||
expect(parameterCounts.length).toBeLessThanOrEqual(100);
|
||||
expect(result.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
expect(result.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
|
||||
expect(result.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
|
||||
expect(result.deferredRootIds.has(shallow.id)).toBe(true);
|
||||
const service = attentionRetentionService(db);
|
||||
const page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const immediate = page.rows!.find(entry => entry.rootId === shallow.id && entry.variant === "terminal")!;
|
||||
expect(immediate.dueAt).toBeLessThanOrEqual(Date.now());
|
||||
await service.check(immediate);
|
||||
expect((await state("blocker_attention", overdue.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
}, 120_000);
|
||||
|
||||
it("preserves observed live work when a single large root exhausts shared fetch capacity", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const live = await insertIssue("in_progress", { assigneeAgentId: agentId });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: live.id, relatedIssueId: root.id, type: "blocks" });
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, ${root.id}::uuid, 'bounded-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 3000) n`);
|
||||
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
|
||||
expect(batch.deferredRootIds.has(root.id)).toBe(true);
|
||||
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
|
||||
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(await service.check(entry)).toBeNull();
|
||||
expect(await state("blocker_attention", root.id)).toBeNull();
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(["in_progress", "run", "confirmation"])("preserves observed child live evidence (%s) after explicit blockers fill the node budget", async (kind) => {
|
||||
const root = await insertIssue("blocked");
|
||||
await db.execute(sql`with blockers as (
|
||||
insert into issues (id, company_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'explicit-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 1999) n returning id)
|
||||
insert into issue_relations (company_id, issue_id, related_issue_id, type)
|
||||
select ${companyId}::uuid, id, ${root.id}::uuid, 'blocks' from blockers`);
|
||||
const child = await insertIssue(kind === "in_progress" ? "in_progress" : "todo", { parentId: root.id, assigneeAgentId: agentId });
|
||||
if (kind === "run") {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, child.id));
|
||||
} else if (kind === "confirmation") {
|
||||
await db.insert(issueThreadInteractions).values({ companyId, issueId: child.id, kind: "request_confirmation", status: "pending", payload: { prompt: "Approve?" } });
|
||||
}
|
||||
parameterCounts.length = 0;
|
||||
const batch = await issueService(db).listBlockerAttentionBatch(companyId, [root]);
|
||||
expect(batch.nodeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES);
|
||||
expect(batch.edgeCount).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 2);
|
||||
expect(batch.fetchedRows).toBeLessThanOrEqual(BLOCKER_ATTENTION_MAX_NODES * 6);
|
||||
expect(parameterCounts.length).toBeLessThanOrEqual(100);
|
||||
expect(Math.max(...parameterCounts)).toBeLessThanOrEqual(1100);
|
||||
expect(batch.deferredRootIds.has(root.id)).toBe(true);
|
||||
expect(batch.attention.get(root.id)?.blockingTreeLive).toBe(true);
|
||||
const service = attentionRetentionService(db), page = await service.load({ companyId, sourceKind: "blocker_attention" });
|
||||
const entry = page.rows!.find(item => item.rootId === root.id && item.variant === "terminal")!;
|
||||
expect(await service.check(entry)).toBeNull();
|
||||
expect(await db.select().from(decisionRetention)).toHaveLength(0);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("disarms finite deferred-root checks when large retained graphs have no archive obligation", async () => {
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'held-root-' || n, 'blocked', ${old().toISOString()}::timestamptz
|
||||
from generate_series(1, 4) n`);
|
||||
await db.execute(sql`insert into issues (id, company_id, parent_id, title, status, assignee_agent_id, updated_at)
|
||||
select gen_random_uuid(), ${companyId}::uuid, root.id, 'held-leaf-' || n, 'backlog', ${agentId}::uuid, ${old().toISOString()}::timestamptz
|
||||
from issues root cross join generate_series(1, 600) n
|
||||
where root.company_id = ${companyId}::uuid and root.status = 'blocked'`);
|
||||
await db.execute(sql`insert into decision_retention (company_id, source_kind, source_id, source_activity_at, keep)
|
||||
select company_id, 'blocker_attention', id::text, updated_at, true from issues
|
||||
where company_id = ${companyId}::uuid and status = 'backlog'`);
|
||||
const service = attentionRetentionService(db), check = vi.spyOn(service, "check");
|
||||
await start(service); await quiet();
|
||||
expect(check).toHaveBeenCalledTimes(4);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("bounds high-fanin ordinary issue writes to one coalescible hint", async () => {
|
||||
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
const leaf = await insertIssue("in_review", { updatedAt: new Date(), assigneeUserId: "board" });
|
||||
await db.execute(sql`with parents as (
|
||||
insert into issues (id, company_id, title, status)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'fan-in-' || n, 'todo'
|
||||
from generate_series(1, 2100) n returning id)
|
||||
insert into issue_relations (company_id, issue_id, related_issue_id, type)
|
||||
select ${companyId}::uuid, ${leaf.id}::uuid, id, 'blocks' from parents`);
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
let release!: () => void, entered = false;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
|
||||
statements.length = 0;
|
||||
try {
|
||||
await issueService(db).update(leaf.id, { title: "High fan-in ordinary edit 0" });
|
||||
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
|
||||
for (let n = 1; n < 8; n++) await issueService(db).update(leaf.id, { title: `High fan-in ordinary edit ${n}` });
|
||||
} finally { release(); unsubscribe(); }
|
||||
expect(statements.filter(query => query.includes("with recursive affected"))).toHaveLength(8);
|
||||
expect(hints).toHaveLength(8);
|
||||
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId })]));
|
||||
await settle();
|
||||
expect(load.mock.calls.length).toBeLessThanOrEqual(3);
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
|
||||
let query!: Parameters<typeof db.execute>[0], returned = 0;
|
||||
await db.transaction(async tx => {
|
||||
const execute = tx.execute.bind(tx);
|
||||
vi.spyOn(tx, "execute").mockImplementation(async statement => {
|
||||
const result = await execute(statement);
|
||||
if (new PgDialect().sqlToQuery(statement as SQL).sql.includes("with recursive affected")) {
|
||||
query = statement; returned = result.length;
|
||||
}
|
||||
return result;
|
||||
});
|
||||
await notifyIssueAttentionRetentionWork(tx, { companyId, issueId: leaf.id });
|
||||
});
|
||||
expect(returned).toBe(101);
|
||||
const plans = await db.execute(sql`explain (analyze, format json) ${query as SQL}`);
|
||||
const plan = (plans[0]["QUERY PLAN"] as any)[0].Plan;
|
||||
expect(plan["Node Type"]).toBe("Limit");
|
||||
expect(plan["Actual Rows"]).toBe(101);
|
||||
const nodes: any[] = [], collect = (node: any) => { nodes.push(node); for (const child of node.Plans ?? []) collect(child); };
|
||||
collect(plan);
|
||||
const recursive = nodes.filter(node => node["Node Type"] === "Recursive Union");
|
||||
const relations = nodes.filter(node => node["Relation Name"] === "issue_relations");
|
||||
expect(recursive).toHaveLength(1);
|
||||
expect(recursive[0]["Actual Rows"]).toBe(101);
|
||||
expect(relations.length).toBeGreaterThan(0);
|
||||
expect(relations.every(node => node["Actual Rows"] * node["Actual Loops"] <= 101)).toBe(true);
|
||||
process.stdout.write(JSON.stringify({ reverseFanInPlan: plan }) + "\n");
|
||||
});
|
||||
|
||||
it("retains far ancestor deadlines when a cyclic chain exceeds the precise hint budget", async () => {
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
await db.execute(sql`insert into issues (id, company_id, title, status, updated_at, assignee_user_id)
|
||||
select gen_random_uuid(), ${companyId}::uuid, 'chain-' || n,
|
||||
case when n = 150 then 'in_review' else 'todo' end,
|
||||
case when n = 150 then ${new Date(Date.now() - delay + 15_000).toISOString()}::timestamptz else now() end,
|
||||
case when n = 150 then 'board' else null end
|
||||
from generate_series(0, 150) n`);
|
||||
await db.execute(sql`update issues child set parent_id = parent.id from issues parent
|
||||
where child.company_id = ${companyId}::uuid and parent.company_id = child.company_id
|
||||
and parent.title = 'chain-' || ((substring(child.title from 7)::integer + 1) % 151)`);
|
||||
const [leaf, far] = await Promise.all([db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-0"))).then(rows => rows[0]),
|
||||
db.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.title, "chain-150"))).then(rows => rows[0])]);
|
||||
await issueService(db).update(leaf.id, { title: "Ordinary deep chain edit" });
|
||||
await settle();
|
||||
expect(load.mock.calls.some(([scope]) => scope?.companyId === companyId && !scope.issueId && !scope.issueIds)).toBe(true);
|
||||
expect(await state("review", far.id)).toBeNull();
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await archived("review", far.id);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("canonicalizes a complete cyclic affected set into one bounded batch hint", async () => {
|
||||
const a = await insertIssue(), b = await insertIssue(), c = await insertIssue();
|
||||
await db.update(issues).set({ parentId: b.id }).where(eq(issues.id, a.id));
|
||||
await db.update(issues).set({ parentId: c.id }).where(eq(issues.id, b.id));
|
||||
await db.update(issues).set({ parentId: a.id }).where(eq(issues.id, c.id));
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try {
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: a.id }));
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, issueId: c.id }));
|
||||
const wide = { companyId, issueIds: [b.id, a.id, c.id, a.id], ignored: {} as unknown };
|
||||
wide.ignored = wide;
|
||||
await db.transaction(tx => notifyAttentionRetentionWork(tx, wide));
|
||||
} finally { unsubscribe(); }
|
||||
expect(hints).toHaveLength(3);
|
||||
expect(new Set(hints)).toEqual(new Set([JSON.stringify({ companyId, issueIds: [a.id, b.id, c.id].sort() })]));
|
||||
});
|
||||
|
||||
it("canonicalizes wide circular notification variables to the supported identity fields", async () => {
|
||||
const sourceId = randomUUID(), issueId = randomUUID(), hints: string[] = [];
|
||||
const wide = { companyId, sourceKind: "review" as const, sourceId, issueId, ignored: {} as unknown };
|
||||
wide.ignored = wide;
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try { await db.transaction(tx => notifyAttentionRetentionWork(tx, wide)); } finally { unsubscribe(); }
|
||||
expect(hints).toEqual([JSON.stringify({ companyId, sourceKind: "review", sourceId, issueId })]);
|
||||
});
|
||||
|
||||
it("keeps held, burst and sustained ordinary issue writes scoped while an unrelated deadline fires", async () => {
|
||||
const row = await issueService(db).create(companyId, { title: "Active review", status: "in_review", assigneeUserId: "board", createdByAgentId: agentId });
|
||||
const urgent = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date(Date.now() - delay + 10_000) });
|
||||
const otherCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Unrelated", issuePrefix: "BURST" });
|
||||
const foreign = await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
|
||||
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() }))).returning();
|
||||
await db.insert(decisionRetention).values(foreign.map(issue => ({ companyId: otherCompany, sourceKind: "review", sourceId: issue.id, sourceActivityAt: issue.updatedAt, keep: true })));
|
||||
const service = attentionRetentionService(db), original = service.load, load = vi.spyOn(service, "load");
|
||||
await start(service); await settle(); load.mockClear();
|
||||
let release!: () => void, entered = false;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
load.mockImplementationOnce(async (...args) => { const page = await original(...args); entered = true; await gate; return page; });
|
||||
statements.length = 0;
|
||||
await issueService(db).update(row.id, { title: "Held mutation" });
|
||||
await vi.waitFor(() => expect(entered).toBe(true), { timeout: 15_000 });
|
||||
await Promise.all(Array.from({ length: 5 }, (_, index) => issueService(db).update(row.id, { title: `Burst ${index}` })));
|
||||
release(); await settle();
|
||||
for (let index = 0; index < 8; index++) {
|
||||
await issueService(db).update(row.id, { title: `Steady ${index}` }); await vi.advanceTimersByTimeAsync(1000); await settle();
|
||||
}
|
||||
await archived("review", urgent.id);
|
||||
const routine = load.mock.calls.map(([scope]) => scope).filter(scope => !scope?.sourceId);
|
||||
expect(routine.length).toBeGreaterThan(0);
|
||||
expect(routine.every(scope => scope?.companyId === companyId && scope.issueId === row.id)).toBe(true);
|
||||
const scans = statements.filter(query => query.includes("with source_candidates"));
|
||||
expect(scans.length).toBeGreaterThan(0);
|
||||
expect(scans.every(query => !query.includes("budget_incidents") && !query.includes("agent_error_alert"))).toBe(true);
|
||||
expect(await state("review", row.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("wakes all linked issue sources when an ordinary visibility mutation reveals their origin", async () => {
|
||||
const origin = await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board", responsibleUserId: "board" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() }).returning();
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: visibility" });
|
||||
const [decision] = await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
|
||||
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() }).returning();
|
||||
const [recovery] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() }).returning();
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() }).returning();
|
||||
await start(); await quiet();
|
||||
await issueService(db).update(origin.id, { hiddenAt: null });
|
||||
for (const [kind, id] of [["decision", decision.id], ["recovery_action", recovery.id], ["failed_run", run.id], ["issue_thread_interaction", interaction.id]]) await archived(kind, id);
|
||||
await settle(); expect(await state("review", origin.id)).toBeNull(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
});
|
||||
|
||||
it("invalidates old and new parents, terminal visibility and repointed blocker relations", async () => {
|
||||
const oldParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
|
||||
const newParent = await insertIssue("blocked", { updatedAt: new Date(), responsibleUserId: "board" });
|
||||
const terminal = await insertIssue("backlog", { parentId: oldParent.id, hiddenAt: old(), assigneeAgentId: agentId, responsibleUserId: "board" });
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load"); await start(service); await settle(); load.mockClear();
|
||||
await issueService(db).update(terminal.id, { parentId: newParent.id }); await settle();
|
||||
const ids = load.mock.calls.flatMap(([scope]) => scope?.issueIds ?? (scope?.issueId ? [scope.issueId] : []));
|
||||
expect(ids).toEqual(expect.arrayContaining([oldParent.id, newParent.id, terminal.id]));
|
||||
load.mockClear();
|
||||
await issueService(db).update(terminal.id, { hiddenAt: null }); await settle();
|
||||
expect(await state("blocker_attention", terminal.id)).toBeNull();
|
||||
const visible = await service.load({ companyId, issueId: newParent.id });
|
||||
expect(visible.rows!.some(entry => entry.sourceId === terminal.id && entry.dueAt > Date.now())).toBe(true);
|
||||
expect(load.mock.calls.some(([scope]) => scope?.issueId === newParent.id || scope?.issueIds?.includes(newParent.id))).toBe(true);
|
||||
const next = await insertIssue("backlog", { assigneeAgentId: agentId });
|
||||
await issueService(db).update(oldParent.id, { blockedByIssueIds: [next.id] }); await archived("blocker_attention", next.id);
|
||||
});
|
||||
|
||||
it("cancels and reactivates a blocker deadline through ordinary liveness incident creation and hiding", async () => {
|
||||
const root = await insertIssue("blocked");
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(Date.now() - delay + 10_000) });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
|
||||
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
const incident = await issueService(db).create(companyId, { title: "Resolve liveness", status: "todo", assigneeUserId: "board",
|
||||
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
|
||||
originId: buildIssueGraphLivenessIncidentKey({ companyId, issueId: root.id, state: "needs_attention", blockerIssueId: terminal.id }) });
|
||||
await quiet(); expect(await state("blocker_attention", terminal.id)).toBeNull();
|
||||
await issueService(db).update(incident.id, { hiddenAt: new Date() });
|
||||
await archived("blocker_attention", terminal.id); await quiet();
|
||||
});
|
||||
|
||||
it("resolves opaque run issue references through trusted company issue rows", async () => {
|
||||
const issue = await insertIssue();
|
||||
const [valid, opaque] = await db.insert(heartbeatRuns).values([
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: issue.id } },
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: "none" } },
|
||||
]).returning();
|
||||
const hints: string[] = [];
|
||||
const unsubscribe = subscribeDatabaseWork(db, DELIVERY_QUEUES.attentionRetention, (event, values) => {
|
||||
if (event === "settled") hints.push(...values ?? []);
|
||||
});
|
||||
try {
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: valid.id }));
|
||||
await db.transaction(tx => notifyIssueAttentionRetentionWork(tx, { companyId, runId: opaque.id }));
|
||||
} finally { unsubscribe(); }
|
||||
expect(hints).toEqual([JSON.stringify({ companyId, issueId: issue.id }), JSON.stringify({ companyId })]);
|
||||
});
|
||||
|
||||
it("accepts ordinary liveness incident creation and completion with a none leaf sentinel", async () => {
|
||||
const source = await insertIssue("blocked", { responsibleUserId: "board" });
|
||||
const service = issueService(db);
|
||||
const incident = await service.create(companyId, { title: "Invalid participant", status: "todo", assigneeUserId: "board",
|
||||
createdByUserId: "board", originKind: RECOVERY_ORIGIN_KINDS.issueGraphLivenessEscalation,
|
||||
originId: `harness_liveness:${companyId}:${source.id}:invalid_review_participant:none` });
|
||||
await service.update(source.id, { blockedByIssueIds: [incident.id] });
|
||||
await service.update(incident.id, { status: "done" });
|
||||
expect((await service.getById(incident.id))?.status).toBe("done");
|
||||
expect((await service.getRelationSummaries(source.id)).blockedBy).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["hidden", "done"] as const)("reactivates an older review when moved stranded recovery is %s", async transition => {
|
||||
const review = await insertIssue("in_review", { assigneeAgentId: agentId, responsibleUserId: "board" });
|
||||
const parent = await insertIssue("todo", { responsibleUserId: "board" });
|
||||
const recovery = await insertIssue("todo", { parentId: review.id, originKind: RECOVERY_ORIGIN_KINDS.strandedIssueRecovery,
|
||||
originId: review.id, assigneeUserId: "board", responsibleUserId: "board" });
|
||||
await issueService(db).update(recovery.id, { parentId: parent.id });
|
||||
expect((await issueService(db).listReviewAttention(companyId, [review])).get(review.id)?.state).not.toBe("stalled");
|
||||
await start(); await quiet();
|
||||
await issueService(db).update(recovery.id, transition === "hidden" ? { hiddenAt: new Date() } : { status: "done" });
|
||||
await settle();
|
||||
expect((await state("review", review.id))?.archivedAt).toBeInstanceOf(Date);
|
||||
expect(await db.select().from(decisionArchiveNotificationOutbox).where(eq(decisionArchiveNotificationOutbox.sourceId, review.id))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("scopes ordinary execution-control writes to their company", async () => {
|
||||
const otherCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Other", issuePrefix: "OTHER" });
|
||||
await db.insert(issues).values(Array.from({ length: 125 }, () => ({ companyId: otherCompany,
|
||||
title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
const service = attentionRetentionService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await quiet(); load.mockClear();
|
||||
for (let count = 0; count < 3; count++) {
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization,
|
||||
tx => tx.update(heartbeatRuns).set({ updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await quiet();
|
||||
}
|
||||
expect(load).toHaveBeenCalled();
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
|
||||
});
|
||||
|
||||
it("registers and cancels budget deadlines through ordinary policy and incident producers", async () => {
|
||||
const service = budgetService(db);
|
||||
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 100,
|
||||
warnPercent: 85, hardStopEnabled: false, notifyEnabled: true }, "board");
|
||||
const retention = attentionRetentionService(db), load = vi.spyOn(retention, "load");
|
||||
await start(retention); await quiet(); load.mockClear();
|
||||
const [event] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai",
|
||||
billingType: "metered_api", model: "fixture", inputTokens: 100, outputTokens: 20,
|
||||
costCents: 90, occurredAt: new Date() }).returning();
|
||||
await service.evaluateCostEvent(event);
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
const [incident] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId));
|
||||
expect(incident).toMatchObject({ status: "open", thresholdType: "soft" });
|
||||
expect(await state("budget_alert", incident.id)).toBeNull();
|
||||
await service.upsertPolicy(companyId, { scopeType: "agent", scopeId: agentId, amount: 200 }, "board");
|
||||
await quiet();
|
||||
expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0].status).toBe("resolved");
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === companyId)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches the existing feed across all eight remaining source kinds and exclusion variants", async () => {
|
||||
const origin = await insertIssue();
|
||||
const review = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
const blocker = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Choose a dependency" }, blockedTransitionAt: old() });
|
||||
await insertIssue("in_review", { hiddenAt: old(), assigneeUserId: "board" });
|
||||
const hidden = await insertIssue("todo", { hiddenAt: old() });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), updatedAt: old(), finishedAt: old() }).returning();
|
||||
await db.insert(heartbeatRunEvents).values({ companyId, agentId, runId: run.id, seq: 1, eventType: "lifecycle", message: "Bounded retry exhausted: fixture" });
|
||||
await db.insert(decisions).values({ companyId, originAgentId: agentId, originIssueId: origin.id, originRunId: run.id,
|
||||
title: "Decision", body: "Choose", options: [], signedSpec: "fixture", targetSnapshots: {}, expiresAt: new Date(Date.now() + 86_400_000), updatedAt: old() });
|
||||
await db.insert(issueThreadInteractions).values([
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeAgentId: agentId, updatedAt: old() },
|
||||
{ companyId, issueId: origin.id, kind: "ask_user_questions", payload: { questions: [] }, addresseeUserId: "board", updatedAt: old() },
|
||||
{ companyId, issueId: hidden.id, kind: "ask_user_questions", payload: { questions: [] }, updatedAt: old() },
|
||||
]);
|
||||
await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: origin.id, kind: "execution", ownerType: "board", cause: "fixture", fingerprint: "fixture", nextAction: "Choose", updatedAt: old() });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: old(), windowEnd: new Date(), thresholdType: "hard", amountLimit: 100, amountObserved: 100, updatedAt: old() });
|
||||
await db.insert(agents).values({ companyId, name: "Error", role: "engineer", status: "error", updatedAt: old() });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true, includeDismissed: true, excludeSources: ["approval", "join_request"] });
|
||||
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
|
||||
const actual = [...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort();
|
||||
expect(new Set(feed.items.map(item => item.sourceKind)).size).toBe(8); expect(actual).toEqual(expected);
|
||||
expect(actual).toContain(`review:${review.id}`); expect(actual).toContain(`blocker_attention:${blocker.id}`);
|
||||
await start(); for (const item of feed.items) await archived(item.sourceKind, item.subject.id); await quiet();
|
||||
});
|
||||
|
||||
it("keeps unseen sources, cancels their deadline, and reactivates when unkept or revived", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: true });
|
||||
await start(); await quiet(); expect((await state("review", row.id))?.archivedAt).toBeNull();
|
||||
await decisionRetentionService(db).setKeep({ ...mutation("review", row.id), keep: false });
|
||||
await archived("review", row.id); await quiet();
|
||||
await decisionRetentionService(db).revive(mutation("review", row.id)); await archived("review", row.id); await quiet();
|
||||
});
|
||||
|
||||
it("moves deadlines on source activity and cancels when the source leaves attention", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board", updatedAt: new Date() });
|
||||
await start(); await settle(); const first = coordinator!.nextWakeAt()!;
|
||||
await change(tx => tx.update(issues).set({ updatedAt: new Date(Date.now() - delay + 600_000) }).where(eq(issues.id, row.id)));
|
||||
await settle(); expect(coordinator!.nextWakeAt()).toBeLessThan(first);
|
||||
await issueService(db).update(row.id, { status: "todo" }); await quiet(); expect(await state("review", row.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves outer transaction ownership and first writes during drain", async () => {
|
||||
await start(); await quiet(); draining = true;
|
||||
let id: string | undefined;
|
||||
await db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await notifyAttentionRetentionWork(nested, { companyId });
|
||||
id = (await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db)).id;
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(60_000); expect(await state("review", id!)).toBeNull();
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(await state("review", id!)).toBeNull();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", id!); await quiet();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => { await notifyAttentionRetentionWork(nested, { companyId }); await insertIssue("in_review", { assigneeUserId: "board" }, nested as unknown as Db); });
|
||||
throw new Error("rollback");
|
||||
})).rejects.toThrow("rollback");
|
||||
await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("isolates companies, catches up at activation, and bounds retained-history pages", async () => {
|
||||
const other = randomUUID(); await db.insert(companies).values({ id: other, name: "Paused", issuePrefix: "RP", status: "paused" });
|
||||
const row = await insertIssue("in_review", { companyId: other, assigneeUserId: "board" });
|
||||
await db.insert(issues).values(Array.from({ length: 1000 }, () => ({ companyId, title: randomUUID(), status: "done", updatedAt: old() })));
|
||||
const service = attentionRetentionService(db); const page = await service.load({ companyId });
|
||||
expect(page).toEqual({ rows: [], complete: true });
|
||||
await start(service); await quiet(); expect(await db.select().from(decisionRetention)).toHaveLength(0);
|
||||
await companyService(db).update(other, { status: "active" });
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(1); });
|
||||
expect((await db.select().from(decisionRetention))[0]).toMatchObject({ companyId: other, sourceId: row.id }); await quiet();
|
||||
});
|
||||
it("preserves terminal blocker source identity and its own activity deadline", async () => {
|
||||
const root = await insertIssue("blocked", { unblockDescriptor: { owner: "board", action: "Pick an owner" }, blockedTransitionAt: old() });
|
||||
const terminal = await insertIssue("backlog", { assigneeAgentId: agentId, updatedAt: new Date(old().getTime() + 60_000) });
|
||||
await db.insert(issueRelations).values({ companyId, issueId: terminal.id, relatedIssueId: root.id, type: "blocks" });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
const entries = await loadAll();
|
||||
const expected = feed.items.filter(item => item.sourceKind === "blocker_attention")
|
||||
.map(item => `${item.subject.id}:${new Date(item.activityAt).toISOString()}`).sort();
|
||||
expect(entries.filter(item => item.sourceKind === "blocker_attention")
|
||||
.map(item => `${item.sourceId}:${new Date(item.activityAt).toISOString()}`).sort()).toEqual(expected);
|
||||
expect(expected).toContain(`${terminal.id}:${terminal.updatedAt.toISOString()}`);
|
||||
await start(); await archived("blocker_attention", terminal.id); await archived("blocker_attention", root.id); await quiet();
|
||||
expect((await state("blocker_attention", terminal.id))?.sourceActivityAt).toEqual(terminal.updatedAt);
|
||||
expect((await state("blocker_attention", root.id))?.sourceActivityAt).toEqual(root.blockedTransitionAt);
|
||||
});
|
||||
|
||||
it("matches newest confirmations, independent gates, org invokability, and conversation visibility", async () => {
|
||||
const origin = await insertIssue();
|
||||
const conversation = await insertIssue("in_review", { conversationAgentId: agentId, conversationUserId: "board", conversationState: "waiting", assigneeAgentId: agentId });
|
||||
const hidden = await insertIssue("todo", { hiddenAt: old() });
|
||||
const [paused, manager, unhealthy] = await db.insert(agents).values([
|
||||
{ companyId, name: "Paused", role: "engineer", status: "paused" },
|
||||
{ companyId, name: "Terminated", role: "manager", status: "terminated" },
|
||||
{ companyId, name: "Unhealthy", role: "engineer", status: "active" },
|
||||
]).returning();
|
||||
await db.update(agents).set({ reportsTo: manager.id }).where(eq(agents.id, unhealthy.id));
|
||||
const common = { companyId, issueId: origin.id, kind: "request_confirmation", payload: { prompt: "Approve?" }, createdAt: old(), updatedAt: old() };
|
||||
const [older, newest, independent] = await db.insert(issueThreadInteractions).values([
|
||||
common, { ...common, createdAt: new Date(old().getTime() + 1000) },
|
||||
{ ...common, payload: { prompt: "Independent", qualityReview: { artifactKey: "copy" } } },
|
||||
{ ...common, createdAt: new Date(), addresseeUserId: "board" },
|
||||
{ ...common, addresseeAgentId: agentId },
|
||||
{ ...common, issueId: hidden.id },
|
||||
{ ...common, issueId: conversation.id, kind: "ask_user_questions", payload: { questions: [] } },
|
||||
{ ...common, issueId: conversation.id, addresseeAgentId: paused.id },
|
||||
{ ...common, issueId: conversation.id, addresseeAgentId: unhealthy.id, kind: "ask_user_questions", payload: { questions: [] } },
|
||||
] as typeof issueThreadInteractions.$inferInsert[]).returning();
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
const expected = [...new Set(feed.items.map(item => `${item.sourceKind}:${item.subject.id}`))].sort();
|
||||
expect([...new Set((await loadAll()).map(item => `${item.sourceKind}:${item.sourceId}`))].sort()).toEqual(expected);
|
||||
expect(expected).not.toContain(`issue_thread_interaction:${older.id}`);
|
||||
expect(expected).toContain(`issue_thread_interaction:${newest.id}`); expect(expected).toContain(`issue_thread_interaction:${independent.id}`);
|
||||
await start(); await archived("issue_thread_interaction", newest.id); await archived("issue_thread_interaction", independent.id); await quiet();
|
||||
await change(tx => tx.update(issueThreadInteractions).set({ status: "cancelled" }).where(eq(issueThreadInteractions.id, newest.id)));
|
||||
await archived("issue_thread_interaction", older.id); await quiet();
|
||||
});
|
||||
|
||||
it("excludes maintained reviews until a pending approval creates human attention", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeAgentId: agentId });
|
||||
const [interaction] = await db.insert(issueThreadInteractions).values({ companyId, issueId: row.id,
|
||||
kind: "request_confirmation", addresseeAgentId: agentId, payload: { prompt: "Agent verdict" } }).returning();
|
||||
expect((await loadAll()).some(entry => entry.sourceKind === "review")).toBe(false);
|
||||
const [approval] = await db.insert(approvals).values({ companyId, type: "hire_agent", payload: {}, status: "pending" }).returning();
|
||||
await db.insert(issueApprovals).values({ companyId, issueId: row.id, approvalId: approval.id });
|
||||
const feed = await attentionService(db).list(companyId, { all: true, allowUnscopedAll: true });
|
||||
expect(feed.items.some(item => item.sourceKind === "review" && item.subject.id === row.id)).toBe(true);
|
||||
await start(); await archived("review", row.id); await quiet();
|
||||
expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
|
||||
});
|
||||
|
||||
it("reactivates an exhausted failed run through the ordinary durable event producer", async () => {
|
||||
const origin = await insertIssue();
|
||||
const [run, newer] = await db.insert(heartbeatRuns).values([
|
||||
{ companyId, agentId, status: "failed", contextSnapshot: { issueId: origin.id }, createdAt: old(), finishedAt: old() },
|
||||
{ companyId, agentId, status: "succeeded", contextSnapshot: { issueId: origin.id }, createdAt: new Date(old().getTime() + 1000), finishedAt: old() },
|
||||
]).returning();
|
||||
await start(); await quiet();
|
||||
await appendHeartbeatRunEvent(db, { companyId, agentId, runId: run.id, eventType: "lifecycle", message: "Bounded retry exhausted after 3 attempts" });
|
||||
await quiet(); expect(await state("failed_run", run.id)).toBeNull();
|
||||
await change(tx => tx.delete(heartbeatRuns).where(eq(heartbeatRuns.id, newer.id)));
|
||||
await archived("failed_run", run.id); await quiet();
|
||||
});
|
||||
|
||||
it("cancels an agent error deadline through ordinary clearError", async () => {
|
||||
await db.update(agents).set({ status: "error", updatedAt: new Date() }).where(eq(agents.id, agentId));
|
||||
await start(); await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
await createAgentLifecycle(db).clearError(agentId); await quiet();
|
||||
expect(await state("agent_error_alert", agentId)).toBeNull();
|
||||
});
|
||||
|
||||
it("drains a paged backlog with source-scoped self notifications and leaves completed history quiet", async () => {
|
||||
await db.insert(issues).values(Array.from({ length: 225 }, () => ({ companyId, title: randomUUID(), status: "in_review", assigneeUserId: "board", updatedAt: old() })));
|
||||
const service = attentionRetentionService(db);
|
||||
const load = vi.spyOn(service, "load");
|
||||
await start(service);
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(await db.select().from(decisionRetention)).toHaveLength(225); }, { timeout: 30_000 });
|
||||
await quiet();
|
||||
const scopes = load.mock.calls.map(([scope]) => scope);
|
||||
expect(scopes.filter(scope => !scope)).toHaveLength(3);
|
||||
expect(scopes.filter(scope => scope && !scope.sourceId)).toHaveLength(0);
|
||||
expect(scopes.filter(scope => scope?.sourceId).length).toBeLessThanOrEqual(225);
|
||||
expect(new Set((await db.select().from(decisionRetention)).map(row => row.sourceId)).size).toBe(225);
|
||||
});
|
||||
|
||||
it("retries a failed atomic archive across drain and preserves notifications during a scan", async () => {
|
||||
const row = await insertIssue("in_review", { assigneeUserId: "board" });
|
||||
await db.execute(sql`create function attention_retention_fail() returns trigger language plpgsql as $$ begin raise exception 'retention injected'; end $$`);
|
||||
await db.execute(sql`create trigger attention_retention_fail before insert on decision_retention for each row execute function attention_retention_fail()`);
|
||||
await start(); expect(errors).toHaveLength(1); errors.length = 0;
|
||||
expect(await state("review", row.id)).toBeNull(); expect(await db.select().from(decisionArchiveNotificationOutbox)).toHaveLength(0);
|
||||
draining = true; await db.execute(sql`drop trigger attention_retention_fail on decision_retention`);
|
||||
statements.length = 0; await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(statements.length).toBeLessThanOrEqual(1);
|
||||
expect(statements.every(query => query.includes("pg_xact_status"))).toBe(true);
|
||||
statements.length = 0; await vi.advanceTimersByTimeAsync(300_000); expect(statements).toEqual([]);
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await archived("review", row.id); await quiet();
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
let release!: () => void; let scanned!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const entered = new Promise<void>(resolve => { scanned = resolve; });
|
||||
const service = attentionRetentionService(db), original = service.load;
|
||||
vi.spyOn(service, "load").mockImplementationOnce(async (...args) => { const result = await original(...args); scanned(); await gate; return result; });
|
||||
const starting = start(service); await entered;
|
||||
const next = await change(async tx => insertIssue("in_review", { assigneeUserId: "board" }, tx));
|
||||
release(); await starting; await archived("review", next.id); await quiet();
|
||||
});
|
||||
|
||||
it("registers and cancels a deadline through ordinary interaction creation and withdrawal", async () => {
|
||||
const origin = await insertIssue(); await start(); await quiet();
|
||||
const service = issueThreadInteractionService(db);
|
||||
const interaction = await service.create(origin, { kind: "request_confirmation", payload: { version: 1, prompt: "Approve the result?" } }, { userId: "board" });
|
||||
await settle(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(await state("issue_thread_interaction", interaction.id)).toBeNull();
|
||||
await service.withdrawInteraction(origin, interaction.id, { reason: "No longer needed" }, { userId: "board" }); await quiet();
|
||||
});
|
||||
|
||||
});
|
||||
@@ -62,7 +62,7 @@ describeEmbeddedPostgres("attention service", () => {
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-attention-service-");
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString);
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 30_000);
|
||||
|
||||
afterEach(async () => {
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, projects, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
vi.mock("../services/activity-log.js", () => ({ logActivity: vi.fn() }));
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("budget history for deleted scopes", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-budget-history-");
|
||||
db = createDb(database.connectionString);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
afterEach(async () => {
|
||||
await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents);
|
||||
await db.delete(projects); await db.delete(agents); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "History company", issuePrefix: `B${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Current agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 300 }).returning();
|
||||
await db.insert(costEvents).values({ companyId, agentId, provider: "openai", model: "history-fixture", inputTokens: 1, outputTokens: 1, costCents: 17, occurredAt: new Date() });
|
||||
return { companyId, agentId, policy: policy! };
|
||||
}
|
||||
|
||||
it.each(["agent", "project"] as const)("keeps current budgets and deleted %s policy/incident history readable without changing stored rows", async scopeType => {
|
||||
const { companyId, policy } = await fixture(), deletedId = randomUUID();
|
||||
if (scopeType === "agent") await db.insert(agents).values({ id: deletedId, companyId, name: "Former agent", role: "engineer" });
|
||||
else await db.insert(projects).values({ id: deletedId, companyId, name: "Former project" });
|
||||
const [historical] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: historical!.id, scopeType, scopeId: deletedId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 });
|
||||
if (scopeType === "agent") await db.delete(agents).where(eq(agents.id, deletedId));
|
||||
else await db.delete(projects).where(eq(projects.id, deletedId));
|
||||
const policiesBefore = await db.select().from(budgetPolicies), incidentsBefore = await db.select().from(budgetIncidents), costsBefore = await db.select().from(costEvents);
|
||||
|
||||
const overview = await budgetService(db).overview(companyId);
|
||||
expect(overview.policies).toHaveLength(2);
|
||||
expect(overview.policies.find(row => row.policyId === policy.id)).toMatchObject({ companyId, scopeName: "Current agent", amount: 300, observedAmount: 17 });
|
||||
expect(overview.policies.find(row => row.policyId === historical!.id)).toMatchObject({ companyId, scopeType, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, amount: 200 });
|
||||
expect(overview.activeIncidents).toMatchObject([{ companyId, policyId: historical!.id, scopeId: deletedId, scopeName: `Deleted ${scopeType}`, status: "open", amountObserved: 201 }]);
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(policiesBefore);
|
||||
expect(await db.select().from(budgetIncidents)).toEqual(incidentsBefore);
|
||||
expect(await db.select().from(costEvents)).toEqual(costsBefore);
|
||||
});
|
||||
it("rejects missing-scope budget grants but can dismiss their historical incidents", async () => {
|
||||
const { companyId, agentId } = await fixture(), missingId = randomUUID(), service = budgetService(db);
|
||||
await expect(service.evaluateAdmission(companyId, missingId)).rejects.toThrow("Agent not found");
|
||||
await expect(service.evaluateAdmission(companyId, agentId, { projectId: missingId })).rejects.toThrow("Project not found");
|
||||
for (const scopeType of ["agent", "project"] as const) {
|
||||
await expect(service.upsertPolicy(companyId, { scopeType, scopeId: missingId, amount: 100 }, "board")).rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType, scopeId: missingId,
|
||||
windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
const [approval] = await db.insert(approvals).values({ companyId, type: "budget_override_required", payload: {} }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, approvalId: approval!.id, scopeType, scopeId: missingId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 100, amountObserved: 101 }).returning();
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) };
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action: "raise_budget_and_resume", amount: 200 }, "board"))
|
||||
.rejects.toThrow(`${scopeType === "agent" ? "Agent" : "Project"} not found`);
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents), approvals: await db.select().from(approvals) }).toEqual(before);
|
||||
const dismissed = await service.resolveIncident(companyId, incident!.id, { action: "keep_paused" }, "board");
|
||||
expect(dismissed).toMatchObject({ companyId, policyId: policy!.id, scopeId: missingId, scopeName: `Deleted ${scopeType}`, status: "dismissed" });
|
||||
const { status: _status, resolvedAt: _resolvedAt, updatedAt: _updatedAt, ...preserved } = incident!;
|
||||
const [saved] = await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident!.id));
|
||||
expect(saved).toMatchObject({ ...preserved, status: "dismissed", resolvedAt: expect.any(Date) });
|
||||
expect((await db.select().from(approvals).where(eq(approvals.id, approval!.id)))[0]).toMatchObject({ status: "rejected" });
|
||||
expect(await db.select().from(budgetPolicies)).toEqual(before.policies);
|
||||
expect(await db.select().from(agents)).toEqual(before.agents);
|
||||
expect(await db.select().from(costEvents)).toEqual(before.costs);
|
||||
expect((await service.overview(companyId)).activeIncidents.some(row => row.id === incident!.id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a corrupted policy pointing to another company's scope instead of exposing its name", async () => {
|
||||
const { companyId } = await fixture(), foreignCompanyId = randomUUID(), foreignAgentId = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompanyId, name: "Other company", issuePrefix: `F${foreignCompanyId.replace(/-/g, "").slice(0, 6).toUpperCase()}` });
|
||||
await db.insert(agents).values({ id: foreignAgentId, companyId: foreignCompanyId, name: "Private other-company agent", role: "engineer" });
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: foreignAgentId, windowKind: "calendar_month_utc", amount: 200 }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy!.id, scopeType: "agent", scopeId: foreignAgentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date("2020-01-01"), windowEnd: new Date("2020-02-01"),
|
||||
thresholdType: "hard", amountLimit: 200, amountObserved: 201 }).returning();
|
||||
const service = budgetService(db);
|
||||
await expect(service.overview(companyId)).rejects.toThrow("Budget scope not found");
|
||||
for (const scopeId of [foreignAgentId, randomUUID()]) {
|
||||
await db.update(budgetIncidents).set({ scopeId }).where(eq(budgetIncidents.id, incident!.id));
|
||||
const before = { policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) };
|
||||
for (const action of ["raise_budget_and_resume", "keep_paused"] as const) {
|
||||
await expect(service.resolveIncident(companyId, incident!.id, { action, amount: 300 }, "board")).rejects.toThrow("Budget scope not found");
|
||||
expect({ policies: await db.select().from(budgetPolicies), incidents: await db.select().from(budgetIncidents),
|
||||
agents: await db.select().from(agents), costs: await db.select().from(costEvents) }).toEqual(before);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
@@ -16341,9 +16341,15 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
await tx.update(toolConnections).set({ credentialSecretRefs: connection.credentialSecretRefs.map(ref => ({ ...ref,
|
||||
versionSelector: ref.versionSelector === "1" ? "latest" : "1" })) }).where(eq(toolConnections.id, connection.id));
|
||||
});
|
||||
await expect.poll(async () => ((await f.checkpoint()).value as Record<string, unknown>).credentialFingerprint,
|
||||
{ timeout: 5000 }).not.toBe(oldWindow.credentialFingerprint);
|
||||
expect((await f.checkpoint()).value).toMatchObject({ generation: oldWindow.generation, outcome: "epoch_initialized" });
|
||||
// Credential synchronization and epoch initialization commit separately.
|
||||
// Require the final epoch receipt, not just the intermediate fingerprint.
|
||||
await expect.poll(async () => {
|
||||
const value = (await f.checkpoint()).value as Record<string, unknown>;
|
||||
return { ...value, fingerprintChanged: typeof value.credentialFingerprint === "string"
|
||||
&& value.credentialFingerprint !== oldWindow.credentialFingerprint };
|
||||
}, { timeout: 5000 }).toMatchObject({
|
||||
fingerprintChanged: true, generation: oldWindow.generation, outcome: "epoch_initialized",
|
||||
});
|
||||
expect(f.requests).toEqual([]);
|
||||
} finally { await coordinator.stop(); }
|
||||
});
|
||||
|
||||
@@ -4168,6 +4168,7 @@ describe("company portability", () => {
|
||||
const insertedRelationValues: Array<Record<string, unknown>> = [];
|
||||
const monitorUpdates: Array<Record<string, unknown>> = [];
|
||||
const db = {
|
||||
transaction: async (write: (tx: unknown) => Promise<unknown>) => write(db),
|
||||
insert: () => ({
|
||||
values: (rows: Array<Record<string, unknown>>) => ({
|
||||
onConflictDoNothing: async () => {
|
||||
@@ -4184,6 +4185,7 @@ describe("company portability", () => {
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any;
|
||||
db.transaction = async (write: (tx: typeof db) => Promise<unknown>) => write(db);
|
||||
return { db, insertedRelationValues, monitorUpdates };
|
||||
}
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
@@ -28,12 +29,13 @@ import {
|
||||
import { issueReadSqlCondition, projectReadSqlCondition } from "../services/authorization.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_COMPANY_SEARCH_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
console.warn(
|
||||
`Skipping embedded Postgres company search tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
|
||||
`Skipping embedded Postgres company search tests on this host: ${("reason" in embeddedPostgresSupport ? embeddedPostgresSupport.reason : "unsupported environment")}`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -80,8 +82,11 @@ describeEmbeddedPostgres("companySearchService", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-company-search-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
// External suites can reuse this dedicated database sequentially. Match
|
||||
// the fresh embedded fixture before this suite creates its own rows.
|
||||
if (externalDatabaseUrl) await db.execute(sql`TRUNCATE TABLE companies CASCADE`);
|
||||
svc = companySearchService(db);
|
||||
await db.execute(sql.raw("CREATE EXTENSION IF NOT EXISTS pg_trgm"));
|
||||
}, 20_000);
|
||||
@@ -100,6 +105,7 @@ describeEmbeddedPostgres("companySearchService", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
|
||||
const claim = (id: string, updatedAt = new Date(), owner = db) => writeDecisionChange(owner, async tx => tx.update(decisions).set({
|
||||
status: "decided", executionStatus: "running", chosenOptionId: "yes", decidedByUserId: userId,
|
||||
updatedAt, inputValues: {}, metadata: { continuationPending: true },
|
||||
}).where(eq(decisions.id, id)));
|
||||
}).where(eq(decisions.id, id)), { companyId });
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
@@ -467,7 +467,7 @@ const support = externalUrl ? { supported: true } : await getEmbeddedPostgresTes
|
||||
} else {
|
||||
await writeDecisionChange(db, async tx => tx.update(decisions).set({ status: "decided", executionStatus: "running",
|
||||
chosenOptionId: "archive", decidedByUserId: userId, updatedAt: new Date(Date.now() - 120_000),
|
||||
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)));
|
||||
metadata: { ...row.metadata, continuationPending: true } }).where(eq(decisions.id, row.id)), { companyId });
|
||||
}
|
||||
await start(); await settled(row.id, committed ? "failed" : "succeeded"); await quiet();
|
||||
const [retained] = await db.select().from(decisionRetention);
|
||||
|
||||
@@ -793,7 +793,7 @@ describePg("decisionService", () => {
|
||||
expect((await service().get(id))?.metadata.continuationPending).toBe(false);
|
||||
}
|
||||
const markReady = (id: string, source = db) => writeDecisionChange(source, async tx => tx.update(decisions)
|
||||
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)));
|
||||
.set({ status: "cancelled", metadata: { continuationPending: true } }).where(eq(decisions.id, id)), { companyId });
|
||||
|
||||
it("coalesces sequential tenant writes without dropping pending recovery", async () => {
|
||||
const row = await createCommentDecision();
|
||||
|
||||
@@ -0,0 +1,427 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import { accessService } from "../services/access.js";
|
||||
import { writeDeferredExecutionWait } from "../services/deferred-execution-wait.js";
|
||||
import { createQueuedCommentIssueLockWriter } from "../modules/wake-queue/adapters/queued-comment-postgres.js";
|
||||
import { createHeartbeatQueue } from "../services/heartbeat/queue.js";
|
||||
import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, agentWakeupRequests, authUsers, chatActions, chatEndpoints, toolApplications, toolConnections, companies, companyMemberships, createDb, reconcileDatabaseWork, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyDeferredExecutionEligibility, notifyDeferredExecutionWork, notifyNativeStatusWakeWork, notifyTaskRetrySettlement } from "../services/deferred-execution-notifications.js";
|
||||
import { registerDeferredReceiptWork } from "../services/heartbeat/deferred-receipt-work.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { requestTaskRetry } from "../services/task-retry.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
describe("deferred receipt durable dispatch", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let drain = false;
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
beforeAll(async () => {
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_DEFERRED_RECEIPT_TEST_DATABASE_URL");
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("deferred-receipts-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
});
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`truncate companies cascade`); drain = false; errors.length = 0;
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
|
||||
} });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), actorId = "deferred-receipt-user";
|
||||
await db.insert(authUsers).values({ id: actorId, name: "Receipt operator", email: `${actorId}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Deferred dispatch", issuePrefix: `DE${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: actorId, requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: actorId, principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Receipt process", adapterType: "process", status: "idle",
|
||||
adapterConfig: { command: process.execPath, args: ["-e", "console.log('receipt-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } } }).returning();
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: "Resume saved task", status: "todo", assigneeAgentId: agent.id }).returning();
|
||||
const [source] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, issueId: issue.id,
|
||||
status: "failed", finishedAt: new Date(), responsibleUserId: actorId, errorCode: "process_failed",
|
||||
contextSnapshot: { issueId: issue.id }, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }).returning();
|
||||
return { companyId, agent, issue, source, actorId };
|
||||
}
|
||||
async function quiet() {
|
||||
await vi.waitFor(async () => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
const reads = vi.spyOn(db, "select"), executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed);
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]); await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
}
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500); await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.status, "scheduled_retry"))).toHaveLength(0);
|
||||
});
|
||||
}
|
||||
it("is quiet after empty reconstruction and terminal receipt history", async () => {
|
||||
const f = await fixture();
|
||||
await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
|
||||
});
|
||||
it("dispatches an ordinary committed task Retry into saved process output", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const receipt = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(receipt.accepted).toBe(true);
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, receipt.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId!;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
await finish(f.issue.id);
|
||||
const linked = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.reason, "task_retry"), sql`${agentWakeupRequests.runId} is not null`));
|
||||
expect(linked).toHaveLength(2);
|
||||
expect(new Set(linked.map(row => row.runId))).toEqual(new Set([runId]));
|
||||
expect(linked.map(row => row.idempotencyKey)).toContain(`task-retry-dispatch:${receipt.requestId}`);
|
||||
expect(linked.find(row => row.id === receipt.requestId)?.status).toBe("coalesced");
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${receipt.requestId}`)).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it.each(["native claim", "execution wait"])("reconstructs an exact fresh %s deadline without probing early", async kind => {
|
||||
const f = await fixture(), updatedAt = new Date(), native = kind === "native claim";
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: native ? "claimed" : "deferred_issue_execution", claimedAt: updatedAt, updatedAt,
|
||||
payload: native ? {} : { executionWait: { reason: "recovery" } },
|
||||
requestedByActorType: "system", requestedByActorId: native ? "native-status-committer" : null }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const check = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await db.update(agentWakeupRequests).set({ status: "skipped" }).where(eq(agentWakeupRequests.id, scope.requestId!));
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: check, settleTaskChat: vi.fn(), dispatchNative: check,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const due = updatedAt.getTime() + (native ? 60_000 : 30_000);
|
||||
expect(coordinator.nextWakeAt()).toBe(due); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(check).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await vi.waitFor(() => expect(check).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id })));
|
||||
await quiet();
|
||||
});
|
||||
it("reconciles the first drained producer and waits for resume without domain SQL", async () => {
|
||||
const f = await fixture(); drain = true;
|
||||
const resume = vi.fn(async () => {}), dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await db.transaction(async tx => {
|
||||
await notifyDeferredExecutionWork(tx, { companyId: f.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id, source: "on_demand",
|
||||
status: "deferred_issue_execution", reason: "task_retry" });
|
||||
});
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(resume).not.toHaveBeenCalled(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
drain = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it.each([
|
||||
{ name: "saved comment", payload: { _paperclipWakeContext: { wakeCommentIds: ["saved-comment"] } } },
|
||||
{ name: "interaction", payload: { mutation: "interaction" } },
|
||||
{ name: "execution wait", payload: { executionWait: { reason: "recovery" } } },
|
||||
{ name: "interrupt", payload: { queuedCommentInterrupt: { actorId: "operator" } } },
|
||||
])("activates only the committed company's $name receipt after an empty start", async ({ payload }) => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [other] = await db.insert(agentWakeupRequests).values({ companyId: foreign.companyId, agentId: foreign.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload }).returning();
|
||||
let requestId: string;
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyDeferredExecutionWork(nested, { companyId: f.companyId });
|
||||
const [row] = await nested.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: new Date(Date.now() - 31_000) }).returning();
|
||||
requestId = row.id;
|
||||
});
|
||||
expect(resume).not.toHaveBeenCalled();
|
||||
});
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId })), { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, other.id)))[0].status).toBe("deferred_issue_execution");
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs a cancelled legacy run's exact deferred queue even without comment payload", async () => {
|
||||
const f = await fixture();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution" }).returning();
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: { queuedCommentInterruptQueueId: wake.id } })
|
||||
.where(eq(heartbeatRuns.id, f.source.id));
|
||||
const resume = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: vi.fn(), dispatchNative: vi.fn(),
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
expect(resume).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ companyId: f.companyId, requestId: wake.id }));
|
||||
await quiet();
|
||||
});
|
||||
it("settles a processed chat receipt without restaging it until an actual eligibility commit", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
const [application] = await db.insert(toolApplications).values({ companyId: f.companyId, name: "Receipt chat", type: "mcp_http" }).returning();
|
||||
const [connection] = await db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id,
|
||||
uid: randomUUID(), name: "Receipt chat", transport: "rest_api" }).returning();
|
||||
const [endpoint] = await db.insert(chatEndpoints).values({ companyId: f.companyId, connectionId: connection.id,
|
||||
provider: "slack", publicId: randomUUID(), assignedAgentId: f.agent.id }).returning();
|
||||
const intentId = randomUUID(), actionId = randomUUID(), previous = new Date(Date.now() - 60_000);
|
||||
await db.insert(agentWakeupRequests).values([
|
||||
{ id: intentId, companyId: f.companyId, agentId: f.agent.id, source: "on_demand", reason: "task_retry",
|
||||
status: "deferred_issue_execution", updatedAt: previous, payload: { issueId: f.issue.id,
|
||||
taskRetry: { schema: "paperclip.task-retry.v1", generationId: intentId, sourceRunId: f.source.id,
|
||||
actorId: f.actorId, strategy: "evaluate", chatActionId: actionId } } },
|
||||
{ id: actionId, companyId: f.companyId, agentId: f.agent.id, source: "automation", status: "deferred_issue_execution",
|
||||
payload: { issueId: f.issue.id, taskRetryRequestId: intentId } },
|
||||
]);
|
||||
await db.transaction(async tx => {
|
||||
await notifyTaskRetrySettlement(tx, { companyId: f.companyId, requestId: intentId });
|
||||
await tx.insert(chatActions).values({ id: actionId, companyId: f.companyId, endpointId: endpoint.id,
|
||||
providerActionId: randomUUID(), kind: "failed_run_retry", status: "processed", updatedAt: new Date(),
|
||||
payload: { issueId: f.issue.id, taskRetryIntentId: intentId, failedRunId: f.source.id } });
|
||||
});
|
||||
await vi.waitFor(async () => {
|
||||
const [intent] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, intentId));
|
||||
expect(intent.updatedAt.getTime()).toBeGreaterThan(previous.getTime());
|
||||
}, { timeout: 5_000 });
|
||||
expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status).toBe("processed");
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.waitFor(async () => expect((await db.select().from(chatActions).where(eq(chatActions.id, actionId)))[0].status)
|
||||
.toBe("issued"), { timeout: 5_000 });
|
||||
await quiet();
|
||||
});
|
||||
it("retains a moved interrupt grace deadline after an ordinary saved queue reorder", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const comments = await db.insert(issueComments).values(["first", "second"].map(body => ({
|
||||
companyId: f.companyId, issueId: f.issue.id, body, authorType: "user", authorUserId: f.actorId,
|
||||
}))).returning();
|
||||
const payload = { issueId: f.issue.id, queuedCommentInterrupt: { actorId: f.actorId },
|
||||
_paperclipWakeContext: { wakeCommentIds: comments.map(row => row.id) } };
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", payload, updatedAt: initialAt }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const deps = createHeartbeatQueueTestDependencies(db), queue = createHeartbeatQueue(db, deps);
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: scope => queue.resumeDeferredExecutionReceipts(scope),
|
||||
settleTaskChat: async () => {}, dispatchNative: async () => {}, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
const writer = createQueuedCommentIssueLockWriter(db, { syncCommentReferences: async () => {},
|
||||
deleteCommentReferenceSource: async () => {}, syncCommentExternalObjectsSafely: async () => {} });
|
||||
await writer.withLockedQueue({ issue: f.issue, queueId: wake.id,
|
||||
actor: { actorType: "user", actorId: f.actorId, agentId: null, runId: null, agentApiKeyId: null } },
|
||||
async (_locked, tx) => { await tx.updateWakeQueuedCommentIds({ wakeId: wake.id, payload,
|
||||
ids: comments.map(row => row.id).reverse(), updatedAt: movedAt }); });
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(deps.resumeQueuedCommentInterrupt).not.toHaveBeenCalled();
|
||||
expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(deps.resumeQueuedCommentInterrupt).toHaveBeenCalledExactlyOnceWith(f.companyId, wake.id));
|
||||
await quiet();
|
||||
});
|
||||
it("reactivates a held task Retry when the authorized subtree hold is released without wakeAgents", async () => {
|
||||
const f = await fixture(), tree = issueTreeControlService(db);
|
||||
const actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const hold = await tree.createHold(f.companyId, f.issue.id, { mode: "pause", actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
expect(request.requestId).toBeTruthy();
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.payload?.executionWait).toMatchObject({ reason: "issue_tree_hold_active" });
|
||||
expect(wake.runId).toBeNull();
|
||||
}, { timeout: 5_000 });
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 40_000 });
|
||||
await tree.releaseHold(f.companyId, f.issue.id, hold.hold.id, { actor, reason: "Resume saved Retry" });
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 40_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("keeps a moved native dispatch error deadline instead of retrying at its old time", async () => {
|
||||
const f = await fixture(), initialAt = new Date();
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "automation", status: "queued", error: "Transient dispatch failure", updatedAt: initialAt,
|
||||
requestedByActorType: "system", requestedByActorId: "native-status-committer" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: async () => {}, settleTaskChat: async () => {},
|
||||
dispatchNative, schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 10_000 - Date.now());
|
||||
const movedAt = new Date();
|
||||
await db.update(agentWakeupRequests).set({ updatedAt: movedAt }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
await vi.advanceTimersByTimeAsync(initialAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
expect(dispatchNative).not.toHaveBeenCalled(); expect(coordinator.nextWakeAt()).toBe(movedAt.getTime() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(movedAt.getTime() + 30_000 - Date.now());
|
||||
await vi.waitFor(() => expect(dispatchNative).toHaveBeenCalledTimes(1)); await quiet();
|
||||
});
|
||||
it("arms one new wait reason grace and stays quiet when that same gate is still closed", async () => {
|
||||
const f = await fixture(); let reason = "operator_pause";
|
||||
const resume = vi.fn(async (scope: { companyId?: string; requestId?: string }) => {
|
||||
await writeDeferredExecutionWait(db, { companyId: scope.companyId!, requestId: scope.requestId! }, { reason });
|
||||
});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume, settleTaskChat: async () => {}, dispatchNative: async () => {},
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution", reason: "task_retry" }).returning();
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
for (const nextReason of ["operator_pause", "budget_pause"]) {
|
||||
reason = nextReason;
|
||||
await writeDeferredExecutionWait(db, { companyId: f.companyId, requestId: wake.id }, { reason });
|
||||
const [current] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
const due = current.updatedAt.getTime() + 30_000;
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(due));
|
||||
const checked = resume.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now());
|
||||
await vi.waitFor(() => expect(resume).toHaveBeenCalledTimes(checked + 1));
|
||||
await quiet();
|
||||
}
|
||||
expect(resume).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it("reactivates a saved Retry through ordinary restore cancelled task", async () => {
|
||||
const f = await fixture(), actor = { actorType: "user" as const, actorId: f.actorId, userId: f.actorId };
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
const tree = issueTreeControlService(db);
|
||||
const cancel = await tree.createHold(f.companyId, f.issue.id, { mode: "cancel", actor });
|
||||
await tree.cancelIssueStatusesForHold(f.companyId, f.issue.id, cancel.hold.id);
|
||||
const restore = await tree.createHold(f.companyId, f.issue.id, { mode: "restore", actor });
|
||||
const commitEligibility = () => tree.restoreIssueStatusesForHold(f.companyId, f.issue.id, restore.hold.id, { actor });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
await vi.waitFor(() => { expect(coordinator.nextWakeAt()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); }, { timeout: 5_000 });
|
||||
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0].runId).toBeNull();
|
||||
await commitEligibility();
|
||||
let runId: string | null = null;
|
||||
await vi.waitFor(async () => {
|
||||
const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!));
|
||||
expect(wake.runId, JSON.stringify(wake)).toBeTruthy(); runId = wake.runId;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id); await quiet();
|
||||
});
|
||||
it("member archive publishes committed eligibility while preserving a cancelled Retry tombstone", async () => {
|
||||
const f = await fixture(), departing = `departing-${randomUUID()}`;
|
||||
const request = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
await db.insert(authUsers).values({ id: departing, name: "Departing member", email: `${departing}@example.test`,
|
||||
createdAt: new Date(), updatedAt: new Date() });
|
||||
const [member] = await db.insert(companyMemberships).values({ companyId: f.companyId, principalId: departing,
|
||||
principalType: "user", status: "active", membershipRole: "member" }).returning();
|
||||
await issueService(db).update(f.issue.id, { assigneeUserId: departing, assigneeAgentId: null });
|
||||
const readIntent = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, request.requestId!)))[0];
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
const deferred = vi.fn(), native = vi.fn();
|
||||
const unsubscribes = [subscribeDeliveryWork(db, DELIVERY_QUEUES.deferredExecutionReceipts, deferred),
|
||||
subscribeDeliveryWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents, native)];
|
||||
try {
|
||||
await expect(db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
throw new Error("roll back member archive");
|
||||
})).rejects.toThrow("roll back member archive");
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0].assigneeUserId).toBe(departing);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.deferredExecutionReceipts);
|
||||
await reconcileDatabaseWork(db, DELIVERY_QUEUES.nativeStatusWakeIntents);
|
||||
deferred.mockClear(); native.mockClear();
|
||||
await db.transaction(async tx => {
|
||||
await accessService(tx as unknown as typeof db).archiveMember(f.companyId, member.id,
|
||||
{ reassignment: { assigneeAgentId: f.agent.id } });
|
||||
expect(deferred).not.toHaveBeenCalled(); expect(native).not.toHaveBeenCalled();
|
||||
});
|
||||
expect(deferred).toHaveBeenCalledTimes(1); expect(native).toHaveBeenCalledTimes(1);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, f.issue.id)))[0]).toMatchObject({
|
||||
assigneeUserId: null, assigneeAgentId: f.agent.id,
|
||||
});
|
||||
expect((await db.select().from(companyMemberships).where(eq(companyMemberships.id, member.id)))[0].status).toBe("archived");
|
||||
expect(await readIntent()).toMatchObject({ status: "cancelled", runId: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${request.requestId}`)).toHaveLength(0);
|
||||
} finally { for (const unsubscribe of unsubscribes) unsubscribe(); }
|
||||
await quiet();
|
||||
});
|
||||
it("outer rollback and nested savepoint rollback do not dispatch receipt intent", async () => {
|
||||
const f = await fixture(); const dispatchNative = vi.fn(async () => {});
|
||||
await registerDeferredReceiptWork(db, coordinator, { resume: vi.fn(), settleTaskChat: vi.fn(), dispatchNative,
|
||||
schedulingSuppressed: async () => false }, () => !drain).ready;
|
||||
await expect(db.transaction(async tx => {
|
||||
await notifyNativeStatusWakeWork(tx, { companyId: f.companyId });
|
||||
throw new Error("outer rollback");
|
||||
})).rejects.toThrow("outer rollback");
|
||||
await db.transaction(async tx => {
|
||||
await expect(tx.transaction(async nested => {
|
||||
await notifyNativeStatusWakeWork(nested, { companyId: f.companyId }); throw new Error("nested rollback");
|
||||
})).rejects.toThrow("nested rollback");
|
||||
});
|
||||
await quiet(); expect(dispatchNative).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -383,11 +383,11 @@ describe.skipIf(!support.supported)("execution-control durable work", () => {
|
||||
const due = new Date(Date.now() + 120_000), earlier = new Date(Date.now() + 60_000);
|
||||
for (const at of [due, earlier]) {
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)));
|
||||
.set({ executionControlDeadlineAt: at }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await settle(); expect(c.nextWakeAt()).toBe(at.getTime());
|
||||
}
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx.update(heartbeatRuns)
|
||||
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)));
|
||||
.set({ executionControlDeadlineAt: null }).where(eq(heartbeatRuns.id, run.id)), { companyId });
|
||||
await quiet(c);
|
||||
});
|
||||
it("delivers a reconciled continuation when marking and resolving commit together", async () => {
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
import { createServer } from "node:http";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { activityLog, approvals, applyPendingMigrations, databaseWorkPending, budgetIncidents, budgetPolicies, companies, costEvents, createDb, externalObjects, externalObjectMentions, issues, issueComments, plugins } from "@paperclipai/db";
|
||||
import { externalObjectService, type ExternalObjectResolveResult } from "../services/external-objects.js";
|
||||
import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { pluginRegistryService } from "../services/plugin-registry.js";
|
||||
import { companyService } from "../services/companies.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const url = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECT_REFRESH_TEST_DATABASE_URL");
|
||||
const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const terminal: ExternalObjectResolveResult = { ok: true, snapshot: {
|
||||
statusKey: "done", statusCategory: "succeeded", statusTone: "success", isTerminal: true,
|
||||
} };
|
||||
const active: ExternalObjectResolveResult = { ok: true, snapshot: {
|
||||
statusKey: "open", statusCategory: "open", statusTone: "neutral", isTerminal: false, ttlSeconds: 60,
|
||||
} };
|
||||
|
||||
(support.supported ? describe : describe.skip)("external object refresh work (real PostgreSQL, simulated provider)", () => {
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let temp: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let admitted = true;
|
||||
let reconcile = true;
|
||||
let enabled = true;
|
||||
const provider = vi.fn(async (): Promise<ExternalObjectResolveResult> => terminal);
|
||||
const errors = vi.fn();
|
||||
const companyIds: string[] = [];
|
||||
beforeAll(async () => {
|
||||
if (url) await applyPendingMigrations(url);
|
||||
if (!url) temp = await startEmbeddedPostgresTestDatabase("external-refresh-work-");
|
||||
db = createDb(url ?? temp!.connectionString, { maxConnections: 6 });
|
||||
await db.execute(sql`truncate table ${companies} cascade`);
|
||||
}, 90_000);
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
for (const id of companyIds.splice(0)) {
|
||||
await db.delete(budgetIncidents).where(eq(budgetIncidents.companyId, id));
|
||||
await db.delete(budgetPolicies).where(eq(budgetPolicies.companyId, id));
|
||||
await db.delete(approvals).where(eq(approvals.companyId, id));
|
||||
await db.delete(costEvents).where(eq(costEvents.companyId, id));
|
||||
await db.delete(activityLog).where(eq(activityLog.companyId, id));
|
||||
await db.delete(issues).where(eq(issues.companyId, id));
|
||||
await db.delete(externalObjects).where(eq(externalObjects.companyId, id));
|
||||
await db.delete(companies).where(eq(companies.id, id));
|
||||
}
|
||||
await db.delete(plugins);
|
||||
admitted = true; reconcile = true; enabled = true; errors.mockClear();
|
||||
provider.mockReset().mockResolvedValue(terminal);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temp?.cleanup(); });
|
||||
|
||||
function service() {
|
||||
return externalObjectService(db, { github: false, enabled: () => enabled,
|
||||
resolvers: [{ providerKey: "url", resolve: provider }] });
|
||||
}
|
||||
async function start(svc = service()) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted,
|
||||
canReconcile: () => reconcile, onError: errors });
|
||||
const worker = svc.registerRefreshWork(coordinator, () => admitted);
|
||||
await worker.ready;
|
||||
return worker;
|
||||
}
|
||||
async function fixture(status = "active") {
|
||||
const id = randomUUID(); companyIds.push(id);
|
||||
await db.insert(companies).values({ id, name: "Refresh test", status,
|
||||
issuePrefix: `E${id.replaceAll("-", "").slice(0, 8)}` });
|
||||
const [issue] = await db.insert(issues).values({ companyId: id, title: "Reference",
|
||||
description: `https://example.com/${id}`, status: "todo" }).returning();
|
||||
return { companyId: id, issueId: issue!.id };
|
||||
}
|
||||
async function row(companyId: string) {
|
||||
return (await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId)))[0]!;
|
||||
}
|
||||
async function settle(predicate: () => Promise<boolean> | boolean) {
|
||||
for (let i = 0; i < 200; i++) {
|
||||
await vi.advanceTimersByTimeAsync(0); await delay(5);
|
||||
if (await predicate()) return;
|
||||
}
|
||||
throw new Error("refresh worker did not settle");
|
||||
}
|
||||
async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); }
|
||||
function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); }
|
||||
async function change(companyId: string, patch: Partial<typeof externalObjects.$inferInsert>) {
|
||||
await db.transaction(async tx => {
|
||||
await notifyExternalObjectRefresh(tx, companyId);
|
||||
await tx.update(externalObjects).set(patch).where(eq(externalObjects.companyId, companyId));
|
||||
});
|
||||
}
|
||||
|
||||
it("activates saved work when the ordinary feature and company settings enable it", async () => {
|
||||
clock(); const settings = instanceSettingsService(db);
|
||||
await settings.updateExperimental({ enableExternalObjects: true });
|
||||
const svc = externalObjectService(db, { github: false,
|
||||
enabled: async () => (await settings.getExperimental()).enableExternalObjects,
|
||||
resolvers: [{ providerKey: "url", resolve: provider }] });
|
||||
const f = await fixture("paused"); await svc.syncIssue(f.issueId);
|
||||
await settings.updateExperimental({ enableExternalObjects: false });
|
||||
await start(svc); await quiet();
|
||||
await companyService(db).update(f.companyId, { status: "active" }); await quiet();
|
||||
expect(provider).not.toHaveBeenCalled();
|
||||
await settings.updateExperimental({ enableExternalObjects: true });
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("settles empty, configured-idle and retained terminal history without recurring SQL or provider calls", async () => {
|
||||
clock();
|
||||
const svc = service(); await start(svc); await quiet();
|
||||
const selects = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(15 * 60_000);
|
||||
expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled();
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
await settle(async () => (await row(f.companyId))?.isTerminal === true); await quiet();
|
||||
expect((await row(f.companyId)).statusKey).toBe("done");
|
||||
selects.mockClear(); provider.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(15 * 60_000);
|
||||
expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled();
|
||||
expect(errors).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("registers exact future deadlines, preempts them on earlier writes and cancels terminal work", async () => {
|
||||
clock(); provider.mockResolvedValue(active);
|
||||
const svc = service(); await start(svc);
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
await settle(() => provider.mock.calls.length === 1);
|
||||
await settle(async () => (await row(f.companyId)).lastResolvedAt !== null);
|
||||
const saved = await row(f.companyId);
|
||||
await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime());
|
||||
const earlier = new Date(Date.now() + 1000); await change(f.companyId, { nextRefreshAt: earlier });
|
||||
await settle(() => coordinator!.nextWakeAt() === earlier.getTime());
|
||||
await vi.advanceTimersByTimeAsync(999); expect(provider).toHaveBeenCalledTimes(1);
|
||||
provider.mockResolvedValue(terminal); await vi.advanceTimersByTimeAsync(1);
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(provider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps disabled and paused company work passive and reconstructs on eligibility mutation", async () => {
|
||||
clock(); enabled = false;
|
||||
const svc = service(); await start(svc); await quiet();
|
||||
const f = await fixture("paused");
|
||||
enabled = true; await svc.syncIssue(f.issueId); await quiet();
|
||||
await vi.advanceTimersByTimeAsync(600_000); expect(provider).not.toHaveBeenCalled();
|
||||
await db.transaction(async tx => {
|
||||
await notifyExternalObjectRefresh(tx, f.companyId);
|
||||
await tx.update(companies).set({ status: "active" }).where(eq(companies.id, f.companyId));
|
||||
});
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("waits for the outer commit and leaves rolled-back ingestion quiet", async () => {
|
||||
clock(); const svc = service(); await start(svc); await quiet();
|
||||
const f = await fixture();
|
||||
await expect(db.transaction(async tx => {
|
||||
await svc.syncIssue(f.issueId, tx); throw new Error("rollback");
|
||||
})).rejects.toThrow("rollback");
|
||||
await quiet(); expect(await row(f.companyId)).toBeUndefined(); expect(provider).not.toHaveBeenCalled();
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
let nestedDone = false;
|
||||
const commit = db.transaction(async tx => {
|
||||
await tx.transaction(async nested => { await svc.syncIssue(f.issueId, nested); });
|
||||
nestedDone = true; await held;
|
||||
});
|
||||
await settle(() => nestedDone); await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(provider).not.toHaveBeenCalled(); release(); await commit;
|
||||
await settle(async () => (await row(f.companyId))?.isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retains first work during drain and recovers a provider failure after restart", async () => {
|
||||
clock(); const svc = service(); await start(svc); await quiet();
|
||||
admitted = false; const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
const selects = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(provider).not.toHaveBeenCalled();
|
||||
selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000); expect(selects).not.toHaveBeenCalled();
|
||||
admitted = true; provider.mockRejectedValueOnce(new Error("simulated transport failure"));
|
||||
await vi.advanceTimersByTimeAsync(30_000); await settle(() => errors.mock.calls.length === 1);
|
||||
const claimed = await row(f.companyId); expect(claimed.refreshToken).not.toBeNull();
|
||||
await coordinator!.stop(); coordinator = undefined; await start(svc);
|
||||
const due = claimed.refreshStartedAt!.getTime() + 300_000;
|
||||
expect(coordinator!.nextWakeAt()).toBe(due);
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("leaves unsupported URLs passive and cancels orphan work until ordinary reattachment", async () => {
|
||||
clock(); const unsupported = externalObjectService(db, { github: false });
|
||||
await start(unsupported); const f = await fixture(); await unsupported.syncIssue(f.issueId); await quiet();
|
||||
const selects = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled();
|
||||
expect((await row(f.companyId)).lastResolvedAt).toBeNull();
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
provider.mockResolvedValue(active); const svc = service(); await start(svc);
|
||||
await settle(async () => (await row(f.companyId)).lastResolvedAt !== null);
|
||||
await db.update(issues).set({ description: null }).where(eq(issues.id, f.issueId));
|
||||
await svc.syncIssue(f.issueId); await quiet();
|
||||
selects.mockClear(); provider.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(900_000);
|
||||
expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled();
|
||||
provider.mockResolvedValue(terminal);
|
||||
await db.update(issues).set({ description: `https://example.com/${f.companyId}` }).where(eq(issues.id, f.issueId));
|
||||
await svc.syncIssue(f.issueId);
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("activates a newly ready plugin and fences a result after company permission is withdrawn", async () => {
|
||||
clock(); const registry = pluginRegistryService(db);
|
||||
const manifest: PaperclipPluginManifestV1 = {
|
||||
id: `paperclip.refresh-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Refresh fixture",
|
||||
description: "Simulated external object provider", author: "Paperclip", categories: ["connector"],
|
||||
capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" },
|
||||
objectReferences: [{ providerKey: "tracker", displayName: "Tracker", objectTypes: ["ticket"], urlPatterns: ["https://example.com/:id"] }],
|
||||
};
|
||||
const plugin = await registry.install({ packageName: "refresh-fixture" }, manifest);
|
||||
let finish!: (result: ExternalObjectResolveResult) => void;
|
||||
const calls = vi.fn(async (_id: string, method: string) => {
|
||||
if (method === "detectExternalObjects") return { detections: [] };
|
||||
if (calls.mock.calls.filter(call => call[1] === "resolveExternalObject").length === 1)
|
||||
return new Promise<ExternalObjectResolveResult>(resolve => { finish = resolve; });
|
||||
return terminal;
|
||||
});
|
||||
const svc = externalObjectService(db, { github: false,
|
||||
pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager,
|
||||
detectors: [{ key: "fixture", detect: ({ urls }) => urls.map(canonical => ({ canonical,
|
||||
detectorKey: "fixture", providerKey: "tracker", objectType: "ticket", externalId: canonical.canonicalIdentityHash })) }],
|
||||
});
|
||||
await start(svc); const f = await fixture(); await svc.syncIssue(f.issueId); await quiet();
|
||||
const selects = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled();
|
||||
await registry.updateStatus(plugin!.id, { status: "ready" });
|
||||
await settle(() => typeof finish === "function");
|
||||
await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} });
|
||||
finish(terminal);
|
||||
await settle(async () => (await row(f.companyId)).refreshToken === null); await quiet();
|
||||
expect((await row(f.companyId)).isTerminal).toBe(false);
|
||||
selects.mockClear(); calls.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(900_000); expect(selects).not.toHaveBeenCalled(); expect(calls).not.toHaveBeenCalled();
|
||||
calls.mockImplementation(async () => terminal);
|
||||
await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: true, settingsJson: {} });
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(calls).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["company setting", "status", "uninstall", "manifest", "other company"] as const)(
|
||||
"revalidates the selected plugin after %s changes even while the built-in fallback remains available", async change => {
|
||||
clock(); const registry = pluginRegistryService(db);
|
||||
const manifest: PaperclipPluginManifestV1 = {
|
||||
id: `paperclip.selected-${randomUUID()}`, apiVersion: 1, version: "1.0.0", displayName: "Selected provider",
|
||||
description: "Selected resolver fixture", author: "Paperclip", categories: ["connector"],
|
||||
capabilities: ["external.objects.detect", "external.objects.read"], entrypoints: { worker: "worker.js" },
|
||||
objectReferences: [{ providerKey: "github", displayName: "GitHub", objectTypes: ["pull_request"], urlPatterns: ["https://github.com/:owner/:repo/pull/:id"] }],
|
||||
};
|
||||
const plugin = await registry.install({ packageName: "selected-provider-fixture" }, manifest);
|
||||
await registry.updateStatus(plugin!.id, { status: "ready" });
|
||||
let finishPlugin!: (result: ExternalObjectResolveResult) => void;
|
||||
const pluginResult = new Promise<ExternalObjectResolveResult>(resolve => { finishPlugin = resolve; });
|
||||
let selected = false;
|
||||
const calls = vi.fn(async (_id: string, method: string) => {
|
||||
if (method === "detectExternalObjects") return { detections: [] };
|
||||
selected = true; return pluginResult;
|
||||
});
|
||||
let finishFallback!: () => void;
|
||||
const fallbackResult = new Promise<void>(resolve => { finishFallback = resolve; });
|
||||
const fallback = vi.fn(async () => {
|
||||
await fallbackResult;
|
||||
return new Response(JSON.stringify({ state: "closed", merged: true, title: "Allowed fallback" }), { status: 200 });
|
||||
});
|
||||
const svc = externalObjectService(db, { pluginWorkerManager: { call: calls } as unknown as PluginWorkerManager,
|
||||
github: { fetch: fallback, credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }) } });
|
||||
try {
|
||||
await start(svc); const f = await fixture();
|
||||
await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId));
|
||||
await svc.syncIssue(f.issueId); await settle(() => selected);
|
||||
if (change === "company setting") await registry.upsertCompanySettings(plugin!.id, f.companyId, { enabled: false, settingsJson: {} });
|
||||
else if (change === "status") await registry.updateStatus(plugin!.id, { status: "error" });
|
||||
else if (change === "uninstall") await registry.uninstall(plugin!.id, true);
|
||||
else if (change === "manifest") await registry.update(plugin!.id, { manifest: { ...manifest, capabilities: [], objectReferences: [] } });
|
||||
else {
|
||||
const other = await fixture();
|
||||
await registry.upsertCompanySettings(plugin!.id, other.companyId, { enabled: false, settingsJson: {} });
|
||||
}
|
||||
finishPlugin({ ok: true, snapshot: { statusKey: "plugin_result", statusCategory: "succeeded", statusTone: "success", isTerminal: true } });
|
||||
if (change === "other company") {
|
||||
await settle(async () => (await row(f.companyId))?.statusKey === "plugin_result"); await quiet();
|
||||
expect(fallback).not.toHaveBeenCalled();
|
||||
} else {
|
||||
await settle(async () => fallback.mock.calls.length > 0 || Boolean((await row(f.companyId))?.lastResolvedAt));
|
||||
const beforeFallback = await row(f.companyId);
|
||||
expect(beforeFallback.isTerminal).toBe(false); expect(beforeFallback.statusKey).toBeNull(); expect(beforeFallback.lastResolvedAt).toBeNull();
|
||||
expect(fallback).toHaveBeenCalledOnce();
|
||||
finishFallback();
|
||||
await settle(async () => (await row(f.companyId))?.statusKey === "merged"); await quiet();
|
||||
expect((await row(f.companyId)).displayTitle).toBe("acme/app#42: Allowed fallback");
|
||||
}
|
||||
} finally { finishPlugin(terminal); finishFallback(); }
|
||||
});
|
||||
|
||||
it("reactivates refresh after ordinary budget pause and budget raise", async () => {
|
||||
clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc);
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
await settle(async () => (await row(f.companyId))?.lastResolvedAt !== null);
|
||||
await db.insert(costEvents).values({ companyId: f.companyId, usageKind: "decision", provider: "fixture",
|
||||
biller: "fixture", billingType: "metered_api", model: "simulated", costCents: 200, occurredAt: new Date() });
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId,
|
||||
amount: 100, hardStopEnabled: true }, null);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0]!.status).toBe("paused");
|
||||
await quiet();
|
||||
const reads = vi.spyOn(db, "select"); provider.mockClear();
|
||||
await vi.advanceTimersByTimeAsync(900_000); expect(reads).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled();
|
||||
provider.mockResolvedValue(terminal);
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 300 }, null);
|
||||
await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["comment", "document"] as const)("keeps hard-deleted %s activation scoped to its retained mention owner", async kind => {
|
||||
clock(); const svc = service(); provider.mockResolvedValue(active); await start(svc);
|
||||
const a = await fixture(); const b = await fixture(); await svc.syncIssue(b.issueId);
|
||||
let id: string;
|
||||
if (kind === "comment") {
|
||||
const [comment] = await db.insert(issueComments).values({ companyId: a.companyId, issueId: a.issueId,
|
||||
body: `https://example.com/${a.companyId}` }).returning();
|
||||
id = comment!.id; await svc.syncComment(id);
|
||||
} else {
|
||||
const result = await documentService(db).upsertIssueDocument({ issueId: a.issueId,
|
||||
key: "notes", format: "markdown", body: `https://example.com/${a.companyId}` });
|
||||
id = result.document.id; await svc.syncDocument(id);
|
||||
}
|
||||
await settle(async () => {
|
||||
const left = await row(a.companyId), right = await row(b.companyId);
|
||||
return Boolean(left?.lastResolvedAt && right?.lastResolvedAt);
|
||||
});
|
||||
const expectedDeadline = Math.min((await row(a.companyId)).nextRefreshAt!.getTime(), (await row(b.companyId)).nextRefreshAt!.getTime());
|
||||
await settle(() => coordinator!.nextWakeAt() === expectedDeadline && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh));
|
||||
const reads: Array<{ query: string; params: unknown[] }> = [];
|
||||
const debug = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, query, params) => {
|
||||
if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"'))
|
||||
reads.push({ query, params: [...params] });
|
||||
};
|
||||
try {
|
||||
if (kind === "comment") { await db.delete(issueComments).where(eq(issueComments.id, id)); await svc.syncComment(id); }
|
||||
else { await documentService(db).deleteIssueDocument(a.issueId, "notes"); await svc.syncDocument(id); }
|
||||
await settle(() => reads.length > 0);
|
||||
for (const read of reads) {
|
||||
expect(read.query).toContain('"external_objects"."company_id" =');
|
||||
expect(read.params).toContain(a.companyId); expect(read.params).not.toContain(b.companyId);
|
||||
}
|
||||
} finally { db.$client.options.debug = debug; }
|
||||
expect(provider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("keeps warm standby startup SQL-free and resumes a first committed producer", async () => {
|
||||
clock(); admitted = false; reconcile = false;
|
||||
const svc = service(); const selects = vi.spyOn(db, "select");
|
||||
await start(svc); await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(selects).not.toHaveBeenCalled();
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
selects.mockClear(); await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(selects).not.toHaveBeenCalled(); expect(provider).not.toHaveBeenCalled();
|
||||
admitted = true; reconcile = true;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
await settle(async () => (await row(f.companyId))?.isTerminal); await quiet();
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("renews an active provider lease so another service cannot duplicate its action", async () => {
|
||||
clock(); const svc = service(); await start(svc);
|
||||
let finish!: (result: ExternalObjectResolveResult) => void;
|
||||
provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
await settle(() => provider.mock.calls.length === 1);
|
||||
const claimed = await row(f.companyId);
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
await settle(async () => (await row(f.companyId)).refreshStartedAt!.getTime() > claimed.refreshStartedAt!.getTime());
|
||||
const competing = await service().refreshObject(claimed.id, { companyId: f.companyId,
|
||||
force: true, now: new Date(claimed.refreshStartedAt!.getTime() + 301_000) });
|
||||
expect(competing.reason).toBe("refresh_in_progress"); expect(provider).toHaveBeenCalledOnce();
|
||||
finish(terminal); await settle(async () => (await row(f.companyId)).isTerminal); await quiet();
|
||||
});
|
||||
|
||||
it("uses durable provider retry deadlines and cancels the observed result on shutdown without overlap", async () => {
|
||||
clock(); const svc = service(); await start(svc);
|
||||
provider.mockResolvedValueOnce({ ok: false, liveness: "rate_limited", errorCode: "slow_down", retryAfterSeconds: 5 });
|
||||
const f = await fixture(); await svc.syncIssue(f.issueId);
|
||||
await settle(async () => (await row(f.companyId))?.lastErrorCode === "slow_down");
|
||||
const saved = await row(f.companyId);
|
||||
await settle(() => coordinator!.nextWakeAt() === saved.nextRefreshAt!.getTime());
|
||||
expect(saved.refreshToken).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(saved.nextRefreshAt!.getTime() - Date.now() - 1);
|
||||
expect(provider).toHaveBeenCalledOnce();
|
||||
let finish!: (result: ExternalObjectResolveResult) => void;
|
||||
provider.mockImplementationOnce(() => new Promise(resolve => { finish = resolve; }));
|
||||
await vi.advanceTimersByTimeAsync(1); await settle(() => provider.mock.calls.length === 2);
|
||||
await svc.syncIssue(f.issueId); await vi.advanceTimersByTimeAsync(0);
|
||||
expect(provider).toHaveBeenCalledTimes(2);
|
||||
let stopped = false; const stop = coordinator!.stop().then(() => { stopped = true; });
|
||||
await stop; expect(stopped).toBe(true); coordinator = undefined;
|
||||
const cancelled = await row(f.companyId); expect(cancelled.refreshToken).toBeNull(); expect(cancelled.isTerminal).toBe(false);
|
||||
finish(terminal); await delay(10);
|
||||
expect((await row(f.companyId)).isTerminal).toBe(false);
|
||||
const selects = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(600_000);
|
||||
expect(selects).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("cancels an actual stalled GitHub HTTP body, retains due debt and resumes after restart", async () => {
|
||||
clock(); let stall = true, started = false, closed = false;
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url?.includes("/pulls/")) {
|
||||
if (stall) {
|
||||
started = true; response.on("close", () => { closed = true; });
|
||||
response.writeHead(200, { "content-type": "application/json" }); response.write('{"state":');
|
||||
} else response.end(JSON.stringify({ state: "closed", merged: true, title: "Done" }));
|
||||
} else response.end(JSON.stringify({ private: false }));
|
||||
});
|
||||
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Missing HTTP test port");
|
||||
const svc = externalObjectService(db, { github: {
|
||||
fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init),
|
||||
credentialProvider: () => ({ token: "simulated-company-token", source: "company_secret" }),
|
||||
} });
|
||||
try {
|
||||
await start(svc); const f = await fixture();
|
||||
await db.update(issues).set({ description: "https://github.com/acme/app/pull/42" }).where(eq(issues.id, f.issueId));
|
||||
await svc.syncIssue(f.issueId); await settle(() => started);
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
await settle(() => closed);
|
||||
const cancelled = await row(f.companyId);
|
||||
expect(cancelled.refreshToken).toBeNull(); expect(cancelled.lastResolvedAt).toBeNull();
|
||||
expect(cancelled.nextRefreshAt!.getTime()).toBeLessThanOrEqual(Date.now());
|
||||
stall = false; await start(svc);
|
||||
await settle(async () => Boolean((await row(f.companyId))?.isTerminal)); await quiet();
|
||||
} finally { await coordinator?.stop(); coordinator = undefined; server.closeAllConnections(); await new Promise<void>(resolve => server.close(() => resolve())); }
|
||||
});
|
||||
|
||||
it("loads only the changed identity and objects after ingestion into ten thousand future objects", async () => {
|
||||
clock(); const svc = service(); const f = await fixture();
|
||||
const future = new Date(Date.now() + 86_400_000);
|
||||
const ids = Array.from({ length: 10_000 }, () => randomUUID());
|
||||
for (let offset = 0; offset < ids.length; offset += 500) {
|
||||
const batch = ids.slice(offset, offset + 500);
|
||||
await db.insert(externalObjects).values(batch.map(id => ({ id, companyId: f.companyId,
|
||||
providerKey: "url", objectType: "link", externalId: id, nextRefreshAt: future })));
|
||||
await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId,
|
||||
sourceIssueId: f.issueId, sourceKind: "description" as const, objectId })));
|
||||
}
|
||||
await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime());
|
||||
const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming",
|
||||
description: "https://example.com/new-object" }).returning();
|
||||
const reads: Array<{ query: string; params: unknown[] }> = [];
|
||||
const debug = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, query, params) => {
|
||||
if (/^select/i.test(query) && query.includes('from "external_objects"') && query.includes('inner join "companies"'))
|
||||
reads.push({ query, params: [...params] });
|
||||
};
|
||||
try {
|
||||
await svc.syncIssue(incoming!.id);
|
||||
await settle(() => provider.mock.calls.length === 1);
|
||||
await settle(() => coordinator!.nextWakeAt() === future.getTime() && !databaseWorkPending(db, DELIVERY_QUEUES.externalObjectRefresh));
|
||||
expect(reads.length).toBeGreaterThan(0); expect(reads.length).toBeLessThanOrEqual(6);
|
||||
for (const read of reads) expect(read.query).toMatch(/"external_objects"\."(?:id|external_id)" =/);
|
||||
} finally { db.$client.options.debug = debug; }
|
||||
});
|
||||
|
||||
it("reconstructs paginated multi-company work and never invokes inactive or never-refresh objects", async () => {
|
||||
clock(); const svc = service(); const a = await fixture(); const b = await fixture(); const inactive = await fixture("paused");
|
||||
for (const companyId of [b.companyId, inactive.companyId]) {
|
||||
await db.insert(externalObjects).values(Array.from({ length: 255 }, (_, index) => ({
|
||||
companyId, providerKey: "url", objectType: "link", externalId: String(index),
|
||||
sanitizedCanonicalUrl: `https://example.com/${companyId}/${index}`, canonicalIdentityHash: `${companyId}-${index}`,
|
||||
nextRefreshAt: null,
|
||||
})));
|
||||
}
|
||||
await db.update(issues).set({ description: Array.from({ length: 255 }, (_, index) =>
|
||||
`https://example.com/${a.companyId}/${index}`).join(" ") }).where(eq(issues.id, a.issueId));
|
||||
await svc.syncIssue(a.issueId);
|
||||
const future = new Date(Date.now() + 60_000);
|
||||
await db.update(externalObjects).set({ nextRefreshAt: future }).where(eq(externalObjects.companyId, a.companyId));
|
||||
await svc.syncIssue(b.issueId);
|
||||
await start(svc); await settle(() => provider.mock.calls.length === 1);
|
||||
await settle(() => coordinator!.nextWakeAt() === future.getTime());
|
||||
expect((await row(a.companyId)).nextRefreshAt).toEqual(future);
|
||||
const companiesSeen = provider.mock.calls.map(call => (call as unknown as [{ companyId: string }])[0].companyId);
|
||||
expect(companiesSeen).toEqual([b.companyId]);
|
||||
await change(a.companyId, { isTerminal: true }); await quiet();
|
||||
await vi.advanceTimersByTimeAsync(600_000); expect(provider).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,11 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import {
|
||||
companies,
|
||||
applyPendingMigrations,
|
||||
createDb,
|
||||
activityLog,
|
||||
externalObjectMentions,
|
||||
@@ -11,6 +14,7 @@ import {
|
||||
issues,
|
||||
plugins,
|
||||
} from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
@@ -26,7 +30,8 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/shared";
|
||||
import type { PluginWorkerManager } from "../services/plugin-worker-manager.js";
|
||||
import { createGitHubExternalObjectProvider } from "../services/github-external-object-provider.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_EXTERNAL_OBJECTS_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -709,9 +714,10 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
}, 20_000);
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-external-objects-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 90_000);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(activityLog);
|
||||
@@ -725,9 +731,22 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
async function runRefreshWork(service: ReturnType<typeof externalObjectService>) {
|
||||
const errors: unknown[] = [];
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
try {
|
||||
await service.registerRefreshWork(coordinator, () => true).ready;
|
||||
expect(errors).toEqual([]);
|
||||
} finally {
|
||||
await coordinator.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function createIssue(companyId = randomUUID()) {
|
||||
const issueId = randomUUID();
|
||||
await db.insert(companies).values({
|
||||
@@ -877,9 +896,9 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
|
||||
expect(object.nextRefreshAt).toBeInstanceOf(Date);
|
||||
|
||||
const refreshed = await svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000));
|
||||
await runRefreshWork(svc);
|
||||
|
||||
expect(refreshed).toHaveLength(1);
|
||||
expect((await db.select().from(externalObjects))[0]?.lastResolvedAt).toBeInstanceOf(Date);
|
||||
expect(resolve).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -916,15 +935,13 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
await svc.syncIssue(issueId);
|
||||
const object = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
|
||||
const dueRefresh = svc.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000));
|
||||
const dueRefresh = runRefreshWork(svc);
|
||||
await resolverStarted;
|
||||
const manualRefresh = svc.refreshObject(object.id, { companyId, force: true });
|
||||
finishResolver();
|
||||
|
||||
const [dueResults, manualResult] = await Promise.all([dueRefresh, manualRefresh]);
|
||||
const [, manualResult] = await Promise.all([dueRefresh, manualRefresh]);
|
||||
|
||||
expect(dueResults).toHaveLength(1);
|
||||
expect(dueResults[0]?.refreshed).toBe(true);
|
||||
expect(manualResult.refreshed).toBe(true);
|
||||
expect(manualResult.object.statusLabel).toBe("Open");
|
||||
expect(resolve).toHaveBeenCalledTimes(1);
|
||||
@@ -964,14 +981,12 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
await scheduledService.syncIssue(issueId);
|
||||
const object = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
|
||||
const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, new Date(Date.now() + 1_000));
|
||||
const dueRefresh = runRefreshWork(scheduledService);
|
||||
await resolverStarted;
|
||||
const manualRefresh = await manualService.refreshObject(object.id, { companyId, force: true });
|
||||
finishResolver();
|
||||
const dueResults = await dueRefresh;
|
||||
await dueRefresh;
|
||||
|
||||
expect(dueResults).toHaveLength(1);
|
||||
expect(dueResults[0]?.refreshed).toBe(true);
|
||||
expect(manualRefresh.refreshed).toBe(false);
|
||||
expect(manualRefresh.reason).toBe("refresh_in_progress");
|
||||
expect(manualRefresh.object).not.toHaveProperty("refreshToken");
|
||||
@@ -1027,7 +1042,7 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
const object = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
const claimedAt = new Date(Date.now() + 1_000);
|
||||
|
||||
const dueRefresh = scheduledService.refreshDueObjects(companyId, 50, claimedAt);
|
||||
const dueRefresh = runRefreshWork(scheduledService);
|
||||
await resolverStarted;
|
||||
const replacementResult = await replacementService.refreshObject(object.id, {
|
||||
companyId,
|
||||
@@ -1035,11 +1050,10 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
now: new Date(claimedAt.getTime() + 301_000),
|
||||
});
|
||||
finishResolver();
|
||||
const dueResults = await dueRefresh;
|
||||
await dueRefresh;
|
||||
const finalObject = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
|
||||
expect(replacementResult.refreshed).toBe(true);
|
||||
expect(dueResults[0]).toMatchObject({ refreshed: false, reason: "refresh_superseded" });
|
||||
expect(finalObject.statusLabel).toBe("Closed");
|
||||
expect(finalObject.refreshStartedAt).toBeNull();
|
||||
expect(finalObject.refreshToken).toBeNull();
|
||||
@@ -1073,9 +1087,7 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
await svc.syncIssue(active.issueId);
|
||||
await svc.syncIssue(paused.issueId);
|
||||
|
||||
const result = await svc.refreshDueObjectsForActiveCompanies(50, new Date(Date.now() + 1_000));
|
||||
|
||||
expect(result).toEqual({ companies: 1, checked: 1, refreshed: 1 });
|
||||
await runRefreshWork(svc);
|
||||
expect(resolve).toHaveBeenCalledTimes(1);
|
||||
const rows = await db.select().from(externalObjects);
|
||||
const activeObject = rows.find((row) => row.companyId === active.companyId);
|
||||
@@ -1134,9 +1146,8 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
.set({ nextRefreshAt: new Date(0) })
|
||||
.where(eq(externalObjects.id, object.id));
|
||||
|
||||
const refreshed = await svc.refreshDueObjects(companyId);
|
||||
await runRefreshWork(svc);
|
||||
|
||||
expect(refreshed).toEqual([]);
|
||||
expect(resolve).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
@@ -1179,7 +1190,7 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
});
|
||||
|
||||
it("re-checks previously frozen GitHub not-found objects so they can self-heal", async () => {
|
||||
const { companyId } = await createIssue();
|
||||
const { companyId, issueId } = await createIssue();
|
||||
const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42");
|
||||
if (!canonical) throw new Error("expected canonical url");
|
||||
const inserted = await db.insert(externalObjects).values({
|
||||
@@ -1220,9 +1231,10 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
const svc = externalObjectService(db, { resolvers: [resolver], github: false });
|
||||
expect(object.isTerminal).toBe(true);
|
||||
|
||||
const refreshed = await svc.refreshDueObjects(companyId);
|
||||
expect(refreshed).toHaveLength(1);
|
||||
expect(refreshed[0]?.refreshed).toBe(true);
|
||||
const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId));
|
||||
await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId,
|
||||
sourceKind: "description", objectId: linked!.id });
|
||||
await runRefreshWork(svc);
|
||||
|
||||
const updated = await db.select().from(externalObjects).then((rows) => rows[0]!);
|
||||
expect(updated.statusLabel).toBe("Open");
|
||||
@@ -1231,7 +1243,7 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
});
|
||||
|
||||
it("does not re-check probe-verified GitHub not-found objects (deleted references are terminal)", async () => {
|
||||
const { companyId } = await createIssue();
|
||||
const { companyId, issueId } = await createIssue();
|
||||
const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42");
|
||||
if (!canonical) throw new Error("expected canonical url");
|
||||
await db.insert(externalObjects).values({
|
||||
@@ -1268,14 +1280,16 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
};
|
||||
const svc = externalObjectService(db, { resolvers: [resolver], github: false });
|
||||
|
||||
const refreshed = await svc.refreshDueObjects(companyId);
|
||||
const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId));
|
||||
await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId,
|
||||
sourceKind: "description", objectId: linked!.id });
|
||||
await runRefreshWork(svc);
|
||||
|
||||
expect(refreshed).toEqual([]);
|
||||
expect(resolver.resolve).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not re-check terminal objects that are not GitHub not-found verdicts", async () => {
|
||||
const { companyId } = await createIssue();
|
||||
const { companyId, issueId } = await createIssue();
|
||||
const canonical = canonicalizeExternalObjectUrl("https://github.com/acme/app/pull/42");
|
||||
if (!canonical) throw new Error("expected canonical url");
|
||||
await db.insert(externalObjects).values({
|
||||
@@ -1311,9 +1325,11 @@ describeEmbeddedPostgres("externalObjectService", () => {
|
||||
};
|
||||
const svc = externalObjectService(db, { resolvers: [resolver], github: false });
|
||||
|
||||
const refreshed = await svc.refreshDueObjects(companyId);
|
||||
const [linked] = await db.select().from(externalObjects).where(eq(externalObjects.companyId, companyId));
|
||||
await db.insert(externalObjectMentions).values({ companyId, sourceIssueId: issueId,
|
||||
sourceKind: "description", objectId: linked!.id });
|
||||
await runRefreshWork(svc);
|
||||
|
||||
expect(refreshed).toEqual([]);
|
||||
expect(resolver.resolve).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ describeEmbeddedPostgres("git restore point retention", () => {
|
||||
} = {}) {
|
||||
const companyId = input.companyId ?? randomUUID();
|
||||
if (!input.companyId) {
|
||||
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.slice(0, 6)}` });
|
||||
await db.insert(companies).values({ id: companyId, name: "Restore", issuePrefix: `R${companyId.replaceAll("-", "")}` });
|
||||
}
|
||||
const issueStatus = input.issueStatus ?? "done";
|
||||
const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { externalObjectService } from "../services/external-objects.js";
|
||||
import { startToolHealthWorker } from "./helpers/tool-health-worker.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
@@ -179,6 +180,19 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
expect(connector.leaseEvents).toHaveBeenCalledTimes(5);
|
||||
}
|
||||
|
||||
let releaseRefresh!: () => void;
|
||||
let markStarted!: () => void;
|
||||
const started = new Promise<void>(resolve => { markStarted = resolve; });
|
||||
const held = new Promise<void>(resolve => { releaseRefresh = resolve; });
|
||||
const resolver = externalObjectService(db, { github: false, resolvers: [{
|
||||
providerKey: "github", resolve: async () => {
|
||||
markStarted(); await held;
|
||||
return { ok: true, snapshot: { statusKey: "open", statusCategory: "open", statusTone: "info", isTerminal: false } };
|
||||
},
|
||||
}] });
|
||||
const oldRefresh = resolver.refreshObject(externalObjectId, { companyId, force: true });
|
||||
await started;
|
||||
|
||||
const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
|
||||
const outcome = await service.pollOnce({ forSetup: activeSetup });
|
||||
const [processingReceipt] = await db.select().from(connectionEventDeliveries).where(eq(connectionEventDeliveries.providerDeliveryId, leasedEvent.id));
|
||||
@@ -188,6 +202,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
deliveryId: "undecryptable-delivery", reason: "inner_event_authentication_failed",
|
||||
}, "GitHub connector event retained for retry"]]);
|
||||
warning.mockRestore();
|
||||
releaseRefresh();
|
||||
await expect(oldRefresh).resolves.toMatchObject({ refreshed: false, reason: "refresh_superseded" });
|
||||
expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, confirmation!.id)))[0]!.status).toBe("accepted");
|
||||
const [snapshot] = await db.select().from(externalObjects).where(eq(externalObjects.id, externalObjectId));
|
||||
expect(snapshot).toMatchObject({
|
||||
@@ -196,6 +212,8 @@ describeEmbeddedPostgres("GitHub connection event delivery", () => {
|
||||
statusCategory: "succeeded",
|
||||
statusTone: "success",
|
||||
isTerminal: true,
|
||||
refreshToken: null,
|
||||
refreshStartedAt: null,
|
||||
remoteVersion: "2026-09-04T11:59:01.000Z",
|
||||
data: expect.objectContaining({ marker: "preserved", merged: true, headRef: "feature", baseSha: "b".repeat(40) }),
|
||||
});
|
||||
|
||||
@@ -19,11 +19,20 @@ import { secretService } from "../services/secrets.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { aiConnectionService } from "../services/ai-connections.js";
|
||||
import * as aiRuntime from "../services/ai-connection-runtime.js";
|
||||
import * as connectorRuntime from "../services/connector-runtime.js";
|
||||
import * as assignedMcp from "../services/native-runtime/assigned-mcp-tools.js";
|
||||
import * as sentry from "../sentry.js";
|
||||
import { waitForPendingRunFailureReports } from "../services/run-failure-report.js";
|
||||
import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js";
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>(complete => { resolve = complete; });
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
const skillFailures = vi.hoisted(() => new Map<string, string>());
|
||||
const skillWaits = vi.hoisted(() => new Map<string, () => Promise<void>>());
|
||||
|
||||
describe("Runner home mode during recovery preparation", () => {
|
||||
it.each([
|
||||
@@ -52,6 +61,7 @@ vi.mock("../services/company-skills.js", async importOriginal => {
|
||||
return { ...service, withRuntimeSkillEntries: async <T>(companyId: string,
|
||||
options: Parameters<typeof service.withRuntimeSkillEntries>[1],
|
||||
consume: (entries: Awaited<ReturnType<typeof service.listRuntimeSkillEntries>>) => T | Promise<T>) => {
|
||||
await skillWaits.get(companyId)?.();
|
||||
const failure = skillFailures.get(companyId);
|
||||
if (failure) throw new Error(failure);
|
||||
return service.withRuntimeSkillEntries(companyId, options, consume);
|
||||
@@ -91,7 +101,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }),
|
||||
});
|
||||
}, 60_000);
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); });
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllEnvs(); skillFailures.clear(); skillWaits.clear(); });
|
||||
afterAll(async () => {
|
||||
await heartbeat?.drainActiveRunExecutions();
|
||||
unregisterServerAdapter(adapterType);
|
||||
@@ -288,6 +298,45 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const [unchangedAgent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
expect(unchangedAgent.adapterConfig).toMatchObject({ paperclipSkillSync: { desiredSkills: [other.key] } });
|
||||
}, 60_000);
|
||||
it("overlaps skills with assignments and dependent connection instructions before composition", async () => {
|
||||
const f = await seed();
|
||||
const skillGate = deferred<void>();
|
||||
skillWaits.set(f.companyId, () => skillGate.promise);
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments");
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun");
|
||||
const configured = await configuration(f, { provider: "codex", model: "gpt-6.1-sol", homeMode: "isolated" });
|
||||
const opening = configured.prepare();
|
||||
try {
|
||||
await vi.waitFor(() => {
|
||||
expect(assignments).toHaveBeenCalledOnce();
|
||||
expect(instructions).toHaveBeenCalledOnce();
|
||||
});
|
||||
} finally {
|
||||
skillGate.resolve();
|
||||
}
|
||||
expect(await opening).toMatchObject({ prepared: true, runtimeConfig: { homeMode: "isolated", model: "gpt-6.1-sol" } });
|
||||
});
|
||||
|
||||
it("settles concurrent configuration reads before exposing the first preparation failure", async () => {
|
||||
const f = await seed();
|
||||
const assignmentGate = deferred<Awaited<ReturnType<typeof connectorRuntime.resolveConnectorAssignments>>>();
|
||||
const assignments = vi.spyOn(connectorRuntime, "resolveConnectorAssignments").mockReturnValue(assignmentGate.promise);
|
||||
const instructions = vi.spyOn(assignedMcp, "resolveAssignedConnectionInstructionsForRun").mockRejectedValue(new Error("later instruction failure"));
|
||||
skillFailures.set(f.companyId, "first skill failure");
|
||||
const configured = await configuration(f, { provider: "codex", homeMode: "isolated" });
|
||||
let settled = false;
|
||||
const opening = configured.prepare();
|
||||
void opening.then(() => { settled = true; }, () => { settled = true; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(assignments).toHaveBeenCalledOnce());
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
expect(instructions).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
assignmentGate.resolve([]);
|
||||
}
|
||||
await expect(opening).rejects.toThrow("first skill failure");
|
||||
});
|
||||
|
||||
it("does not install new environment declarations when recovering a saved skill selection", async () => {
|
||||
const f = await seed(), pluginId = randomUUID(), pluginKey = `fixture.${randomUUID()}`;
|
||||
@@ -445,11 +494,11 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
const runIdHash = createHash("sha256").update(finished.id).digest("hex").slice(0, 12);
|
||||
const steps = exporter.getFinishedSpans().filter(span => span.name.startsWith("heartbeat.prepare.configuration_")
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.map(span => span.name)).toEqual([
|
||||
expect(steps.map(span => span.name).sort()).toEqual([
|
||||
"configuration_mentions", "configuration_github_identity", "configuration_secrets", "configuration_ai_runtime",
|
||||
"configuration_skills", "configuration_connector_assignments", "configuration_connector_skills",
|
||||
"configuration_connector_delivery", "configuration_connection_instructions", "configuration_agent_identity",
|
||||
].map(step => `heartbeat.prepare.${step}`));
|
||||
].map(step => `heartbeat.prepare.${step}`).sort());
|
||||
for (const span of steps) {
|
||||
expect(span.status).toEqual({ code: 0 });
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
@@ -494,7 +543,7 @@ describePostgres("heartbeat execution configuration through adapter dispatch", (
|
||||
&& span.attributes["paperclip.task.run.run_id"] === runIdHash);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_ai_runtime")?.status.code).toBe(0);
|
||||
expect(steps.find(span => span.name === "heartbeat.prepare.configuration_skills")?.status).toEqual({ code: 2 });
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(false);
|
||||
expect(steps.some(span => span.name === "heartbeat.prepare.configuration_connector_assignments")).toBe(true);
|
||||
for (const span of steps) {
|
||||
expect(span.attributes).toEqual({ "paperclip.task.run.run_id": runIdHash });
|
||||
expect(span.events).toEqual([]);
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
import { registerRunDispatchWork } from "../services/heartbeat/dispatch-work.js";
|
||||
|
||||
const cleanup: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => {
|
||||
for (const stop of cleanup.splice(0)) await stop();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
function fixture(input: { due?: number; suppressed?: boolean; draining?: boolean } = {}) {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected database query");
|
||||
});
|
||||
const db: { execute: typeof execute; transaction: (callback: (tx: any) => Promise<any>) => Promise<any> } = installDatabaseWorkSignals({ execute, async transaction(callback: (tx: any) => Promise<any>): Promise<any> {
|
||||
return callback({ execute, transaction: db.transaction });
|
||||
} });
|
||||
const state = { due: input.due ?? null as number | null, suppressed: input.suppressed ?? false,
|
||||
draining: input.draining ?? false, standby: false, fail: false };
|
||||
const promote = vi.fn(async () => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.due !== null && state.due <= Date.now()) state.due = null;
|
||||
});
|
||||
const next = vi.fn(async () => state.due);
|
||||
const dispatch = vi.fn(async (_scope?: { companyId: string }) => {});
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db,
|
||||
canRun: () => !state.draining && !state.standby, canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerRunDispatchWork(coordinator, {
|
||||
promoteDueScheduledRetries: promote, nextScheduledRetryAt: next,
|
||||
resumeQueuedRunsAfterDrain: dispatch, schedulingSuppressed: async () => state.suppressed,
|
||||
}, () => !state.draining && !state.standby);
|
||||
cleanup.push(() => coordinator.stop());
|
||||
return { db, state, promote, next, dispatch, coordinator, worker,
|
||||
notify: (scope?: { companyId: string }) => db.transaction(tx => notifyRunDispatch(tx, scope)),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
|
||||
describe("run dispatch obligations", () => {
|
||||
it.each([false, true])("settles empty/feature-suppressed state and stops all owner work (suppressed=%s)", async suppressed => {
|
||||
const f = fixture({ suppressed }); await f.worker.ready;
|
||||
const calls = [f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length];
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect([f.promote.mock.calls.length, f.next.mock.calls.length, f.dispatch.mock.calls.length]).toEqual(calls);
|
||||
expect(f.db.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
it("restores an exact deadline and does no earlier recovery work", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_010_000);
|
||||
await vi.advanceTimersByTimeAsync(9_999); expect(f.promote).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
expect(f.state.due).toBeNull(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.dispatch).toHaveBeenCalledOnce();
|
||||
});
|
||||
it("rearms earlier work and cancels a future deadline on a committed producer change", async () => {
|
||||
const f = fixture({ due: 1_600_000 }); await f.worker.ready;
|
||||
f.state.due = 1_005_000; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_005_000);
|
||||
f.state.due = null; await f.notify(); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const before = f.next.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.next).toHaveBeenCalledTimes(before);
|
||||
});
|
||||
it("batches repeated tenant writes and passes only their company to queued dispatch", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const source = { companyId: "company-a", issueId: `issue-${index % 2}`,
|
||||
runId: `run-${index}`, agentId: `agent-${index}`, interactionBody: "not a dispatch key" };
|
||||
await f.notify(source);
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
}
|
||||
expect(f.promote.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
expect(f.dispatch.mock.calls.length).toBeLessThanOrEqual(2);
|
||||
await f.settle();
|
||||
expect(f.dispatch).toHaveBeenLastCalledWith({ companyId: "company-a" });
|
||||
for (const [scope] of f.dispatch.mock.calls.slice(1)) expect(scope).toEqual({ companyId: "company-a" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const count = f.dispatch.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.dispatch).toHaveBeenCalledTimes(count);
|
||||
});
|
||||
it("remembers the first producer arriving during drain without owner SQL", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
await f.notify(); await f.settle(); await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.dispatch).toHaveBeenCalledTimes(2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases a failed producer hold during drain and retains recovery for resume", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
f.state.fail = true; await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
f.state.draining = true; await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(2); expect(f.next).toHaveBeenCalledOnce();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.fail = false; f.state.draining = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("preserves a notification during a nonoverlapping pass", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
let release!: () => void;
|
||||
const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.promote.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle();
|
||||
await f.notify(); await f.settle();
|
||||
expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
release(); await f.settle();
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledTimes(3); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("does not dispatch on warm standby and cancels deadlines at shutdown", async () => {
|
||||
const f = fixture({ due: 1_010_000 }); f.state.standby = true; await f.worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).not.toHaveBeenCalled();
|
||||
f.state.standby = false; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.promote).toHaveBeenCalledOnce(); expect(f.state.due).toBeNull();
|
||||
f.state.due = Date.now() + 60_000; await f.notify(); await f.settle(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(90_000); expect(f.promote).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,238 @@
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, costEvents, createDb, databaseWorkPending, heartbeatRuns } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { notifyRunDispatch } from "../services/run-dispatch-notifications.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_DISPATCH_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("run dispatch production wiring (controlled local process adapter)", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]> | undefined;
|
||||
let reads: ReturnType<typeof vi.spyOn>;
|
||||
let draining = false;
|
||||
const errors: unknown[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("run-dispatch-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
reads = vi.spyOn(db, "select");
|
||||
});
|
||||
afterEach(async () => {
|
||||
await worker?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
worker = undefined; coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining,
|
||||
canReconcile: () => true, onError: error => errors.push(error) });
|
||||
worker = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
await worker.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Dispatch", issuePrefix: `D${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "dispatch-user" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Controlled process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('dispatch-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId: agent.id, status: "failed",
|
||||
responsibleUserId: "dispatch-user", errorCode: "codex_transient_upstream", finishedAt: new Date(),
|
||||
resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, contextSnapshot: {},
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id, runId: run.id };
|
||||
}
|
||||
async function quiet() {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBeNull(), { timeout: 5_000 });
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
}
|
||||
async function status(id: string) {
|
||||
return db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]);
|
||||
}
|
||||
async function completed(id: string) {
|
||||
await vi.waitFor(async () => expect(["running", "succeeded"]).toContain((await status(id))?.status), { timeout: 10_000 });
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const completed = await status(id);
|
||||
expect(completed.status).toBe("succeeded");
|
||||
expect(completed.stdoutExcerpt).toContain("dispatch-proof");
|
||||
}
|
||||
it("keeps fresh empty, configured idle and retained terminal history quiet", async () => {
|
||||
await start(); await quiet();
|
||||
await fixture(); await worker!.stop(); await coordinator!.stop();
|
||||
await start(); await quiet();
|
||||
});
|
||||
it("ordinary retry creation wakes the worker, respects the exact deadline, dispatches and settles", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const result = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 4_000, random: () => 0 });
|
||||
expect(result.outcome).toBe("scheduled");
|
||||
if (result.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(result.dueAt.getTime()), { timeout: 5_000 });
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(Date.now()).toBeLessThan(result.dueAt.getTime());
|
||||
expect((await status(result.run.id)).status).toBe("scheduled_retry");
|
||||
await completed(result.run.id);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), eq(heartbeatRuns.retryOfRunId, f.runId)))).toHaveLength(1);
|
||||
await quiet();
|
||||
});
|
||||
it("reconstructs an overdue retry after process restart without a periodic tick", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 120_000), delayMs: 60_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
heartbeat = heartbeatService(db);
|
||||
await start(); await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("cancels a future retry and removes its old deadline", async () => {
|
||||
const f = await fixture(); await start();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 600_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(coordinator!.nextWakeAt()).toBe(scheduled.dueAt.getTime()), { timeout: 5_000 });
|
||||
await heartbeat.cancelRun(scheduled.run.id, "operator stop", { suppressImmediateRecovery: true });
|
||||
await quiet(); expect((await status(scheduled.run.id)).status).toBe("cancelled");
|
||||
});
|
||||
it("retains the first retry written during drain and catches it up after resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { delayMs: 200 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
const count = reads.mock.calls.length;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
draining = false; worker!.wake();
|
||||
await completed(scheduled.run.id); await quiet();
|
||||
});
|
||||
it("uses a committed budget raise to resume a queued run in the company it paused", async () => {
|
||||
const f = await fixture();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 100,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
const [cost] = await db.insert(costEvents).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
provider: "fixture", biller: "fixture", model: "controlled", inputTokens: 0, outputTokens: 0,
|
||||
costCents: 125, occurredAt: new Date() }).returning();
|
||||
await budgets.evaluateCostEvent(cost);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, f.companyId)))[0].status).toBe("paused");
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
await start(); await quiet(); expect((await status(queued.id)).status).toBe("queued");
|
||||
await budgets.upsertPolicy(f.companyId, { scopeType: "company", scopeId: f.companyId, amount: 10_000,
|
||||
metric: "billed_cents", windowKind: "lifetime", hardStopEnabled: true }, "dispatch-user");
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("clearing an agent error wakes saved queue work only after its own transaction commits", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error" }).where(eq(agents.id, f.agentId));
|
||||
const [queued] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "queued", responsibleUserId: "dispatch-user", invocationSource: "automation", contextSnapshot: {} }).returning();
|
||||
let writing: ReturnType<ReturnType<typeof agentService>["clearError"]> | undefined;
|
||||
try {
|
||||
await db.transaction(async blocker => {
|
||||
await blocker.select().from(agents).where(eq(agents.id, f.agentId)).for("update");
|
||||
writing = agentService(db).clearError(f.agentId);
|
||||
void writing.catch(() => {});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(databaseWorkPending(db, DELIVERY_QUEUES.queuedRunDispatch)).toBe(false);
|
||||
expect((await status(queued.id)).status).toBe("queued");
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
});
|
||||
} finally { await writing; }
|
||||
await completed(queued.id); await quiet();
|
||||
});
|
||||
it("leaves cancellation-owned retries quiet even when their old promotion time is overdue", async () => {
|
||||
const f = await fixture();
|
||||
const scheduled = await heartbeat.scheduleBoundedRetry(f.runId, { now: new Date(Date.now() - 60_000), delayMs: 1_000 });
|
||||
if (scheduled.outcome !== "scheduled") throw new Error("Expected retry creation");
|
||||
await db.update(heartbeatRuns).set({ resultJson: { startupCancellation: { requestedAt: new Date().toISOString() },
|
||||
cancellation: { expected: true, reason: "operator stop", source: "operator" } } }).where(eq(heartbeatRuns.id, scheduled.run.id));
|
||||
await start(); await quiet();
|
||||
expect((await status(scheduled.run.id)).status).toBe("scheduled_retry");
|
||||
expect(await heartbeat.promoteDueScheduledRetries()).toEqual({ promoted: 0, runIds: [] });
|
||||
await quiet();
|
||||
});
|
||||
it("rejects an error clear without publishing work when the lifecycle is no longer ready", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
await db.update(agents).set({ status: "error", lifecycleState: "paused" }).where(eq(agents.id, f.agentId));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try {
|
||||
await expect(agentService(db).clearError(f.agentId)).rejects.toThrow("Only agents in error status");
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const count = reads.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(reads).toHaveBeenCalledTimes(count);
|
||||
} finally { vi.useRealTimers(); }
|
||||
expect((await db.select().from(agents).where(eq(agents.id, f.agentId)))[0].status).toBe("error");
|
||||
await quiet();
|
||||
});
|
||||
it("keeps a foreign company's future retry intact while concurrent workers claim one due retry", async () => {
|
||||
const selected = await fixture(), foreign = await fixture(); await start();
|
||||
const later = await heartbeat.scheduleBoundedRetry(foreign.runId, { delayMs: 600_000 });
|
||||
const due = await heartbeat.scheduleBoundedRetry(selected.runId, { delayMs: 500 });
|
||||
if (later.outcome !== "scheduled" || due.outcome !== "scheduled") throw new Error("Expected retries");
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const competingWorker = other.registerRunDispatchWork(competing, () => true);
|
||||
try {
|
||||
await competingWorker.ready; await completed(due.run.id); await other.drainActiveRunExecutions();
|
||||
expect((await status(later.run.id)).status).toBe("scheduled_retry");
|
||||
expect((await status(later.run.id)).scheduledRetryAt).toEqual(later.dueAt);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.retryOfRunId, selected.runId))).toHaveLength(1);
|
||||
await heartbeat.cancelRun(later.run.id, "fixture teardown", { suppressImmediateRecovery: true });
|
||||
await quiet();
|
||||
} finally { await competingWorker.stop(); await competing.stop(); await other.drainActiveRunExecutions(); }
|
||||
});
|
||||
it("holds nested producer work until outer commit and rolls back without a successor", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
let queuedId = "";
|
||||
const writing = db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await notifyRunDispatch(nested);
|
||||
const [row] = await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
responsibleUserId: "dispatch-user", status: "queued", contextSnapshot: {} }).returning();
|
||||
queuedId = row.id;
|
||||
});
|
||||
await held;
|
||||
});
|
||||
await vi.waitFor(() => expect(queuedId).not.toBe(""));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await status(queuedId)).toBeUndefined();
|
||||
release(); await writing; await completed(queuedId); await quiet();
|
||||
await expect(db.transaction(async tx => {
|
||||
await tx.transaction(async nested => { await notifyRunDispatch(nested); await nested.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "queued" }); });
|
||||
throw new Error("rollback outer");
|
||||
})).rejects.toThrow("rollback outer");
|
||||
await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
@@ -262,6 +262,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
const [delivered] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
expect(delivered.runId).toBeTruthy();
|
||||
// The receipt can commit before execution enters liveRunExecutions.
|
||||
// Observe actual adapter admission before awaiting its drain.
|
||||
await vi.waitFor(async () => {
|
||||
const current = await heartbeat.getRun(delivered.runId!);
|
||||
expect(countExecuteCallsForRun(delivered.runId!), JSON.stringify({
|
||||
status: current?.status, errorCode: current?.errorCode,
|
||||
})).toBe(1);
|
||||
}, { timeout: 5_000 });
|
||||
await heartbeat.waitForRunExecutionDrain(delivered.runId!);
|
||||
expect(countExecuteCallsForRun(delivered.runId!)).toBe(1);
|
||||
expect((await heartbeat.getRun(delivered.runId!))?.contextSnapshot).toMatchObject({ wakeCommentIds: [comment.id] });
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE } from "@paperclipai/shared";
|
||||
import { registerServerAdapter, unregisterServerAdapter, type AdapterExecutionContext } from "../adapters/index.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { prepareHeartbeatTaskContext } from "../services/heartbeat/task-context.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import { ISSUE_CONTINUATION_SUMMARY_DOCUMENT_KEY } from "../services/issue-continuation-summary.js";
|
||||
import { createRunSecretRedactionRegistry } from "../services/run-secret-redaction.js";
|
||||
@@ -19,6 +20,7 @@ if (!support.supported) console.warn(`Skipping heartbeat task context tests: ${s
|
||||
describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
const adapterType = "task_context_test";
|
||||
const captured = new Map<string, AdapterExecutionContext>();
|
||||
const inspectPreparation = new Map<string, () => void>();
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
@@ -33,6 +35,7 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
type: adapterType,
|
||||
supportsLocalAgentJwt: false,
|
||||
execute: async (input) => {
|
||||
inspectPreparation.get(input.agent.id)?.();
|
||||
captured.set(input.runId, input);
|
||||
return { exitCode: 0, signal: null, timedOut: false, sessionId: "context-session", resultJson: {} };
|
||||
},
|
||||
@@ -118,6 +121,101 @@ describePostgres("heartbeat task context delivered to an adapter", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reads selected comments once for both wake projections", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({
|
||||
id: commentId, companyId: f.companyId, issueId: f.issueId, authorType: "user", authorUserId: "context-user",
|
||||
body: "selected wake canary",
|
||||
});
|
||||
const select = vi.spyOn(db, "select");
|
||||
inspectPreparation.set(f.agentId, () => {
|
||||
const commentReads = select.mock.calls.filter(([projection]) => projection
|
||||
&& projection.body === issueComments.body && projection.id === issueComments.id && projection.deletedAt === issueComments.deletedAt);
|
||||
expect(commentReads).toHaveLength(1);
|
||||
});
|
||||
try {
|
||||
const { context } = await execute(f, { wakeCommentId: commentId, wakeCommentIds: [commentId] });
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "selected wake canary" });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "selected wake canary" }] });
|
||||
} finally {
|
||||
inspectPreparation.delete(f.agentId);
|
||||
select.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels deleted-only queued comments before adapter dispatch", async () => {
|
||||
const f = await seed(), commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "discarded wake", deletedAt: new Date() });
|
||||
const run = await heartbeat.wakeup(f.agentId, { source: "on_demand", triggerDetail: "manual", reason: "issue_commented",
|
||||
payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] } });
|
||||
expect(run).not.toBeNull();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const finished = await heartbeat.getRun(run!.id);
|
||||
expect(finished).toMatchObject({ status: "cancelled", error: "Queued messages were discarded before dispatch" });
|
||||
expect(captured.has(run!.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves deleted comment metadata without delivering its body", async () => {
|
||||
const f = await seed();
|
||||
const commentId = randomUUID();
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: f.issueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "deleted wake canary", deletedAt: new Date(),
|
||||
presentation: { fixture: "deleted" }, metadata: { fixture: "deleted" } });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(context.paperclipWakeComment).toMatchObject({ id: commentId, body: "", presentation: null, metadata: null });
|
||||
expect(context.paperclipWake).toMatchObject({ comments: [{ id: commentId, body: "", presentation: null, metadata: null }] });
|
||||
expect(context.paperclipTaskMarkdown).not.toContain("deleted wake canary");
|
||||
});
|
||||
|
||||
it.each([false, true])("uses admitted skill-test scope when recovery is %s", async (recovering) => {
|
||||
const f = await seed();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const [issueRef] = await db.select().from(issues).where(eq(issues.id, f.issueId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, paperclipAgentJwtScope: { kind: "skill_test", issueId: f.issueId } };
|
||||
const pinned = vi.fn(async () => null);
|
||||
const preparing = prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef, issueContext: null, context, taskSession: null },
|
||||
policy: { recovering, trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: pinned },
|
||||
});
|
||||
if (recovering) await expect(preparing).rejects.toThrow("skill test's saved version is unavailable");
|
||||
else await preparing;
|
||||
expect(pinned).toHaveBeenCalledTimes(recovering ? 1 : 0);
|
||||
expect(context.paperclipAgentJwtScope).toEqual({ kind: recovering ? "skill_test" : "standard", ...(recovering ? { issueId: f.issueId } : {}) });
|
||||
});
|
||||
|
||||
it("retains the context issue scope when no issue reference is available", async () => {
|
||||
const f = await seed();
|
||||
const otherIssueId = randomUUID(), commentId = randomUUID();
|
||||
await db.insert(issues).values({ id: otherIssueId, companyId: f.companyId, title: "Other issue", issueNumber: 2 });
|
||||
await db.insert(issueComments).values({ id: commentId, companyId: f.companyId, issueId: otherIssueId,
|
||||
authorType: "user", authorUserId: "context-user", body: "other issue canary" });
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", issueId: f.issueId }).returning();
|
||||
const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId));
|
||||
const context: Record<string, unknown> = { issueId: f.issueId, wakeCommentId: commentId, wakeCommentIds: [commentId] };
|
||||
await prepareHeartbeatTaskContext(db, {
|
||||
run, agent, task: { issueRef: null, issueContext: null, context, taskSession: null },
|
||||
policy: { trustPreset: { kind: "standard" }, experimentalInstanceSettings: {} },
|
||||
services: { issuesSvc: { getAncestors: async () => [] }, getPinnedSkillTestContext: async () => null },
|
||||
});
|
||||
expect(JSON.stringify(context.paperclipWake)).not.toContain("other issue canary");
|
||||
expect(context.paperclipWakeComment).toBeUndefined();
|
||||
});
|
||||
|
||||
it("quarantines low-trust summaries and comments before preparing higher-trust prompts", async () => {
|
||||
const f = await seed();
|
||||
const sourceTrust = buildLowTrustSourceTrust({ issueId: f.issueId });
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { subscribeDatabaseWork } from "@paperclipai/db";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
@@ -888,7 +890,7 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
beforeAll(async () => {
|
||||
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl);
|
||||
if (externalTestDatabaseUrl) db = createDb(externalTestDatabaseUrl, { maxConnections: 6 });
|
||||
else {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-branch-containment-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
@@ -1173,15 +1175,15 @@ describeEmbeddedPostgres("heartbeat workspace branch containment", () => {
|
||||
},
|
||||
});
|
||||
|
||||
const routineEvents: string[] = [];
|
||||
const unsubscribeRoutine = subscribeDatabaseWork(db, DELIVERY_QUEUES.routineExecutionStatus, event => routineEvents.push(event));
|
||||
const heartbeat = heartbeatService(db);
|
||||
const run = await heartbeat.wakeup(agentId, {
|
||||
source: "assignment",
|
||||
triggerDetail: "system",
|
||||
reason: "issue_assigned",
|
||||
payload: { issueId },
|
||||
contextSnapshot: { issueId, wakeReason: "issue_assigned" },
|
||||
});
|
||||
|
||||
let run;
|
||||
try {
|
||||
run = await heartbeat.wakeup(agentId, { source: "assignment", triggerDetail: "system", reason: "issue_assigned",
|
||||
payload: { issueId }, contextSnapshot: { issueId, wakeReason: "issue_assigned" } });
|
||||
} finally { unsubscribeRoutine(); }
|
||||
expect(routineEvents).toContain("intent"); expect(routineEvents).toContain("settled");
|
||||
expect(run).toBeNull();
|
||||
expect(adapterExecute).not.toHaveBeenCalled();
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, environmentCustomImageSetupSessions, environmentLeases, environments, heartbeatRuns } from "@paperclipai/db";
|
||||
import { historicalCleanupVerificationService } from "../services/historical-cleanup-verification.js";
|
||||
import { environmentRuntimeService } from "../services/environment-runtime.js";
|
||||
import { withoutCleanupVerification } from "../services/cleanup-verification-proof.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js";
|
||||
@@ -16,7 +17,7 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
(support.supported ? describe : describe.skip)("historical cleanup verification", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("historical-cleanup-"); db = createDb(database.connectionString); }, 60_000);
|
||||
afterEach(async () => { await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterEach(async () => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies, plugins, environments CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, async () => "none", undefined, undefined, true, true);
|
||||
const state = async () => (await read()).backgroundWork;
|
||||
@@ -37,6 +38,55 @@ if (!support.supported) console.warn(`Skipping historical cleanup database tests
|
||||
inspectSetup: async row => ({ providerLeaseId: row.providerLeaseId, state: setupState }),
|
||||
});
|
||||
}
|
||||
it.each([false, true])("reconciles a legacy Cloud lease through signed absence and permits idle sleep (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc);
|
||||
const { privateKey, publicKey } = generateKeyPairSync("ed25519");
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [{ ...publicKey.export({ format: "jwk" }), kid: "fixture", alg: "EdDSA", use: "sig" }] }) })) vi.stubEnv(key, value);
|
||||
let absent = false;
|
||||
const fetchImpl = vi.fn(async (_url: string, init: RequestInit) => {
|
||||
if (!absent) return Response.json({ absent: false });
|
||||
const binding = JSON.parse(init.body as string), seconds = Math.floor(Date.now() / 1000);
|
||||
expect(binding).toEqual({ purpose: "cleanup", companyId: f.companyId, environmentId: f.environment.id,
|
||||
leaseId: f.lease.id, heartbeatRunId: adHoc ? null : f.runId, provider: "daytona", providerLeaseId: f.lease.providerLeaseId });
|
||||
const header = Buffer.from(JSON.stringify({ alg: "EdDSA", kid: "fixture", typ: "paperclip-sandbox-cleanup-absence+jwt" })).toString("base64url");
|
||||
const claims = Buffer.from(JSON.stringify({ v: 1, iss: "paperclip-cloud", aud: "paperclip-sandbox-cleanup-absence/v1",
|
||||
sub: "fixture-stack", ...binding, absent: true, iat: seconds, exp: seconds + 60 })).toString("base64url");
|
||||
return Response.json({ receipt: `${header}.${claims}.${sign(null, Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}` });
|
||||
});
|
||||
vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db);
|
||||
const service = historicalCleanupVerificationService(db, {
|
||||
inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined,
|
||||
});
|
||||
expect(await state()).toBe("present");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, deferred: 1, verified: 0 });
|
||||
expect(await state()).toBe("present");
|
||||
absent = true;
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
expect(await state()).toBe("none");
|
||||
const [current] = await db.select().from(environmentLeases);
|
||||
expect(current).toMatchObject({ status: "failed", cleanupStatus: "success", heartbeatRunId: adHoc ? null : f.runId });
|
||||
expect(current!.metadata).not.toHaveProperty("remoteExecutionTermination");
|
||||
expect(await service.sweep()).toMatchObject({ checked: 0 });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
it.each(["active", "stopped", "unknown"] as const)("does not replace an explicit %s provider observation with a Cloud read", async providerState => {
|
||||
await seed(true);
|
||||
for (const [key, value] of Object.entries({ PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_API_ORIGIN: "https://cloud.example.test", PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "fleet-fixture",
|
||||
PAPERCLIP_PRIMARY_HOST: "fixture.paperclip.test", PAPERCLIP_AUTH_SECRET: "fixture-secret" })) vi.stubEnv(key, value);
|
||||
const fetchImpl = vi.fn(); vi.stubGlobal("fetch", fetchImpl);
|
||||
const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox",
|
||||
acquireRunLease: async () => { throw new Error("unexpected acquire"); }, releaseRunLease: async () => null,
|
||||
inspectLeaseResource: async ({ lease }) => ({ providerLeaseId: lease.providerLeaseId!, state: providerState }),
|
||||
}] });
|
||||
const service = historicalCleanupVerificationService(db, { inspectLease: row => runtime.inspectLeaseResource(row.id), inspectSetup: async () => undefined });
|
||||
expect(await service.sweep()).toMatchObject({ checked: 1, verified: 0, deferred: 1 });
|
||||
expect(await state()).toBe("present"); expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([false, true])("accepts exact deletion proof without changing outcomes or granting run authority (ad-hoc %s)", async adHoc => {
|
||||
const f = await seed(adHoc); expect(await state()).toBe("present");
|
||||
const service = verifier(); expect(await service.sweep()).toMatchObject({ checked: 1, verified: 1, failed: 0 });
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
approvals,
|
||||
companies,
|
||||
createDb,
|
||||
applyPendingMigrations,
|
||||
heartbeatRuns,
|
||||
issueApprovals,
|
||||
issueRelations,
|
||||
@@ -21,7 +22,10 @@ import {
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { buildIssueGraphLivenessIncidentKey } from "../services/recovery/origins.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const externalUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_BLOCKER_ATTENTION_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -36,8 +40,9 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (externalUrl) await applyPendingMigrations(externalUrl);
|
||||
else tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-blocker-attention-");
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
svc = issueService(db);
|
||||
}, 20_000);
|
||||
|
||||
@@ -55,6 +60,7 @@ describeEmbeddedPostgres("issue blocker attention", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,382 @@
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issueRecoveryActions, issueRelations, issues, issueWatchdogs } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { issueRecoveryActionService } from "../services/issue-recovery-actions.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { taskWatchdogService } from "../services/task-watchdogs.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSUE_RECOVERY_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("issue recovery production producers and deadlines", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let heartbeat: ReturnType<typeof heartbeatService>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let worker: ReturnType<ReturnType<typeof heartbeatService>["registerIssueStateRecoveryWork"]>;
|
||||
let dispatch: ReturnType<ReturnType<typeof heartbeatService>["registerRunDispatchWork"]>;
|
||||
let cancellation: ReturnType<ReturnType<typeof heartbeatService>["registerScheduledRetryCancellationWork"]>;
|
||||
let draining = false;
|
||||
const tasks = new Map<string, { nextRunAt?: () => number | null; hasPending: () => Promise<boolean> }>();
|
||||
const errors: unknown[] = [];
|
||||
const passes = new Set<Promise<unknown>>();
|
||||
const extraStops: Array<() => Promise<void>> = [];
|
||||
const extraDeadlines: Array<() => number | null> = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) database = await startEmbeddedPostgresTestDatabase("issue-recovery-work-");
|
||||
db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
await db.execute(sql`TRUNCATE companies CASCADE`);
|
||||
heartbeat = heartbeatService(db); draining = false; errors.length = 0;
|
||||
});
|
||||
afterEach(async () => {
|
||||
for (const stop of extraStops.splice(0)) await stop();
|
||||
extraDeadlines.length = 0;
|
||||
await worker?.stop(); await dispatch?.stop(); await cancellation?.stop(); await coordinator?.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); });
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
tasks.clear();
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => {
|
||||
tasks.set(queue, task);
|
||||
return register(queue, { ...task, run: signal => {
|
||||
const work = task.run(signal); passes.add(work);
|
||||
void work.then(() => passes.delete(work), () => passes.delete(work));
|
||||
return work;
|
||||
} });
|
||||
};
|
||||
worker = heartbeat.registerIssueStateRecoveryWork(coordinator, () => !draining);
|
||||
dispatch = heartbeat.registerRunDispatchWork(coordinator, () => !draining);
|
||||
cancellation = heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !draining);
|
||||
await Promise.all([worker.ready, dispatch.ready, cancellation.ready]);
|
||||
}
|
||||
async function compete() {
|
||||
const other = heartbeatService(db);
|
||||
const competing = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const work = other.registerIssueStateRecoveryWork(competing, () => true);
|
||||
extraStops.push(() => competing.stop()); extraDeadlines.push(competing.nextWakeAt); await work.ready;
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID();
|
||||
await db.insert(authUsers).values({ id: "recovery-user", name: "Recovery operator", email: "recovery-user@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
|
||||
await db.insert(companies).values({ id: companyId, name: "Recovery", issuePrefix: `R${companyId.slice(0, 7)}`,
|
||||
requireBoardApprovalForNewAgents: false, defaultResponsibleUserId: "recovery-user" });
|
||||
await db.insert(companyMemberships).values({ companyId, principalId: "recovery-user", principalType: "user", status: "active", membershipRole: "owner" });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Recovery process", role: "engineer", status: "idle",
|
||||
adapterType: "process", adapterConfig: { command: process.execPath, args: ["-e", "console.log('recovery-proof')"], cwd: process.cwd() },
|
||||
runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, wakeOnDemand: true } },
|
||||
}).returning();
|
||||
return { companyId, agentId: agent.id };
|
||||
}
|
||||
async function settled() {
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const wake = coordinator.nextWakeAt();
|
||||
const detail = wake === null ? "" : JSON.stringify({ now: Date.now(), tasks: await Promise.all([...tasks].map(async ([queue, task]) => ({ queue, at: task.nextRunAt?.(), pending: await task.hasPending() }))),
|
||||
runs: (await db.select().from(heartbeatRuns)).map(run => ({ id: run.id, status: run.status, error: run.errorCode, lease: run.controllerLeaseExpiresAt, reason: run.contextSnapshot?.wakeReason })) });
|
||||
expect(wake, detail).toBeNull();
|
||||
expect(extraDeadlines.map(next => next())).toEqual(extraDeadlines.map(() => null));
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
}, { timeout: 40_000 });
|
||||
}
|
||||
async function quiet() {
|
||||
await settled();
|
||||
expect(await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.status} in ('queued', 'running')`)).toHaveLength(0);
|
||||
const reads = vi.spyOn(db, "select"); const executes = vi.spyOn(db, "execute");
|
||||
const selected = reads.mock.calls.length, executed = executes.mock.calls.length;
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(20 * 60_000); expect(reads).toHaveBeenCalledTimes(selected); expect(executes).toHaveBeenCalledTimes(executed); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); executes.mockRestore(); }
|
||||
}
|
||||
async function finish(issueId: string) {
|
||||
await issueService(db).update(issueId, { status: "done" });
|
||||
// Closing one blocker can legitimately start its dependent. Wait for this
|
||||
// task's dispatch and execution; quiet() verifies the whole fixture later.
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
|
||||
sql`${heartbeatRuns.status} in ('queued', 'running')`,
|
||||
))).toHaveLength(0);
|
||||
}, { timeout: 15_000 });
|
||||
const saved = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
if (saved.length > 0) {
|
||||
const due = Math.max(...saved.map(run => run.scheduledRetryAt!.getTime()));
|
||||
// Keep PostgreSQL I/O and coordinator settlement ahead of virtual time.
|
||||
// A single leap can otherwise occur before the retry deadline is armed.
|
||||
while (Date.now() <= due) {
|
||||
await vi.advanceTimersByTimeAsync(Math.min(500, due - Date.now() + 1));
|
||||
await Promise.allSettled([...passes]);
|
||||
await immediate();
|
||||
}
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await Promise.allSettled([...passes]);
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const remaining = await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`));
|
||||
expect(remaining).toHaveLength(0);
|
||||
});
|
||||
for (const retry of saved) {
|
||||
const [settledRetry] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry.id));
|
||||
expect(settledRetry.status).toBe("cancelled");
|
||||
}
|
||||
}
|
||||
}
|
||||
async function completed(issueId: string) {
|
||||
let runId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const runs = await db.select().from(heartbeatRuns).where(sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`);
|
||||
const run = runs.find(row => row.status === "running" || row.status === "succeeded");
|
||||
expect(run).toBeDefined(); runId = run!.id;
|
||||
}, { timeout: 15_000 });
|
||||
await vi.waitFor(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(current.status, JSON.stringify({ error: current.error, code: current.errorCode, stderr: current.stderrExcerpt })).toBe("succeeded");
|
||||
}, { timeout: 15_000 });
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId!));
|
||||
expect(run.stdoutExcerpt).toContain("recovery-proof");
|
||||
return run;
|
||||
}
|
||||
it("keeps empty, configured idle, and terminal history quiet", async () => {
|
||||
await fixture(); await start(); await quiet();
|
||||
});
|
||||
it("creates a blocking child through the ordinary root service transaction", async () => {
|
||||
const f = await fixture();
|
||||
const parent = await issueService(db).create(f.companyId, { title: "Parent", status: "todo" });
|
||||
const child = await issueService(db).createChild(parent.id, { title: "Blocking child", status: "todo", blockParentUntilDone: true });
|
||||
expect(child.parentBlockerAdded).toBe(true);
|
||||
expect(await db.select().from(issueRelations).where(and(eq(issueRelations.companyId, f.companyId),
|
||||
eq(issueRelations.issueId, child.issue.id), eq(issueRelations.relatedIssueId, parent.id)))).toHaveLength(1);
|
||||
});
|
||||
it.each(["startup", "committed"])("restores blocker-only and completed-action debt with no assignment candidates (%s)", async mode => {
|
||||
const f = await fixture();
|
||||
if (mode === "committed") { await start(); await quiet(); }
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Orphan blocker", status: "blocked", createdByAgentId: f.agentId });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Blocked dependent", status: "blocked", blockedByIssueIds: [blocker.id] });
|
||||
const completedSource = await issueService(db).create(f.companyId, { title: "Completed source", status: "done" });
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: completedSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "source-terminal", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
if (mode === "startup") await start();
|
||||
await vi.waitFor(async () => {
|
||||
expect((await db.select().from(issues).where(eq(issues.id, blocker.id)))[0].assigneeAgentId).toBe(f.agentId);
|
||||
expect((await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
}, { timeout: 40_000 });
|
||||
await finish(blocker.id); await finish(dependent.id); await quiet();
|
||||
});
|
||||
it("bounds subsidiary scans for100 assignment candidates and coalesces repeated company writes", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
await db.update(agents).set({ lifecycleState: "preparing" }).where(eq(agents.id, f.agentId));
|
||||
await db.insert(issues).values(Array.from({ length: 100 }, (_, index) => ({ companyId: f.companyId,
|
||||
title: `Waiting setup ${index}`, status: "todo", assigneeAgentId: f.agentId })));
|
||||
const [source] = await db.insert(issues).values({ companyId: f.companyId, title: "Done source", status: "done" }).returning();
|
||||
const action = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, sourceIssueId: source.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "agent", ownerAgentId: f.agentId, cause: "stranded_assigned_issue",
|
||||
fingerprint: "query-budget", nextAction: "Restore work", wakePolicy: { type: "wake_owner" }, evidence: {} });
|
||||
const scans = { blockers: 0, actions: 0, loads: 0 };
|
||||
const originalSelect = db.select.bind(db);
|
||||
vi.spyOn(db, "select").mockImplementation(((fields?: Record<string, unknown>) => {
|
||||
const query = originalSelect(fields as never); const from = query.from.bind(query);
|
||||
query.from = ((table: unknown) => {
|
||||
if (table === issues && fields?.id === issues.id && fields.companyId === issues.companyId && Object.keys(fields).length === 2) scans.loads++;
|
||||
if (table === issueRelations && fields?.createdByAgentId === issues.createdByAgentId) scans.blockers++;
|
||||
if (table === issueRecoveryActions && fields?.action === issueRecoveryActions) scans.actions++;
|
||||
return from(table as never);
|
||||
}) as typeof query.from;
|
||||
return query;
|
||||
}) as typeof db.select);
|
||||
await start(); await settled();
|
||||
expect(scans.actions).toBe(1); expect(scans.blockers).toBe(0); expect(scans.loads).toBeLessThanOrEqual(8);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, action.id)))[0].status).toBe("resolved");
|
||||
const [foreignSource] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign done", status: "done" }).returning();
|
||||
const [foreignAction] = await db.insert(issueRecoveryActions).values({ companyId: foreign.companyId, sourceIssueId: foreignSource.id,
|
||||
kind: "stranded_assigned_issue", ownerType: "board", cause: "stranded_assigned_issue", fingerprint: "foreign-query-budget", nextAction: "Inspect" }).returning();
|
||||
const before = { ...scans };
|
||||
for (let index = 0; index < 100; index++) await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(scans.loads).toBe(before.loads);
|
||||
await settled();
|
||||
expect(scans.actions).toBe(before.actions); expect(scans.blockers).toBe(before.blockers);
|
||||
expect(scans.loads).toBeLessThanOrEqual(before.loads + 4);
|
||||
expect((await originalSelect().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, foreignAction.id)))[0].status).toBe("active");
|
||||
expect(await originalSelect().from(heartbeatRuns)).toHaveLength(0); await quiet();
|
||||
});
|
||||
it("limits an ordinary comment to its issue among one thousand open assignments", async () => {
|
||||
const f = await fixture();
|
||||
const rows = await db.insert(issues).values(Array.from({ length: 1000 }, (_, index) => ({
|
||||
companyId: f.companyId, title: `Open work ${index}`, status: "todo", assigneeAgentId: f.agentId,
|
||||
}))).returning({ id: issues.id });
|
||||
const checked: string[] = [];
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = task.run(signal); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass));
|
||||
return pass;
|
||||
} });
|
||||
// Observe the actual paged reconstruction boundary without executing one
|
||||
// thousand adapters. The producer, transaction, coordinator and SQL are real;
|
||||
// the other acceptance cases exercise the resulting recovery decisions.
|
||||
worker = registerIssueRecoveryWork(db, coordinator, {
|
||||
stranded: async scope => { checked.push(scope.issueId); },
|
||||
orphanBlockers: async () => {}, recoveryActions: async () => {},
|
||||
dependencies: async () => ({ enqueueFailed: 0 }), locks: async () => {},
|
||||
watchdogs: async () => ({ incomplete: 0 }), schedulingSuppressed: async () => false,
|
||||
}, () => true);
|
||||
await worker.ready;
|
||||
async function settleWork() {
|
||||
for (let step = 0; step < 120; step++) {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await Promise.allSettled([...passes]); await immediate();
|
||||
if (coordinator.nextWakeAt() === null && idleWorkSnapshot().active === 0) return;
|
||||
}
|
||||
throw new Error("Recovery did not settle its committed work within two virtual minutes");
|
||||
}
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
checked.length = 0;
|
||||
await issueService(db).addComment(rows[0].id, "New evidence on this task", { userId: "recovery-user" });
|
||||
await settleWork();
|
||||
expect(checked).toEqual([rows[0].id]);
|
||||
// Company eligibility changes still reconstruct all assignments. Narrowing
|
||||
// comment hints must not hide unrelated work after an operator resumes it.
|
||||
checked.length = 0;
|
||||
await db.transaction(tx => notifyIssueRecovery(tx, { companyId: f.companyId }));
|
||||
await settleWork();
|
||||
expect(new Set(checked).size).toBe(1000);
|
||||
await quiet();
|
||||
});
|
||||
it("dispatches an ordinary new assignment after commit and returns to quiet after completion", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Recover assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await completed(issue.id);
|
||||
await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("reacts to blocker resolution once and dispatches the durable dependency wake", async () => {
|
||||
const f = await fixture();
|
||||
const blocker = await issueService(db).create(f.companyId, { title: "Blocker", status: "todo" });
|
||||
const dependent = await issueService(db).create(f.companyId, { title: "Dependent", status: "blocked", assigneeAgentId: f.agentId,
|
||||
blockedByIssueIds: [blocker.id] });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
await issueService(db).update(blocker.id, { status: "done" });
|
||||
const run = await completed(dependent.id);
|
||||
expect(run.contextSnapshot?.wakeReason).toBe("issue_blockers_resolved");
|
||||
await finish(dependent.id); await quiet();
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'issue_blockers_resolved'`))).toHaveLength(1);
|
||||
}, 90_000);
|
||||
it("uses ordinary watchdog configuration and subtree closure to dispatch a reusable review", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Finished subtree", status: "done" });
|
||||
await start(); await quiet();
|
||||
await compete();
|
||||
const watchdog = await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
let reviewId: string | undefined;
|
||||
await vi.waitFor(async () => {
|
||||
const [row] = await db.select().from(issueWatchdogs).where(eq(issueWatchdogs.id, watchdog.watchdog.id));
|
||||
expect(row.watchdogIssueId).toBeTruthy(); reviewId = row.watchdogIssueId!;
|
||||
}, { timeout: 15_000 });
|
||||
await completed(reviewId!);
|
||||
expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId),
|
||||
sql`${heartbeatRuns.contextSnapshot}->>'wakeReason' = 'task_watchdog_stopped_subtree'`))).toHaveLength(1);
|
||||
await finish(reviewId!);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
it("bounds ordinary writes to their company and reconstructs foreign stale debt after restart", async () => {
|
||||
const selected = await fixture(); await start(); await quiet();
|
||||
const foreign = await fixture();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: foreign.companyId, agentId: foreign.agentId,
|
||||
status: "failed", finishedAt: new Date() }).returning();
|
||||
const [stale] = await db.insert(issues).values({ companyId: foreign.companyId, title: "Foreign stale lock", status: "done",
|
||||
checkoutRunId: run.id }).returning();
|
||||
await issueService(db).create(selected.companyId, { title: "Selected change", status: "done" }); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBe(run.id);
|
||||
await worker.stop(); await dispatch.stop(); await cancellation.stop(); await coordinator.stop();
|
||||
heartbeat = heartbeatService(db); await start(); await quiet();
|
||||
expect((await db.select().from(issues).where(eq(issues.id, stale.id)))[0].checkoutRunId).toBeNull();
|
||||
});
|
||||
it("fences nested assignment writes until outer commit and ignores rollback", async () => {
|
||||
const f = await fixture(); await start(); await quiet();
|
||||
let committedId = "";
|
||||
await db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Nested commit", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
committedId = issue.id;
|
||||
});
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
});
|
||||
await completed(committedId); await finish(committedId); await settled();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await issueService(db).create(f.companyId, { title: "Nested rollback", status: "todo", assigneeAgentId: f.agentId }, nested);
|
||||
});
|
||||
throw new Error("rollback control");
|
||||
})).rejects.toThrow("rollback control");
|
||||
await quiet();
|
||||
expect(await db.select().from(issues).where(eq(issues.title, "Nested rollback"))).toHaveLength(0);
|
||||
}, 90_000);
|
||||
it("retains the first committed assignment during drain and executes on resume", async () => {
|
||||
const f = await fixture(); await start(); await quiet(); draining = true;
|
||||
const issue = await issueService(db).create(f.companyId, { title: "Drained assignment", status: "todo", assigneeAgentId: f.agentId });
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
const reads = vi.spyOn(db, "select");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] });
|
||||
try { await vi.advanceTimersByTimeAsync(90_000); expect(reads).not.toHaveBeenCalled(); }
|
||||
finally { vi.useRealTimers(); reads.mockRestore(); }
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
draining = false; worker.wake(); dispatch.wake();
|
||||
await completed(issue.id); await finish(issue.id); await quiet();
|
||||
}, 90_000);
|
||||
it("retains a fresh subtree's exact first-run grace then disarms on disable", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Fresh unassigned subtree", status: "todo" });
|
||||
await start(); await quiet();
|
||||
await taskWatchdogService(db).upsertForIssue(f.companyId, source.id, { agentId: f.agentId });
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(source.createdAt.getTime() + 15_000), { timeout: 5_000 });
|
||||
expect(await db.select().from(issues).where(eq(issues.originKind, "task_watchdog"))).toHaveLength(0);
|
||||
await taskWatchdogService(db).disableForIssue(f.companyId, source.id);
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
});
|
||||
it("waits for a live foreign controller's exact stored lease before clearing a terminal lock", async () => {
|
||||
const f = await fixture();
|
||||
const source = await issueService(db).create(f.companyId, { title: "Remote owner", status: "done" });
|
||||
const lease = new Date(Date.now() + 5_000);
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running",
|
||||
runtimeMode: "legacy", controllerBootId: randomUUID(), controllerLeaseExpiresAt: lease,
|
||||
contextSnapshot: { issueId: source.id }, processPid: 2_000_000_000, startedAt: new Date() }).returning();
|
||||
await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, source.id));
|
||||
await start();
|
||||
await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBe(lease.getTime()), { timeout: 3_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("running");
|
||||
await vi.waitFor(async () => expect((await db.select().from(issues).where(eq(issues.id, source.id)))[0].executionRunId).toBeNull(), { timeout: 12_000 });
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].status).toBe("succeeded");
|
||||
if (!vi.isFakeTimers()) vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); await quiet();
|
||||
}, 90_000);
|
||||
});
|
||||
@@ -0,0 +1,149 @@
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { issueRecoveryActions, issueWatchdogs } from "@paperclipai/db";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyIssueRecovery } from "../services/issue-recovery-notifications.js";
|
||||
import { registerIssueRecoveryWork } from "../services/recovery/work.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const stops: Array<() => Promise<void>> = [];
|
||||
afterEach(async () => { for (const stop of stops.splice(0)) await stop(); vi.useRealTimers(); });
|
||||
function fixture() {
|
||||
vi.useFakeTimers(); vi.setSystemTime(1_000_000);
|
||||
const dialect = new PgDialect();
|
||||
const rows = { issues: [] as Array<{ id: string; companyId: string }>, watchdogs: [] as Array<{ id: string; companyId: string }>, blockers: [] as Array<{ id: string; companyId: string }>, actions: [] as Array<{ id: string; companyId: string }> };
|
||||
const execute = vi.fn(async (query: any) => {
|
||||
if (dialect.sqlToQuery(query).sql.includes("pg_current_xact_id")) return [{ xid: "42" }];
|
||||
throw new Error("unexpected SQL");
|
||||
});
|
||||
const load = vi.fn((table: unknown, distinct = false) => distinct
|
||||
? table === issueRecoveryActions ? rows.actions : rows.blockers
|
||||
: table === issueWatchdogs ? rows.watchdogs : rows.issues);
|
||||
const query = (distinct = false) => ({ from: (table: unknown) => ({ where: () => ({ orderBy: () => ({ limit: async () => load(table, distinct) }) }) }) });
|
||||
const db: any = installDatabaseWorkSignals({ execute,
|
||||
select: () => query(), selectDistinct: () => query(true),
|
||||
transaction: async (callback: (tx: any) => Promise<unknown>) => callback({ execute, select: db.select, selectDistinct: db.selectDistinct, transaction: db.transaction }),
|
||||
});
|
||||
const state = { draining: false, standby: false, suppressed: false, deadline: null as number | null, fail: false, incomplete: false };
|
||||
const check = vi.fn(async (scope: { onDeferredDeadline(at: number): void }) => {
|
||||
if (state.fail) throw new Error("database unavailable");
|
||||
if (state.deadline !== null && state.deadline > Date.now()) scope.onDeferredDeadline(state.deadline);
|
||||
});
|
||||
const orphanBlockers = vi.fn(async () => {}), recoveryActions = vi.fn(async () => {});
|
||||
const suppression = vi.fn(async () => state.suppressed);
|
||||
const owner = { orphanBlockers, recoveryActions, stranded: check, dependencies: async (scope: any) => { await check(scope); return { enqueueFailed: 0 }; },
|
||||
locks: check, watchdogs: async (scope: any) => { await check(scope); return { incomplete: state.incomplete ? 1 : 0 }; },
|
||||
schedulingSuppressed: suppression };
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !state.standby,
|
||||
canReconcile: () => !state.standby, onError: vi.fn() });
|
||||
const worker = registerIssueRecoveryWork(db, coordinator, owner, () => !state.draining && !state.standby);
|
||||
stops.push(() => coordinator.stop());
|
||||
return { db, rows, state, check, load, coordinator, worker, orphanBlockers, recoveryActions, suppression,
|
||||
notify: () => db.transaction((tx: any) => notifyIssueRecovery(tx, { companyId: "selected" })),
|
||||
settle: () => vi.advanceTimersByTimeAsync(1_000) };
|
||||
}
|
||||
describe("issue recovery obligation lifecycle", () => {
|
||||
it("has no timer, SQL, or owner probe after empty reconstruction", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
});
|
||||
it("reads scheduling suppression once per pass instead of once per issue", async () => {
|
||||
const f = fixture();
|
||||
f.rows.issues.push(...Array.from({ length: 25 }, (_, index) => ({ id: `issue-${index}`, companyId: "selected" })));
|
||||
await f.worker.ready;
|
||||
expect(f.check).toHaveBeenCalledTimes(75);
|
||||
expect(f.suppression).toHaveBeenCalledTimes(6);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("backs off a failed suppression read without probing an empty or failed owner repeatedly", async () => {
|
||||
const f = fixture(); f.suppression.mockRejectedValueOnce(new Error("settings unavailable"));
|
||||
await f.worker.ready; const reads = f.suppression.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); expect(f.suppression).toHaveBeenCalledTimes(reads);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.suppression).toHaveBeenCalledTimes(reads + 1);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("normalizes a wide scope with cyclic SQL objects before publishing work hints", async () => {
|
||||
const f = fixture(); await f.worker.ready;
|
||||
const wide = { companyId: "selected", endpointId: issueRecoveryActions.id, table: issueRecoveryActions };
|
||||
await f.db.transaction((tx: object) => notifyIssueRecovery(tx, wide)); await f.settle();
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
const loads = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(f.load).toHaveBeenCalledTimes(loads);
|
||||
});
|
||||
it("runs orphan blocker and action duties without assignment entries", async () => {
|
||||
const f = fixture(); f.rows.blockers.push({ id: "selected", companyId: "selected" });
|
||||
f.rows.actions.push({ id: "selected", companyId: "selected" }); await f.worker.ready;
|
||||
expect(f.check).not.toHaveBeenCalled();
|
||||
expect(f.orphanBlockers).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.recoveryActions).toHaveBeenCalledExactlyOnceWith({ companyId: "selected" });
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("coalesces repeated company reconstruction without delaying an existing lease deadline", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.state.deadline = 1_015_000; await f.worker.ready;
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length, checked = f.check.mock.calls.length;
|
||||
for (let index = 0; index < 100; index++) { await f.notify(); await vi.advanceTimersByTimeAsync(10); }
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(13_000);
|
||||
expect(f.state.deadline).toBe(1_015_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_031_000);
|
||||
await vi.advanceTimersByTimeAsync(16_000);
|
||||
expect(f.load.mock.calls.length).toBeGreaterThan(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retains exact grace and lease deadlines without early owner probes", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.deadline = 1_015_000;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_015_000);
|
||||
const checked = f.check.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(14_999); expect(f.check).toHaveBeenCalledTimes(checked);
|
||||
await vi.advanceTimersByTimeAsync(1); expect(f.check.mock.calls.length).toBe(checked * 2);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("releases settled producer holds during domain drain while coordinator remains open", async () => {
|
||||
const f = fixture(); await f.worker.ready; f.state.draining = true;
|
||||
f.rows.issues.push({ id: "issue", companyId: "selected" });
|
||||
await f.notify(); await f.settle(); const loaded = f.load.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(90_000);
|
||||
expect(f.load).toHaveBeenCalledTimes(loaded); expect(f.check).not.toHaveBeenCalled();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
f.state.draining = false; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalled(); expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("retries concrete failed entries and then disarms without reloading the company", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); f.state.fail = true;
|
||||
await f.worker.ready; const loaded = f.load.mock.calls.length;
|
||||
expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); const checked = f.check.mock.calls.length;
|
||||
f.state.fail = false; await vi.advanceTimersByTimeAsync(1);
|
||||
expect(f.check.mock.calls.length).toBe(checked * 2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps an incomplete watchdog subtree as concrete retry debt", async () => {
|
||||
const f = fixture(); f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" }); f.state.incomplete = true;
|
||||
await f.worker.ready; expect(f.coordinator.nextWakeAt()).toBe(1_030_000);
|
||||
const loaded = f.load.mock.calls.length; f.state.incomplete = false;
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.check).toHaveBeenCalledTimes(2); expect(f.load).toHaveBeenCalledTimes(loaded);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
it("keeps a concurrent producer dirty until the in-flight owner has completed", async () => {
|
||||
const f = fixture(); f.rows.issues.push({ id: "issue", companyId: "selected" }); await f.worker.ready;
|
||||
let release!: () => void; const held = new Promise<void>(resolve => { release = resolve; });
|
||||
f.check.mockImplementationOnce(() => held);
|
||||
await f.notify(); await f.settle(); await f.notify(); release(); await f.settle(); await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(f.coordinator.nextWakeAt()).toBeNull();
|
||||
expect(f.check.mock.calls.length).toBeGreaterThan(3);
|
||||
});
|
||||
it("retains startup work on standby and cancels future deadlines on shutdown", async () => {
|
||||
const f = fixture(); f.state.standby = true; f.rows.watchdogs.push({ id: "watchdog", companyId: "selected" });
|
||||
await f.worker.ready; await vi.advanceTimersByTimeAsync(90_000); expect(f.load).not.toHaveBeenCalled();
|
||||
f.state.standby = false; f.state.deadline = Date.now() + 60_000; f.worker.wake(); await f.settle();
|
||||
expect(f.check).toHaveBeenCalledOnce(); await f.worker.stop();
|
||||
await vi.advanceTimersByTimeAsync(120_000); expect(f.check).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -41,7 +41,7 @@ describeEmbeddedPostgres("issueTreeControlService", () => {
|
||||
beforeAll(async () => {
|
||||
if (externalUrl) {
|
||||
await applyPendingMigrations(externalUrl);
|
||||
db = createDb(externalUrl);
|
||||
db = createDb(externalUrl, { maxConnections: 6 });
|
||||
return;
|
||||
}
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issue-tree-control-");
|
||||
|
||||
@@ -4151,7 +4151,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!dependencyTestDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-issues-blockers-");
|
||||
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString);
|
||||
db = createDb(dependencyTestDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
svc = issueService(db);
|
||||
await ensureIssueRelationsTable(db);
|
||||
}, 20_000);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { createServer } from "node:http";
|
||||
import { createS3StorageProvider } from "../storage/s3-provider.js";
|
||||
import { skillInstallationMaintenance } from "../services/skill-installation-maintenance.js";
|
||||
import { removeRuntimeAssetTree } from "../services/native-runtime/runtime-asset-retention.js";
|
||||
import { eq } from "drizzle-orm";
|
||||
@@ -77,6 +79,93 @@ describe.skipIf(!support.supported)("managed skills use committed versions", ()
|
||||
.toMatchObject({ content: "complete root guide" });
|
||||
});
|
||||
|
||||
it("loads persisted Cloud configuration for read-only maintenance without seeding it", async () => {
|
||||
const companyId = await company();
|
||||
const objects = new Map<string, Buffer>();
|
||||
const server = createServer((request, response) => {
|
||||
void (async () => {
|
||||
const key = new URL(request.url!, "http://fixture").pathname;
|
||||
if (request.method === "PUT") {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk));
|
||||
let body = Buffer.concat(chunks);
|
||||
if (request.headers["content-encoding"]?.includes("aws-chunked")) {
|
||||
const decoded: Buffer[] = [];
|
||||
let offset = 0;
|
||||
while (offset < body.length) {
|
||||
const end = body.indexOf("\r\n", offset);
|
||||
const size = Number.parseInt(body.subarray(offset, end).toString().split(";")[0]!, 16);
|
||||
if (size === 0) break;
|
||||
decoded.push(body.subarray(end + 2, end + 2 + size)); offset = end + 2 + size + 2;
|
||||
}
|
||||
body = Buffer.concat(decoded);
|
||||
}
|
||||
objects.set(key, body); response.writeHead(200, { etag: '"fixture"' }).end(); return;
|
||||
}
|
||||
const body = objects.get(key);
|
||||
if (!body) { response.writeHead(404).end(); return; }
|
||||
response.writeHead(200, { "content-length": body.length, "content-type": "application/octet-stream", etag: '"fixture"' });
|
||||
response.end(request.method === "HEAD" ? undefined : body);
|
||||
})().catch(() => { response.writeHead(500).end(); });
|
||||
});
|
||||
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected fixture address");
|
||||
const endpoint = `http://127.0.0.1:${address.port}`;
|
||||
const credentials = { AWS_ACCESS_KEY_ID: "fixture-access-key", AWS_SECRET_ACCESS_KEY: "fixture-secret-key", AWS_SESSION_TOKEN: "" };
|
||||
const previousCredentials = Object.fromEntries(Object.keys(credentials).map(key => [key, process.env[key]]));
|
||||
Object.assign(process.env, credentials);
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createS3StorageProvider({
|
||||
bucket: "managed-skill-fixture", region: "us-east-1", endpoint, forcePathStyle: true,
|
||||
})));
|
||||
const key = "cloud-runtime-config:v1";
|
||||
try {
|
||||
await companySkillService(db).createLocalSkill(companyId, { name: "Cloud CLI", markdown: "# Cloud CLI" });
|
||||
const run = (phase: string) => promisify(execFile)(process.execPath, [
|
||||
"--import", fileURLToPath(import.meta.resolve("tsx")),
|
||||
fileURLToPath(new URL("../../scripts/migrate-skill-packages.ts", import.meta.url)),
|
||||
`--phase=${phase}`, `--company=${companyId}`, "--limit=50",
|
||||
], { env: { ...process.env, ...credentials, DATABASE_URL: database.connectionString,
|
||||
PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "disposable-cloud-cli-fixture",
|
||||
PAPERCLIP_SECRETS_MASTER_KEY: "a".repeat(64), PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
PAPERCLIP_STORAGE_S3_BUCKET: "managed-skill-fixture", PAPERCLIP_STORAGE_S3_REGION: "us-east-1",
|
||||
PAPERCLIP_STORAGE_S3_ENDPOINT: endpoint, PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "true", PAPERCLIP_STORAGE_S3_PREFIX: "" },
|
||||
timeout: 60_000, maxBuffer: 1024 * 1024 });
|
||||
await expect(run("eligibility")).rejects.toThrow("Initialize Cloud runtime configuration");
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key))).toHaveLength(0);
|
||||
await db.insert(instanceSettings).values({ singletonKey: key, general: { version: 1, revision: 0, config: null } });
|
||||
const eligibility = JSON.parse((await run("eligibility")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(eligibility.results).toEqual([expect.objectContaining({ status: "version_backed" })]);
|
||||
const before = [...objects.entries()].map(([key, body]) => [key, Buffer.from(body)] as const);
|
||||
const verification = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(verification.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()]).toEqual(before);
|
||||
const archiveKey = [...objects.keys()].find(key => key.endsWith("/files.tar.gz"))!;
|
||||
const archive = objects.get(archiveKey)!;
|
||||
const corrupted = Buffer.from(archive); corrupted[0] = corrupted[0]! ^ 1;
|
||||
objects.set(archiveKey, corrupted);
|
||||
const corrupt = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(corrupt.results).toEqual([expect.objectContaining({ status: "corrupt_package" })]);
|
||||
objects.set(archiveKey, archive);
|
||||
const manifestKey = [...objects.keys()].find(key => key.endsWith("/manifest.json"))!;
|
||||
const manifest = objects.get(manifestKey)!; objects.delete(manifestKey);
|
||||
const unavailable = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(unavailable.results).toEqual([expect.objectContaining({ status: "unavailable_or_corrupt_package" })]);
|
||||
objects.set(manifestKey, manifest);
|
||||
const recovered = JSON.parse((await run("verification")).stdout.trim().split("\n").at(-1)!);
|
||||
expect(recovered.results).toEqual([expect.objectContaining({ status: "remote_storage" })]);
|
||||
expect([...objects.entries()].sort()).toEqual([...before].sort());
|
||||
expect(await db.select().from(instanceSettings).where(eq(instanceSettings.singletonKey, key)))
|
||||
.toEqual([expect.objectContaining({ general: { version: 1, revision: 0, config: null } })]);
|
||||
} finally {
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, key));
|
||||
vi.mocked(storageModule.getStorageService).mockReturnValue(createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))));
|
||||
for (const [key, value] of Object.entries(previousCredentials)) {
|
||||
if (value === undefined) delete process.env[key]; else process.env[key] = value;
|
||||
}
|
||||
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
it.each([
|
||||
{ storage: "legacy local", env: { PAPERCLIP_STORAGE_PROVIDER: "local_disk" }, expectedStatus: "local_storage" },
|
||||
{ storage: "shared S3", env: { PAPERCLIP_STORAGE_PROVIDER: "s3",
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { inArray } from "drizzle-orm";
|
||||
@@ -131,7 +132,8 @@ describe("merged pull-request confirmation extraction", () => {
|
||||
});
|
||||
});
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_SWEEP_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
|
||||
@@ -139,8 +141,8 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-merged-pr-confirmations-");
|
||||
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -153,6 +155,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
@@ -276,7 +279,7 @@ describeEmbeddedPostgres("merged pull-request confirmation sweep", () => {
|
||||
repo: "paperclip",
|
||||
number: 39,
|
||||
}])).resolves.toEqual({
|
||||
checked: 6,
|
||||
checked: 3,
|
||||
candidates: 3,
|
||||
accepted: 2,
|
||||
woken: 2,
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
import { createServer } from "node:http";
|
||||
import { createPullRequestMergeStateResolver } from "../services/github-pull-request-merge.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { activityLog, agents, chatActions, chatConversations, chatEndpoints, chatPublications, companies, createDb, issues, issueThreadInteractions, toolApplications, toolConnections, type Db } from "@paperclipai/db";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { issueThreadInteractionService } from "../services/issue-thread-interactions.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { registerMergeConfirmationWork, MERGE_CONFIRMATION_RECONCILE_MS } from "../services/merged-pr-confirmation-work.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_MERGE_WORK_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("merge confirmation work with PostgreSQL and simulated GitHub", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let companyId: string, issueId: string, agentId: string;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let draining = false, standby = false;
|
||||
const errors: unknown[] = [];
|
||||
const provider = vi.fn(async (_companyId: string, _reference: { number: number }): Promise<"open" | "merged" | "unknown"> => "merged");
|
||||
const wakeup = vi.fn(async () => ({ id: "simulated-durable-wake" }));
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("merged-pr-work-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
}, 30_000);
|
||||
beforeEach(async () => {
|
||||
errors.length = 0; draining = false; standby = false;
|
||||
provider.mockReset().mockResolvedValue("merged"); wakeup.mockClear();
|
||||
companyId = randomUUID(); agentId = randomUUID(); issueId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Merge worker", issuePrefix: `M${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Coder", role: "engineer", status: "idle", adapterType: "codex_local" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Merge task", status: "todo", assigneeAgentId: agentId });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.delete(chatActions); await db.delete(chatPublications); await db.delete(chatConversations); await db.delete(chatEndpoints);
|
||||
await db.delete(toolConnections); await db.delete(toolApplications);
|
||||
await db.delete(activityLog); await db.delete(issueThreadInteractions); await db.delete(issues);
|
||||
await db.delete(agents); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
const service = (database: Db = db) => issueThreadInteractionService(database, { resolvePullRequestState: provider, wakeup });
|
||||
const input = (number = 39) => ({ kind: "request_confirmation" as const, title: `Merge paperclipai/paperclip#${number}?`,
|
||||
resolverPolicy: "human_only" as const, continuationPolicy: "wake_assignee_on_accept" as const,
|
||||
payload: { version: 1 as const, prompt: "Merge the PR?" } });
|
||||
const create = (database: Db = db, number = 39) => service(database).create({ id: issueId, companyId }, input(number), { agentId });
|
||||
const stored = (id: string) => db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, id)).then(rows => rows[0]);
|
||||
async function start() {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining && !standby,
|
||||
canReconcile: () => !standby, onError: error => errors.push(error) });
|
||||
const worker = registerMergeConfirmationWork(coordinator, service(), () => !draining && !standby);
|
||||
await worker.ready;
|
||||
return worker;
|
||||
}
|
||||
async function waitFor(check: () => Promise<void> | void) {
|
||||
for (let attempt = 0; attempt < 100; attempt++) {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await delay(10);
|
||||
try { await check(); return; } catch (error) { if (attempt === 99) throw error; }
|
||||
}
|
||||
}
|
||||
const settle = () => waitFor(() => expect(idleWorkSnapshot().active).toBe(0));
|
||||
async function quiet() {
|
||||
await waitFor(() => { expect(idleWorkSnapshot().active).toBe(0); expect(coordinator!.nextWakeAt()).toBeNull(); });
|
||||
const reads = vi.spyOn(db, "select"), transactions = vi.spyOn(db, "transaction"), executes = vi.spyOn(db, "execute");
|
||||
const calls = provider.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(3 * MERGE_CONFIRMATION_RECONCILE_MS);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(transactions).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
|
||||
expect(provider).toHaveBeenCalledTimes(calls);
|
||||
reads.mockRestore(); transactions.mockRestore(); executes.mockRestore();
|
||||
}
|
||||
|
||||
it("settles empty and configured idle, then ordinary creation accepts, audits, wakes once and becomes quiet with retained history", async () => {
|
||||
await start(); await quiet(); expect(provider).not.toHaveBeenCalled();
|
||||
const card = await create();
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted"));
|
||||
await quiet();
|
||||
expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${card.id}:accepted`, allowRunCoalescing: false }));
|
||||
expect(await db.select().from(activityLog)).toEqual(expect.arrayContaining([expect.objectContaining({
|
||||
action: "issue.thread_interaction_accepted", actorId: "system:pr-merged", details: expect.objectContaining({ interactionId: card.id }),
|
||||
})]));
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("reconciles an open candidate at its deadline, never early, then stops after merge", async () => {
|
||||
provider.mockResolvedValue("open"); await start(); const card = await create();
|
||||
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
|
||||
const dueAt = coordinator!.nextWakeAt()!;
|
||||
expect(dueAt).toBeGreaterThan(Date.now());
|
||||
await vi.advanceTimersByTimeAsync(dueAt - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1);
|
||||
provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1);
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(provider).toHaveBeenCalledTimes(2); expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("manual cancellation removes the candidate deadline without provider work", async () => {
|
||||
provider.mockResolvedValue("open"); await start(); const card = await create();
|
||||
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
|
||||
await service().withdrawInteraction({ id: issueId, companyId }, card.id, {}, { agentId });
|
||||
await quiet(); expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).not.toHaveBeenCalled();
|
||||
expect((await stored(card.id)).status).toBe("cancelled");
|
||||
});
|
||||
|
||||
it("does not accept before caller-owned outer commit and forgets a rolled-back create", async () => {
|
||||
await start(); await quiet();
|
||||
let release!: () => void, created!: () => void;
|
||||
const barrier = new Promise<void>(resolve => { release = resolve; });
|
||||
const inserted = new Promise<void>(resolve => { created = resolve; });
|
||||
let id = "";
|
||||
const transaction = db.transaction(async tx => {
|
||||
id = (await create(tx as unknown as Db)).id; created(); await barrier;
|
||||
});
|
||||
await inserted;
|
||||
await vi.advanceTimersByTimeAsync(10_000); expect(provider).not.toHaveBeenCalled();
|
||||
expect(await stored(id)).toBeUndefined(); release(); await transaction;
|
||||
await waitFor(async () => expect((await stored(id)).status).toBe("accepted")); await quiet();
|
||||
const prior = provider.mock.calls.length;
|
||||
await expect(db.transaction(async tx => { await create(tx as unknown as Db, 41); throw new Error("rollback"); })).rejects.toThrow("rollback");
|
||||
await quiet(); expect(provider).toHaveBeenCalledTimes(prior); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("retains provider failures across drain, resumes without overlap, and clears retry after success", async () => {
|
||||
provider.mockRejectedValue(new Error("GitHub unavailable")); await start(); const card = await create();
|
||||
await waitFor(() => expect(errors).toHaveLength(1)); await settle();
|
||||
expect((await stored(card.id)).status).toBe("pending");
|
||||
draining = true;
|
||||
const reads = vi.spyOn(db, "select"), calls = provider.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(provider).toHaveBeenCalledTimes(calls); reads.mockRestore();
|
||||
draining = false; provider.mockResolvedValue("merged");
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps first work committed during drain and startup during standby until resume", async () => {
|
||||
standby = true; await start(); const card = await create();
|
||||
await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled();
|
||||
standby = false; draining = true;
|
||||
await vi.advanceTimersByTimeAsync(60_000); expect(provider).not.toHaveBeenCalled();
|
||||
draining = false;
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
});
|
||||
|
||||
it("reconstructs pending cards on restart and ignores terminal or hidden tasks", async () => {
|
||||
provider.mockResolvedValue("open"); const card = await create(); await start();
|
||||
await waitFor(() => expect(provider).toHaveBeenCalledTimes(1)); await settle();
|
||||
await coordinator!.stop(); coordinator = undefined; provider.mockResolvedValue("merged");
|
||||
await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
const hidden = await db.transaction(async tx => {
|
||||
const card = await create(tx as unknown as Db, 41);
|
||||
await issueService(tx as unknown as Db).update(issueId, { hiddenAt: new Date() });
|
||||
return card;
|
||||
});
|
||||
await quiet(); expect((await stored(hidden.id)).status).toBe("pending");
|
||||
await issueService(db).update(issueId, { hiddenAt: null });
|
||||
await waitFor(async () => expect((await stored(hidden.id)).status).toBe("accepted")); await quiet();
|
||||
});
|
||||
|
||||
it("preserves a real chat binding's durable continuation outbox on automatic acceptance", async () => {
|
||||
const applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID();
|
||||
await db.insert(toolApplications).values({ id: applicationId, companyId, applicationKey: `chat:slack:${endpointId}`, name: "Slack", type: "chat", status: "active" });
|
||||
await db.insert(toolConnections).values({ id: connectionId, companyId, applicationId, name: "Slack", uid: endpointId,
|
||||
connectionPurpose: "channel", transport: "chat_sdk", status: "active", enabled: true });
|
||||
await db.insert(chatEndpoints).values({ id: endpointId, companyId, connectionId, provider: "slack", publicId: endpointId,
|
||||
assignedAgentId: agentId, status: "active" });
|
||||
const [conversation] = await db.insert(chatConversations).values({ companyId, endpointId, issueId,
|
||||
externalConversationId: "merge-channel", externalThreadId: "merge-thread", externalLabel: "Merge thread", state: "active" }).returning();
|
||||
await start(); await quiet(); const card = await create();
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(await db.select().from(chatActions).where(eq(chatActions.kind, "interaction_wakeup"))).toEqual([
|
||||
expect.objectContaining({ companyId, endpointId, conversationId: conversation.id, providerActionId: `interaction_wakeup:${card.id}`,
|
||||
payload: expect.objectContaining({ interactionId: card.id, issueId, agentId, interactionStatus: "accepted" }) }),
|
||||
]);
|
||||
expect(await db.select().from(chatPublications)).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ idempotencyKey: `interaction:${card.id}:${endpointId}`, state: "cancelled" }),
|
||||
]));
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not lose a newer card created during provider observation or wake the superseded card", async () => {
|
||||
let release!: () => void, observed!: () => void;
|
||||
const barrier = new Promise<void>(resolve => { release = resolve; });
|
||||
const observing = new Promise<void>(resolve => { observed = resolve; });
|
||||
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
|
||||
await start(); const old = await create();
|
||||
await vi.advanceTimersByTimeAsync(1000); await observing;
|
||||
const current = await create(db, 41);
|
||||
expect((await stored(old.id)).status).toBe("expired");
|
||||
release();
|
||||
await waitFor(async () => expect((await stored(current.id)).status).toBe("accepted")); await quiet();
|
||||
expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
expect(wakeup).toHaveBeenCalledWith(agentId, expect.objectContaining({ idempotencyKey: `interaction:${current.id}:accepted` }));
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps unknown provider results as retryable debt and does not act before retry", async () => {
|
||||
provider.mockResolvedValue("unknown"); await start(); const card = await create();
|
||||
await waitFor(() => expect(errors).toHaveLength(1));
|
||||
const next = coordinator!.nextWakeAt()!;
|
||||
await vi.advanceTimersByTimeAsync(next - Date.now() - 1); expect(provider).toHaveBeenCalledTimes(1);
|
||||
provider.mockResolvedValue("merged"); await vi.advanceTimersByTimeAsync(1);
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(provider).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("honors terminal issues, pure merge URL targets and continuationPolicy none", async () => {
|
||||
await start(); await quiet();
|
||||
const target = await service().create({ id: issueId, companyId }, { ...input(), title: "Merge the linked PR?", continuationPolicy: "none",
|
||||
payload: { version: 1, prompt: "Merge the PR?", target: { type: "custom", key: "github-pr-39", href: "https://github.com/paperclipai/paperclip/pull/39" } } }, { agentId });
|
||||
await waitFor(async () => expect((await stored(target.id)).status).toBe("accepted")); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
provider.mockClear();
|
||||
const terminal = await db.transaction(async tx => {
|
||||
const card = await create(tx as unknown as Db, 41);
|
||||
await issueService(tx as unknown as Db).update(issueId, { status: "cancelled" });
|
||||
return card;
|
||||
});
|
||||
await quiet(); expect((await stored(terminal.id)).status).toBe("expired");
|
||||
expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rechecks hidden state under the acceptance lock when visibility changes during a provider read", async () => {
|
||||
let release!: () => void, observed!: () => void;
|
||||
const barrier = new Promise<void>(resolve => { release = resolve; });
|
||||
const observing = new Promise<void>(resolve => { observed = resolve; });
|
||||
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
|
||||
await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing;
|
||||
await issueService(db).update(issueId, { hiddenAt: new Date() }); release(); await quiet();
|
||||
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
|
||||
await issueService(db).update(issueId, { hiddenAt: null });
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("unrelated pending confirmations do not retain worker SQL or provider work", async () => {
|
||||
await start(); await quiet();
|
||||
await service().create({ id: issueId, companyId }, { ...input(), title: "Approve the plan?", payload: { version: 1, prompt: "Approve the plan?" } }, { agentId });
|
||||
await quiet(); expect(provider).not.toHaveBeenCalled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("awaits an in-flight observation at shutdown and reconstructs its unaccepted card on restart", async () => {
|
||||
let release!: () => void, observed!: () => void;
|
||||
const barrier = new Promise<void>(resolve => { release = resolve; });
|
||||
const observing = new Promise<void>(resolve => { observed = resolve; });
|
||||
provider.mockImplementationOnce(async () => { observed(); await barrier; return "merged"; });
|
||||
await start(); const card = await create(); await vi.advanceTimersByTimeAsync(1000); await observing;
|
||||
let stopped = false;
|
||||
const stopping = coordinator!.stop().then(() => { stopped = true; });
|
||||
await delay(10); expect(stopped).toBe(false); release(); await stopping; coordinator = undefined;
|
||||
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
|
||||
const reads = vi.spyOn(db, "select");
|
||||
await vi.advanceTimersByTimeAsync(2 * MERGE_CONFIRMATION_RECONCILE_MS); expect(reads).not.toHaveBeenCalled(); reads.mockRestore();
|
||||
await start(); await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["headers", "body", "not-found probe", "public visibility probe"] as const)("cancels an actual unresponsive HTTP %s read promptly at stop", async phase => {
|
||||
vi.useRealTimers();
|
||||
const card = await create();
|
||||
let observed!: () => void, closed!: () => void;
|
||||
const requested = new Promise<void>(resolve => { observed = resolve; });
|
||||
const disconnected = new Promise<void>(resolve => { closed = resolve; });
|
||||
const signals: AbortSignal[] = [];
|
||||
const server = createServer((request, response) => {
|
||||
if (phase === "not-found probe" && request.url?.includes("/pulls/")) { response.writeHead(404).end(); return; }
|
||||
response.on("close", closed);
|
||||
if (phase === "body") { response.writeHead(200, { "content-type": "application/json" }); response.write('{"merged":'); }
|
||||
observed();
|
||||
});
|
||||
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("Expected test HTTP address");
|
||||
const resolveState = createPullRequestMergeStateResolver(db, {
|
||||
requestTimeoutMs: 10_000,
|
||||
credentialProvider: () => phase === "public visibility probe" ? { token: "fixture", source: "server_env", secretName: null } : null,
|
||||
fetch: (url, init) => { signals.push(init!.signal!); return fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init); },
|
||||
});
|
||||
const real = issueThreadInteractionService(db, { resolvePullRequestState: resolveState, wakeup });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
try {
|
||||
const startupAt = performance.now();
|
||||
const worker = registerMergeConfirmationWork(coordinator, real, () => true);
|
||||
expect(performance.now() - startupAt).toBeLessThan(100);
|
||||
await Promise.race([requested, delay(2000).then(() => { throw new Error("HTTP observation did not start"); })]);
|
||||
const stoppingAt = performance.now();
|
||||
await coordinator.stop(); coordinator = undefined;
|
||||
expect(performance.now() - stoppingAt).toBeLessThan(1000);
|
||||
await worker.ready;
|
||||
await Promise.race([disconnected, delay(1000).then(() => { throw new Error("HTTP request remained open after stop"); })]);
|
||||
expect(signals.at(-1)?.aborted).toBe(true);
|
||||
expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]);
|
||||
} finally {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
|
||||
}
|
||||
});
|
||||
|
||||
it("times out a real unresponsive HTTP read and recovers its durable candidate on restart", async () => {
|
||||
vi.useRealTimers(); const card = await create();
|
||||
let serveMerge = false, closed!: () => void;
|
||||
const disconnected = new Promise<void>(resolve => { closed = resolve; });
|
||||
const server = createServer((_request, response) => {
|
||||
if (serveMerge) { response.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ merged: true, state: "closed" })); return; }
|
||||
response.on("close", closed);
|
||||
});
|
||||
await new Promise<void>(resolve => { server.listen(0, "127.0.0.1", resolve); });
|
||||
const address = server.address(); if (!address || typeof address === "string") throw new Error("Expected test HTTP address");
|
||||
const real = issueThreadInteractionService(db, { resolvePullRequestState: createPullRequestMergeStateResolver(db, {
|
||||
requestTimeoutMs: 100, tokenProvider: () => null,
|
||||
fetch: (url, init) => fetch(`http://127.0.0.1:${address.port}${new URL(url).pathname}`, init),
|
||||
}), wakeup });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
try {
|
||||
const startedAt = performance.now();
|
||||
await registerMergeConfirmationWork(coordinator, real, () => true).ready;
|
||||
expect(performance.now() - startedAt).toBeLessThan(2000);
|
||||
await Promise.race([disconnected, delay(1000).then(() => { throw new Error("Timed out HTTP request remained open"); })]);
|
||||
expect(errors).toHaveLength(1); expect((await stored(card.id)).status).toBe("pending"); expect(wakeup).not.toHaveBeenCalled();
|
||||
expect(coordinator.nextWakeAt()).not.toBeNull();
|
||||
await coordinator.stop(); coordinator = undefined; serveMerge = true;
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => errors.push(error) });
|
||||
await registerMergeConfirmationWork(coordinator, real, () => true).ready;
|
||||
expect((await stored(card.id)).status).toBe("accepted"); expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
server.closeAllConnections(); await new Promise<void>(resolve => { server.close(() => resolve()); });
|
||||
}
|
||||
});
|
||||
|
||||
it("pages beyond a full pending-card page without duplicating the merge parser in SQL", async () => {
|
||||
const card = await create();
|
||||
const id = BigInt(`0x${card.id.replaceAll("-", "")}`);
|
||||
await db.insert(issueThreadInteractions).values(Array.from({ length: 100 }, (_, index) => {
|
||||
const hex = (id - BigInt(index + 1)).toString(16).padStart(32, "0");
|
||||
return { id: `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20)}`,
|
||||
companyId, issueId, kind: "request_confirmation", title: "Approve the plan?", payload: { version: 1 as const, prompt: "Approve the plan?" } };
|
||||
}));
|
||||
await start(); expect(provider).not.toHaveBeenCalled();
|
||||
await waitFor(async () => expect((await stored(card.id)).status).toBe("accepted")); await quiet();
|
||||
expect(provider).toHaveBeenCalledTimes(1); expect(wakeup).toHaveBeenCalledTimes(1); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("limits normalized merge hints to the owning company and matching PR and tolerates duplicates", async () => {
|
||||
provider.mockResolvedValue("open"); const own = await create();
|
||||
const foreignCompany = randomUUID(), foreignIssue = randomUUID();
|
||||
await db.insert(companies).values({ id: foreignCompany, name: "Other company", issuePrefix: "OTH" });
|
||||
await db.insert(issues).values({ id: foreignIssue, companyId: foreignCompany, title: "Other", status: "todo" });
|
||||
const foreign = await service().create({ id: foreignIssue, companyId: foreignCompany }, input(), { userId: "board" });
|
||||
const local = service();
|
||||
const hint = [{ companyId, owner: "PAPERCLIPAI", repo: "paperclip", number: 39 }];
|
||||
expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(1);
|
||||
expect((await stored(own.id)).status).toBe("accepted"); expect((await stored(foreign.id)).status).toBe("pending");
|
||||
expect(provider).not.toHaveBeenCalled();
|
||||
expect((await local.sweepMergedPullRequestConfirmations(hint)).accepted).toBe(0); expect(wakeup).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,12 @@
|
||||
import { notifyNativeEnvironmentSettlement } from "../services/native-environment-settlement-notifications.js";
|
||||
import { notifyRetainedNativeSessionCleanup, onRetainedNativeSessionCleanupRelease, releaseRetainedNativeSessionCleanup } from "../services/retained-native-session-cleanup-notifications.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { publishNativeHarnessBackup } from "../services/native-runtime/native-harness-backup-publication.js";
|
||||
import { registerNativeHarnessBackupExpirationWork } from "../services/native-runtime/native-harness-backup-expiration-work.js";
|
||||
import { noteNativeHarnessBackupChange } from "../services/native-harness-backup-expiration-notifications.js";
|
||||
import { writeEnvironmentCleanup } from "../services/environment-cleanup-notifications.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { applyPendingMigrations } from "@paperclipai/db";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { registerNativeRunnerEvidenceRetentionWork } from "../services/native-runtime/native-runner-evidence-retention-work.js";
|
||||
@@ -8,7 +17,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
agents,
|
||||
agentTaskSessions,
|
||||
@@ -55,7 +64,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let worker: ReturnType<typeof registerNativeRunnerEvidenceRetentionWork> | undefined;
|
||||
let admitted = true;
|
||||
let backupWorker: ReturnType<typeof registerNativeHarnessBackupExpirationWork> | undefined;
|
||||
const backupErrors: unknown[] = [];
|
||||
afterEach(async () => {
|
||||
await backupWorker?.stop(); backupWorker = undefined; backupErrors.length = 0;
|
||||
worker?.unsubscribe(); await coordinator?.stop(); coordinator = undefined; worker = undefined; admitted = true;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.delete(agentTaskSessions);
|
||||
@@ -75,6 +87,7 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
});
|
||||
|
||||
async function seedSession(input: {
|
||||
companyId?: string; agentId?: string;
|
||||
issueStatus?: string;
|
||||
closedDaysAgo?: number;
|
||||
runStatus?: string;
|
||||
@@ -84,9 +97,10 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
cleanupActive?: boolean;
|
||||
} = {}) {
|
||||
if (!stateBase) stateBase = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-durable-sessions-"));
|
||||
const companyId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` });
|
||||
const [agent] = await db.insert(agents).values({ companyId, name: "Coder" }).returning();
|
||||
const companyId = input.companyId ?? randomUUID();
|
||||
if (!input.companyId) await db.insert(companies).values({ id: companyId, name: "Backups", issuePrefix: `B${companyId.slice(0, 6)}` });
|
||||
const [agent] = input.agentId ? [{ id: input.agentId }]
|
||||
: await db.insert(agents).values({ companyId, name: "Coder" }).returning();
|
||||
const issueStatus = input.issueStatus ?? "done";
|
||||
const closedAt = new Date(now.getTime() - (input.closedDaysAgo ?? 10) * DAY_MS);
|
||||
const [issue] = await db.insert(issues).values({
|
||||
@@ -558,6 +572,444 @@ describeEmbeddedPostgres("native harness backup expiration", () => {
|
||||
reads.mockRestore(); writes.mockRestore(); directories.mockRestore();
|
||||
}
|
||||
|
||||
async function startBackups(options: Parameters<typeof nativeHarnessBackupExpirationService>[1] = {}) {
|
||||
if (!stateBase) await emptyStateBase();
|
||||
if (!vi.isFakeTimers()) {
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
vi.setSystemTime(now);
|
||||
}
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => admitted, canReconcile: () => true,
|
||||
onError: error => backupErrors.push(error) });
|
||||
backupWorker = registerNativeHarnessBackupExpirationWork(coordinator, db,
|
||||
{ stateBase, isSessionBusy: () => false, ...options }, () => admitted);
|
||||
await backupWorker.ready;
|
||||
}
|
||||
|
||||
it.each([false, true])("backup deadline worker leaves empty and retained terminal history quiet (%s)", async history => {
|
||||
await emptyStateBase();
|
||||
if (history) {
|
||||
const session = await seedSession();
|
||||
await fs.rm(path.join(session.root, "failover-backups"), { recursive: true });
|
||||
await db.insert(heartbeatRuns).values(Array.from({ length: 500 }, () => ({
|
||||
companyId: session.companyId, agentId: session.agentId, status: "succeeded", issueId: session.issueId,
|
||||
nativeIssueId: session.issueId, nativeSessionId: randomUUID(), finishedAt: now,
|
||||
})));
|
||||
for (let index = 0; index < 250; index++) await fs.mkdir(path.join(stateBase,
|
||||
createHash("sha256").update(String(index)).digest("hex")));
|
||||
}
|
||||
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction");
|
||||
await startBackups({ maxEntries: 20 }); await settled();
|
||||
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled();
|
||||
reads.mockRestore(); writes.mockRestore(); await quiet();
|
||||
expect(backupErrors).toEqual([]);
|
||||
});
|
||||
|
||||
it("ordinary task close arms exact backup retention, deletes physical bytes and resume pointer, then stays quiet", async () => {
|
||||
const session = await seedSession({ issueStatus: "todo" });
|
||||
await startBackups(); await settled(); await quiet();
|
||||
const closed = await issueService(db).update(session.issueId, { status: "done", companyGuard: session.companyId });
|
||||
const deadline = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1;
|
||||
await settled(deadline); await quiet(60_000);
|
||||
await vi.advanceTimersByTimeAsync(deadline - Date.now() - 1);
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
expect(await taskSessionCount(session.agentId)).toBe(1);
|
||||
await vi.advanceTimersByTimeAsync(1); await settled();
|
||||
expect(await exists(session.current)).toBe(false);
|
||||
expect(await taskSessionCount(session.agentId)).toBe(0);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toHaveLength(1);
|
||||
expect(backupErrors).toEqual([]); await quiet();
|
||||
});
|
||||
|
||||
it("ordinary backup publication after quiet schedules its actual file age and leaves a durable cleanup result", async () => {
|
||||
const session = await seedSession();
|
||||
const backupRoot = path.dirname(session.current);
|
||||
const pendingRoot = path.join(backupRoot, "pending-publication");
|
||||
await fs.rename(session.current, pendingRoot);
|
||||
await startBackups(); await settled(); await quiet();
|
||||
const publishedAt = Date.now();
|
||||
await fs.utimes(pendingRoot, new Date(publishedAt), new Date(publishedAt));
|
||||
await publishNativeHarnessBackup({ db, companyId: session.companyId, scopeDigest: session.scope, backupRoot, pendingRoot });
|
||||
const stat = await fs.stat(session.current);
|
||||
const due = stat.mtime.getTime() + 7 * DAY_MS + 1;
|
||||
await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(session.current)).toBe(false); expect(await taskSessionCount(session.agentId)).toBe(0);
|
||||
expect(await fs.readdir(backupRoot)).toEqual([]); await quiet();
|
||||
});
|
||||
|
||||
it("a committed earlier close timestamp replaces the later deadline without early cleanup", async () => {
|
||||
const session = await seedSession({ closedDaysAgo: 2 });
|
||||
await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1);
|
||||
const earlier = new Date(now.getTime() - 6 * DAY_MS);
|
||||
await issueService(db).update(session.issueId, { completedAt: earlier });
|
||||
const due = earlier.getTime() + 7 * DAY_MS + 1;
|
||||
await settled(due); await quiet(60_000);
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(session.current)).toBe(false); await quiet();
|
||||
});
|
||||
|
||||
it("holds nested ordinary close activation until the outer transaction commits", async () => {
|
||||
const session = await seedSession({ issueStatus: "todo" });
|
||||
await startBackups(); await settled();
|
||||
let release!: () => void, written!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const staged = new Promise<void>(resolve => { written = resolve; });
|
||||
const transaction = db.transaction(async outer => {
|
||||
await outer.transaction(nested => issueService(db).update(session.issueId, { status: "done" }, nested));
|
||||
written(); await gate;
|
||||
});
|
||||
await staged;
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
expect(await taskSessionCount(session.agentId)).toBe(1);
|
||||
release(); await transaction;
|
||||
const [closed] = await db.select().from(issues).where(eq(issues.id, session.issueId));
|
||||
const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1;
|
||||
await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(session.current)).toBe(false); await quiet();
|
||||
});
|
||||
|
||||
it("preserves failed cleanup retry across drain, then finishes its physical tombstone on resume", async () => {
|
||||
const session = await seedSession(); const remove = fs.rm;
|
||||
const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => {
|
||||
if (String(args[0]).includes(".expired-")) throw new Error("controlled drain retry failure");
|
||||
return remove(...args);
|
||||
});
|
||||
await startBackups();
|
||||
expect(backupErrors).toHaveLength(1); admitted = false; fault.mockRestore();
|
||||
await quiet(60_000);
|
||||
expect((await fs.readdir(path.dirname(session.current))).some(name => name.startsWith(".expired-"))).toBe(true);
|
||||
admitted = true; await settled();
|
||||
expect(await fs.readdir(path.dirname(session.current))).toEqual([]); await quiet();
|
||||
});
|
||||
|
||||
it("reopen cancels a saved deadline and a later ordinary close restores retention", async () => {
|
||||
const session = await seedSession({ closedDaysAgo: 2 });
|
||||
await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1);
|
||||
await issueService(db).update(session.issueId, { status: "todo" });
|
||||
await settled(); await quiet(8 * DAY_MS); expect(await exists(session.current)).toBe(true);
|
||||
const closed = await issueService(db).update(session.issueId, { status: "done" });
|
||||
const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1;
|
||||
await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(session.current)).toBe(false); await quiet();
|
||||
});
|
||||
|
||||
it("reconstructs a retained backup deadline on restart without changing business timestamps", async () => {
|
||||
const session = await seedSession({ closedDaysAgo: 2 });
|
||||
const [before] = await db.select().from(issues).where(eq(issues.id, session.issueId));
|
||||
const due = now.getTime() + 5 * DAY_MS + 1;
|
||||
await startBackups(); await settled(due);
|
||||
await backupWorker!.stop(); await coordinator!.stop();
|
||||
await startBackups(); await settled(due);
|
||||
const [after] = await db.select().from(issues).where(eq(issues.id, session.issueId));
|
||||
expect(after!.updatedAt).toEqual(before!.updatedAt); expect(after!.completedAt).toEqual(before!.completedAt);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("reports dry-run backups once and rechecks only after source invalidation", async () => {
|
||||
const session = await seedSession(); const report = vi.spyOn(logger, "info");
|
||||
await startBackups({ dryRun: true }); await settled();
|
||||
expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(1);
|
||||
expect(await taskSessionCount(session.agentId)).toBe(1); expect(await exists(session.current)).toBe(true);
|
||||
await quiet();
|
||||
noteNativeHarnessBackupChange(db, { companyId: session.companyId, scopeDigest: session.scope });
|
||||
await settled();
|
||||
expect(report.mock.calls.filter(call => call[1] === "native harness backup expiration")).toHaveLength(2);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("company-only notifications inspect only the indexed company's blocked backup sources", async () => {
|
||||
const own = await seedSession({ issueStatus: "todo" });
|
||||
const other = await seedSession({ issueStatus: "todo" });
|
||||
await startBackups({ maxEntries: 1 }); await settled(); await quiet();
|
||||
const sql: Array<{ query: string; parameters: unknown[] }> = [];
|
||||
const previous = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, query, parameters) => { sql.push({ query, parameters }); };
|
||||
const directories = vi.spyOn(fs, "readdir");
|
||||
try {
|
||||
const closed = await issueService(db).update(own.issueId, { status: "done" });
|
||||
await settled(new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1);
|
||||
const unrelated = [other.companyId, other.agentId, other.runId, other.sessionId, other.issueId];
|
||||
expect(sql.filter(query => query.parameters.some(parameter =>
|
||||
unrelated.some(id => String(parameter).includes(id))))).toEqual([]);
|
||||
expect(directories.mock.calls.some(call => call[0] === stateBase
|
||||
|| String(call[0]).startsWith(other.root))).toBe(false);
|
||||
expect(await exists(other.current)).toBe(true);
|
||||
} finally { db.$client.options.debug = previous; directories.mockRestore(); }
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("coalesces repeated task writes to one retained root in a large company with the same historical agent", async () => {
|
||||
const own = await seedSession({ issueStatus: "todo" });
|
||||
const unrelated = [];
|
||||
for (let index = 0; index < 80; index++) unrelated.push(await seedSession({ companyId: own.companyId,
|
||||
agentId: own.agentId, issueStatus: index % 2 ? "todo" : "done", closedDaysAgo: 2,
|
||||
runStatus: index % 3 ? "succeeded" : "running" }));
|
||||
const due = now.getTime() + 5 * DAY_MS + 1;
|
||||
await startBackups({ isSessionBusy: digest => unrelated.some((entry, index) => index % 4 === 0 && entry.scope === digest) });
|
||||
await settled(due); await quiet(60_000);
|
||||
const reads = vi.spyOn(fs, "readFile");
|
||||
const queries: Array<{ query: string; parameters: unknown[] }> = [];
|
||||
const previous = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, query, parameters) => { queries.push({ query, parameters }); };
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await issueService(db).update(own.issueId, { status: "done" }, tx);
|
||||
for (let index = 0; index < 12; index++) await issueService(db).update(own.issueId,
|
||||
{ completedAt: new Date(now.getTime() - DAY_MS) }, tx);
|
||||
});
|
||||
const writes = queries.length;
|
||||
await settled(due);
|
||||
expect(reads.mock.calls.map(call => String(call[0]))).toEqual([path.join(own.root, "control-plane", "control-plane-state.json")]);
|
||||
expect(queries.length - writes).toBeLessThanOrEqual(20);
|
||||
const otherIds = unrelated.flatMap(entry => [entry.issueId, entry.runId, entry.sessionId]);
|
||||
expect(queries.filter(query => query.parameters.some(parameter => otherIds.some(id => String(parameter).includes(id))))).toEqual([]);
|
||||
} finally { reads.mockRestore(); db.$client.options.debug = previous; }
|
||||
await issueService(db).update(own.issueId, { status: "todo" }); await settled(due);
|
||||
await quiet(); expect(await exists(own.current)).toBe(true);
|
||||
});
|
||||
|
||||
it("indexes protected session relations so a new run, historical task close and exact process release reactivate only its backup", async () => {
|
||||
const own = await seedSession({ closedDaysAgo: 2 });
|
||||
const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" });
|
||||
const [linked] = await db.insert(issues).values({ companyId: own.companyId, title: "New linked task", status: "todo" }).returning();
|
||||
const due = now.getTime() + 5 * DAY_MS + 1;
|
||||
await startBackups(); await settled(due); await quiet(60_000);
|
||||
const reads = vi.spyOn(fs, "readFile");
|
||||
const runId = randomUUID();
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await notifyNativeEnvironmentSettlement(tx, { companyId: own.companyId, runId, issueId: linked!.id });
|
||||
await tx.insert(heartbeatRuns).values({ id: runId, companyId: own.companyId, agentId: own.agentId,
|
||||
nativeSessionId: own.sessionId, nativeIssueId: linked!.id, issueId: other.issueId, status: "running" });
|
||||
});
|
||||
await settled();
|
||||
expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true);
|
||||
expect(reads).toHaveBeenCalled(); reads.mockClear();
|
||||
await issueService(db).update(linked!.id, { status: "done" }); await settled();
|
||||
expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true); reads.mockClear();
|
||||
await db.transaction(async tx => {
|
||||
await notifyRetainedNativeSessionCleanup(tx, { companyId: own.companyId, runId, agentId: own.agentId });
|
||||
await tx.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, runId));
|
||||
});
|
||||
const [closed] = await db.select().from(issues).where(eq(issues.id, linked!.id));
|
||||
const later = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1;
|
||||
await settled(later); reads.mockClear();
|
||||
const cleanupScope = { companyId: own.companyId, agentId: own.agentId };
|
||||
const observed = vi.fn(); const stop = onRetainedNativeSessionCleanupRelease(db, observed);
|
||||
releaseRetainedNativeSessionCleanup(db, cleanupScope, { companyId: own.companyId, runId });
|
||||
expect(observed).toHaveBeenCalledWith(cleanupScope, { companyId: own.companyId, runId }); stop();
|
||||
await settled(later);
|
||||
expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true);
|
||||
expect(reads).toHaveBeenCalled();
|
||||
await issueService(db).update(linked!.id, { status: "todo" }); await settled();
|
||||
} finally { reads.mockRestore(); }
|
||||
expect(await exists(other.current)).toBe(true); await quiet();
|
||||
});
|
||||
|
||||
it("ordinary lease release without a run uses its source identity instead of other retained roots", async () => {
|
||||
const own = await seedSession({ leaseStatus: "retained" });
|
||||
const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" });
|
||||
await startBackups(); await settled(); await quiet(60_000);
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId));
|
||||
const reads = vi.spyOn(fs, "readFile");
|
||||
try {
|
||||
await environmentService(db).releaseLease(lease!.id, "expired"); await settled();
|
||||
expect(reads).toHaveBeenCalled();
|
||||
expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true);
|
||||
} finally { reads.mockRestore(); }
|
||||
expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet();
|
||||
});
|
||||
|
||||
it("retains the future deadline when a lease metadata edit removes its last backup reference", async () => {
|
||||
const own = await seedSession({ leaseStatus: "retained", closedDaysAgo: 2 });
|
||||
const other = await seedSession({ companyId: own.companyId, agentId: own.agentId, issueStatus: "todo" });
|
||||
await startBackups(); await settled();
|
||||
const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.companyId, own.companyId));
|
||||
const reads = vi.spyOn(fs, "readFile"); const due = now.getTime() + 5 * DAY_MS + 1;
|
||||
try {
|
||||
await environmentService(db).updateLeaseMetadata(lease!.id, {}); await settled(due);
|
||||
expect(reads).toHaveBeenCalled(); expect(reads.mock.calls.every(call => String(call[0]).startsWith(own.root))).toBe(true);
|
||||
} finally { reads.mockRestore(); }
|
||||
await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(own.current)).toBe(false); expect(await exists(other.current)).toBe(true); await quiet();
|
||||
});
|
||||
|
||||
it("allows an ordinary task reopen while backup byte counting is held outside the retirement lock", async () => {
|
||||
const own = await seedSession(); const lstat = fs.lstat;
|
||||
const entered = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const held = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => {
|
||||
if (String(args[0]) === path.join(own.current, "codex-home", "history.jsonl")) {
|
||||
entered.resolve(); await release.promise;
|
||||
}
|
||||
return lstat(...args);
|
||||
});
|
||||
const startup = startBackups(); await entered.promise;
|
||||
try {
|
||||
await db.transaction(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '500ms'`);
|
||||
await issueService(db).update(own.issueId, { status: "todo" }, tx);
|
||||
});
|
||||
} finally { release.resolve(); await startup; held.mockRestore(); }
|
||||
await settled(); expect(await exists(own.current)).toBe(true); expect(backupErrors).toEqual([]); await quiet();
|
||||
});
|
||||
|
||||
it.each(["EIO", "EACCES"])("retains a failed identity read as real retry debt (%s)", async code => {
|
||||
const own = await seedSession(); const read = fs.readFile;
|
||||
const fault = vi.spyOn(fs, "readFile").mockImplementation(async (...args) => {
|
||||
if (String(args[0]) === path.join(own.root, "control-plane", "control-plane-state.json")) {
|
||||
throw Object.assign(new Error("controlled identity I/O failure"), { code });
|
||||
}
|
||||
return read(...args);
|
||||
});
|
||||
await startBackups(); expect(backupErrors).toHaveLength(1);
|
||||
const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThanOrEqual(29_000);
|
||||
expect(await exists(own.current)).toBe(true); fault.mockRestore(); await quiet(5000);
|
||||
await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled();
|
||||
expect(await exists(own.current)).toBe(false); expect(await taskSessionCount(own.agentId)).toBe(0); await quiet();
|
||||
});
|
||||
|
||||
it.each(["EIO", "EACCES"])("retains expired-only cleanup after a directory read failure (%s)", async code => {
|
||||
const session = await seedSession();
|
||||
const backupRoot = path.dirname(session.current);
|
||||
const expired = path.join(backupRoot, `.expired-${randomUUID()}`);
|
||||
await fs.rename(session.current, expired);
|
||||
const read = fs.readdir;
|
||||
const fault = vi.spyOn(fs, "readdir").mockImplementation(async (...args) => {
|
||||
if (String(args[0]) === backupRoot) throw Object.assign(new Error("controlled cleanup listing failure"), { code });
|
||||
return read(...args);
|
||||
});
|
||||
await startBackups();
|
||||
expect(backupErrors).toHaveLength(1);
|
||||
const retry = coordinator!.nextWakeAt();
|
||||
expect(retry).not.toBeNull();
|
||||
expect(retry! - Date.now()).toBeGreaterThanOrEqual(29_000);
|
||||
expect(await exists(session.current)).toBe(false);
|
||||
expect(await exists(expired)).toBe(true);
|
||||
fault.mockRestore();
|
||||
await quiet(5000);
|
||||
await vi.advanceTimersByTimeAsync(retry! - Date.now()); await settled();
|
||||
expect(await fs.readdir(backupRoot)).toEqual([]);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("reconstructs future retention without issue row locks", async () => {
|
||||
await seedSession({ closedDaysAgo: 2 }); const queries: string[] = [];
|
||||
const previous = db.$client.options.debug;
|
||||
db.$client.options.debug = (_connection, query) => { queries.push(query); };
|
||||
try {
|
||||
await startBackups(); await settled(now.getTime() + 5 * DAY_MS + 1);
|
||||
expect(queries.some(query => /for\s+share/i.test(query))).toBe(false);
|
||||
} finally { db.$client.options.debug = previous; }
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("rejects destructive backup scopes outside the controller digest namespace", async () => {
|
||||
await emptyStateBase(); const remove = vi.spyOn(fs, "rm");
|
||||
await expect(service().expireBackup("../outside")).rejects.toThrow("Invalid native harness backup scope");
|
||||
expect(remove).not.toHaveBeenCalled(); remove.mockRestore();
|
||||
});
|
||||
|
||||
it.each([false, true])("finishes cleanup journals after ordinary agent deletion (restart %s)", async restart => {
|
||||
const session = await seedSession(); const remove = fs.rm;
|
||||
const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => {
|
||||
if (String(args[0]).includes(".expired-")) throw new Error("controlled orphan cleanup failure");
|
||||
return remove(...args);
|
||||
});
|
||||
await startBackups();
|
||||
expect(backupErrors).toHaveLength(1);
|
||||
const retryAt = coordinator!.nextWakeAt()!;
|
||||
const backupRoot = path.dirname(session.current);
|
||||
expect((await fs.readdir(backupRoot)).some(name => name.startsWith(".expired-"))).toBe(true);
|
||||
if (restart) { await backupWorker!.stop(); await coordinator!.stop(); }
|
||||
await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId));
|
||||
await agentService(db).remove(session.agentId);
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, session.runId))).toEqual([]);
|
||||
fault.mockRestore();
|
||||
if (restart) await startBackups(); else await vi.advanceTimersByTimeAsync(retryAt - Date.now());
|
||||
await settled();
|
||||
expect(await fs.readdir(backupRoot)).toEqual([]); expect(await taskSessionCount(session.agentId)).toBe(0);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("restores an orphan retiring journal without expiring the recovered backup or unrelated names", async () => {
|
||||
const session = await seedSession(); const backupRoot = path.dirname(session.current);
|
||||
const retiring = path.join(backupRoot, `.retiring-${randomUUID()}`);
|
||||
await fs.rename(session.current, retiring);
|
||||
await fs.mkdir(path.join(backupRoot, ".expired-unrecognized"));
|
||||
await db.update(agents).set({ status: "terminated", lifecycleState: "terminated" }).where(eq(agents.id, session.agentId));
|
||||
await agentService(db).remove(session.agentId);
|
||||
await fs.rm(path.join(session.root, "control-plane"), { recursive: true });
|
||||
await startBackups(); await settled();
|
||||
expect(await exists(session.current)).toBe(true); expect(await exists(retiring)).toBe(false);
|
||||
expect(await exists(path.join(session.current, "manifest.json"))).toBe(true);
|
||||
expect(await exists(path.join(backupRoot, ".expired-unrecognized"))).toBe(true);
|
||||
await quiet();
|
||||
});
|
||||
|
||||
it("retains live lease references and only removes the owning company's backup after lease release", async () => {
|
||||
const first = await seedSession({ leaseStatus: "retained" });
|
||||
const other = await seedSession({ leaseStatus: "retained" });
|
||||
await startBackups(); await settled(); await quiet();
|
||||
await writeEnvironmentCleanup(db, tx => tx.update(environmentLeases).set({ status: "expired" })
|
||||
.where(eq(environmentLeases.companyId, first.companyId)), { companyId: first.companyId });
|
||||
await settled(); expect(await exists(first.current)).toBe(false); expect(await exists(other.current)).toBe(true);
|
||||
expect(await taskSessionCount(other.agentId)).toBe(1); await quiet();
|
||||
});
|
||||
|
||||
it("retries a partial physical deletion from its tombstone without recreating a resume pointer", async () => {
|
||||
const session = await seedSession(); const remove = fs.rm; let failed = false;
|
||||
const fault = vi.spyOn(fs, "rm").mockImplementation(async (...args) => {
|
||||
if (!failed && String(args[0]).includes(".expired-")) { failed = true; throw new Error("controlled filesystem delete failure"); }
|
||||
return remove(...args);
|
||||
});
|
||||
await startBackups();
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(backupErrors).toHaveLength(1); });
|
||||
const retry = coordinator!.nextWakeAt()!; expect(retry - Date.now()).toBeGreaterThan(20_000);
|
||||
expect(await taskSessionCount(session.agentId)).toBe(0); expect(await exists(session.current)).toBe(false);
|
||||
fault.mockRestore(); await quiet(5000);
|
||||
await vi.advanceTimersByTimeAsync(retry - Date.now()); await settled();
|
||||
expect(await fs.readdir(path.join(session.root, "failover-backups"))).toEqual([]); await quiet();
|
||||
});
|
||||
|
||||
it("retains a first ordinary close during drain and dispatches it after resume", async () => {
|
||||
const session = await seedSession({ issueStatus: "todo" });
|
||||
await startBackups(); await settled(); admitted = false;
|
||||
const closed = await issueService(db).update(session.issueId, { status: "done" });
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect(idleWorkSnapshot().active).toBe(0); });
|
||||
await quiet(60_000); admitted = true;
|
||||
const due = new Date(closed!.completedAt!).getTime() + 7 * DAY_MS + 1;
|
||||
await settled(due); await vi.advanceTimersByTimeAsync(due - Date.now()); await settled();
|
||||
expect(await exists(session.current)).toBe(false); await quiet();
|
||||
});
|
||||
|
||||
it("honors outer rollback for nested ordinary close and activates only on committed work", async () => {
|
||||
const session = await seedSession({ issueStatus: "todo" });
|
||||
await startBackups(); await settled();
|
||||
await expect(db.transaction(async outer => {
|
||||
await outer.transaction(async nested => {
|
||||
await issueService(db).update(session.issueId, { status: "done" }, nested);
|
||||
});
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
throw new Error("rollback outer close");
|
||||
})).rejects.toThrow("rollback outer close");
|
||||
await settled(); await quiet(8 * DAY_MS);
|
||||
expect(await exists(session.current)).toBe(true);
|
||||
const [saved] = await db.select().from(issues).where(eq(issues.id, session.issueId));
|
||||
expect(saved!.status).toBe("todo");
|
||||
});
|
||||
|
||||
it("pages multiple company backups, preserves a busy one and wakes from process release", async () => {
|
||||
const sessions = [await seedSession(), await seedSession(), await seedSession()]; let busy = true;
|
||||
await startBackups({ maxEntries: 1, maxRemovals: 1, isSessionBusy: digest => busy && digest === sessions[1]!.scope });
|
||||
await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(await exists(sessions[0]!.current)).toBe(false); expect(await exists(sessions[2]!.current)).toBe(false);
|
||||
expect(backupErrors.length).toBeGreaterThan(0);
|
||||
});
|
||||
expect(await exists(sessions[1]!.current)).toBe(true); busy = false;
|
||||
noteNativeHarnessBackupChange(db, { companyId: sessions[1]!.companyId });
|
||||
await settled(); expect(await exists(sessions[1]!.current)).toBe(false); await quiet();
|
||||
});
|
||||
|
||||
it("leaves empty evidence and retained backups quiet without invoking backup SQL", async () => {
|
||||
const session = await seedSession();
|
||||
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction");
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
import { writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, createDb, heartbeatRuns, issues, pluginManagedResources, plugins, routineRuns, type Db } from "@paperclipai/db";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { notifyRoutineExecutionStatus } from "../services/routine-execution-status-notifications.js";
|
||||
import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js";
|
||||
import { routineExecutionStatusService } from "../services/routine-run-status.js";
|
||||
import { routineService } from "../services/routines.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js";
|
||||
import { createHeartbeatRunState } from "../services/heartbeat/run-state.js";
|
||||
import { createHeartbeatRunPreparation } from "../services/heartbeat/run-preparation.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_WORK_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("routine execution status work with PostgreSQL", () => {
|
||||
let db: Db, temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let enabled = true, draining = false, standby = false;
|
||||
const errors: unknown[] = [];
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("routine-status-work-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6 });
|
||||
}, 60_000);
|
||||
beforeEach(() => {
|
||||
enabled = true; draining = false; standby = false; errors.length = 0;
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end(); await temporary?.cleanup(); });
|
||||
const canRun = () => enabled && !draining && !standby;
|
||||
async function start(service = routineExecutionStatusService(db)) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun, canReconcile: () => !standby,
|
||||
onError: error => errors.push(error) });
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, canRun);
|
||||
await worker.ready;
|
||||
return worker;
|
||||
}
|
||||
async function settled() { await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(0)); }
|
||||
async function quiet() {
|
||||
await settled();
|
||||
const select = vi.spyOn(db, "select"), execute = vi.spyOn(db, "execute"), transaction = vi.spyOn(db, "transaction");
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(select).not.toHaveBeenCalled(); expect(execute).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
|
||||
} finally { select.mockRestore(); execute.mockRestore(); transaction.mockRestore(); }
|
||||
}
|
||||
async function fixture() {
|
||||
const companyId = randomUUID(), agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Routine status work", issuePrefix: `R${companyId.slice(0, 6)}`,
|
||||
defaultResponsibleUserId: randomUUID(), requireBoardApprovalForNewAgents: false });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Routine worker", status: "active", lifecycleState: "ready", adapterType: "codex_local" });
|
||||
const service = routineService(db, { heartbeat: { wakeup: async () => null }, runtimeEnv: {} });
|
||||
const routine = await service.create(companyId, { title: "Status projection", assigneeAgentId: agentId,
|
||||
priority: "medium", status: "active", concurrencyPolicy: "always_enqueue", catchUpPolicy: "skip_missed" }, {});
|
||||
return { companyId, agentId, routine, service };
|
||||
}
|
||||
async function dispatch(f: Awaited<ReturnType<typeof fixture>>) {
|
||||
return f.service.runRoutine(f.routine.id, { source: "manual", idempotencyKey: randomUUID() });
|
||||
}
|
||||
async function saved(id: string) { return (await db.select().from(routineRuns).where(eq(routineRuns.id, id)))[0]; }
|
||||
async function projected(id: string, status: string) {
|
||||
await vi.waitFor(async () => expect((await saved(id)).status).toBe(status));
|
||||
await settled();
|
||||
}
|
||||
|
||||
it("settles empty startup, configured idle, healthy active and retained terminal history without recurring SQL", async () => {
|
||||
await start(); await quiet();
|
||||
const f = await fixture(); await quiet();
|
||||
const run = await dispatch(f); await quiet();
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "done" });
|
||||
await projected(run.id, "completed");
|
||||
const before = await saved(run.id); await quiet();
|
||||
expect(await saved(run.id)).toEqual(before); expect(errors).toEqual([]);
|
||||
});
|
||||
it.each(["done", "blocked", "cancelled"])("reactivates from the first ordinary %s transition and preserves input", async status => {
|
||||
await start(); await quiet();
|
||||
const f = await fixture(), run = await dispatch(f);
|
||||
await issueService(db).update(run.linkedIssueId!, { status });
|
||||
await projected(run.id, status === "done" ? "completed" : "failed");
|
||||
expect((await saved(run.id)).completedAt).toBeInstanceOf(Date);
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
if (status !== "done") {
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "in_progress" });
|
||||
await projected(run.id, "issue_created");
|
||||
expect(await saved(run.id)).toMatchObject({ completedAt: null, failureReason: null });
|
||||
await quiet();
|
||||
}
|
||||
});
|
||||
it("reconstructs legacy terminal debt and reopened transient failures but leaves skipped and coalesced history unchanged", async () => {
|
||||
const f = await fixture(), first = await dispatch(f), second = await dispatch(f);
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, first.linkedIssueId!));
|
||||
await db.update(routineRuns).set({ status: "failed", failureReason: "Execution issue moved to blocked", completedAt: new Date() }).where(eq(routineRuns.id, second.id));
|
||||
const terminal = await db.insert(routineRuns).values(["skipped", "coalesced"].map(status => ({ companyId: f.companyId,
|
||||
routineId: f.routine.id, source: "manual", linkedIssueId: first.linkedIssueId, status, completedAt: new Date() }))).returning();
|
||||
await start(); await projected(first.id, "completed"); await projected(second.id, "issue_created");
|
||||
for (const row of terminal) expect(await saved(row.id)).toEqual(row);
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("settles the declared custom origin of a real plugin-managed dispatch and preserves other companies", async () => {
|
||||
const f = await fixture(), other = await fixture();
|
||||
const pluginKey = `routine-status-${randomUUID()}`, originId = `operation:${randomUUID()}`;
|
||||
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0",
|
||||
manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test",
|
||||
author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning();
|
||||
await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey,
|
||||
resourceKind: "routine", resourceKey: "status", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } });
|
||||
const run = await dispatch(f), otherRun = await dispatch(other);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId);
|
||||
await start(); await quiet();
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "done" }); await projected(run.id, "completed");
|
||||
const before = await saved(run.id);
|
||||
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); await notifyRoutineExecutionStatus(tx, run.linkedIssueId!); });
|
||||
await settled(); await quiet();
|
||||
expect(await saved(run.id)).toEqual(before); expect((await saved(otherRun.id)).status).toBe("issue_created"); expect(errors).toEqual([]);
|
||||
});
|
||||
it.each(["changed", "removed"])("keeps historical custom-origin custody after its plugin declaration is %s", async mode => {
|
||||
const f = await fixture(), pluginKey = `routine-history-${randomUUID()}`, originId = `operation:${randomUUID()}`;
|
||||
const [plugin] = await db.insert(plugins).values({ pluginKey, packageName: pluginKey, version: "0.1.0",
|
||||
manifestJson: { id: pluginKey, apiVersion: 1, version: "0.1.0", displayName: "Routine test", description: "Routine test",
|
||||
author: "Paperclip", categories: ["automation"], capabilities: ["routines.managed"], entrypoints: { worker: "./worker.js" } } }).returning();
|
||||
const [resource] = await db.insert(pluginManagedResources).values({ companyId: f.companyId, pluginId: plugin.id, pluginKey,
|
||||
resourceKind: "routine", resourceKey: "history", resourceId: f.routine.id, defaultsJson: { issueTemplate: { originId } } }).returning();
|
||||
const run = await dispatch(f);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, run.linkedIssueId!)))[0].originId).toBe(originId);
|
||||
if (mode === "changed") await db.update(pluginManagedResources).set({ defaultsJson: { issueTemplate: { originId: "new-origin" } } }).where(eq(pluginManagedResources.id, resource.id));
|
||||
else await db.delete(pluginManagedResources).where(eq(pluginManagedResources.id, resource.id));
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!));
|
||||
await start(); await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("reconstructs more than one page and does not lose a committed issue hint behind the startup cursor", async () => {
|
||||
const f = await fixture(), ids = Array.from({ length: 53 }, () => randomUUID()).sort();
|
||||
const issueIds = ids.map(() => randomUUID());
|
||||
await db.insert(issues).values(ids.map((id, index) => ({ id: issueIds[index], companyId: f.companyId, title: "Legacy execution",
|
||||
status: index === 0 ? "todo" : "done", originKind: "routine_execution", originRunId: id, originId: f.routine.id })));
|
||||
await db.insert(routineRuns).values(ids.map((id, index) => ({ id, companyId: f.companyId, routineId: f.routine.id,
|
||||
source: "schedule", linkedIssueId: issueIds[index], status: "issue_created" })));
|
||||
const service = routineExecutionStatusService(db), load = service.load;
|
||||
let entered!: () => void, release!: () => void;
|
||||
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
|
||||
const releaseGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const loaded = vi.spyOn(service, "load").mockImplementationOnce(async (scope, afterId) => {
|
||||
const page = await load(scope, afterId); expect(page).toHaveLength(50); entered(); await releaseGate; return page;
|
||||
});
|
||||
const starting = start(service); await enteredGate;
|
||||
await issueService(db).update(issueIds[0]!, { status: "done" }); release(); await starting;
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
await vi.waitFor(async () => expect((await db.select().from(routineRuns)).every(row => row.status === "completed")).toBe(true));
|
||||
expect(loaded.mock.calls.some(([scope, afterId]) => scope === undefined && afterId !== undefined)).toBe(true);
|
||||
expect(loaded.mock.calls.some(([scope]) => scope?.issueId === issueIds[0])).toBe(true);
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("scopes shared execution writes to their company and settles paused-company projection debt", async () => {
|
||||
const f = await fixture(), other = await fixture(), run = await dispatch(f), otherRun = await dispatch(other);
|
||||
const service = routineExecutionStatusService(db), load = vi.spyOn(service, "load");
|
||||
await start(service); await quiet(); load.mockClear();
|
||||
await db.update(companies).set({ status: "paused" }).where(eq(companies.id, f.companyId));
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, otherRun.linkedIssueId!));
|
||||
await writeExecutionControl(db, DELIVERY_QUEUES.executionStatus,
|
||||
tx => tx.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!)), { companyId: f.companyId });
|
||||
await projected(run.id, "completed");
|
||||
expect(load.mock.calls.every(([scope]) => scope?.companyId === f.companyId)).toBe(true);
|
||||
expect((await saved(otherRun.id)).status).toBe("issue_created"); await quiet();
|
||||
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, otherRun.linkedIssueId!); });
|
||||
await projected(otherRun.id, "completed"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("projects subtree cancel and restore without waking agents, then becomes quiet", async () => {
|
||||
const f = await fixture(), root = await dispatch(f), child = await dispatch(f);
|
||||
await issueService(db).update(child.linkedIssueId!, { parentId: root.linkedIssueId });
|
||||
const tree = issueTreeControlService(db), actor = { actorType: "system" as const, actorId: "routine-status-test" };
|
||||
await start(); await quiet();
|
||||
const cancel = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "cancel", actor });
|
||||
expect((await tree.cancelIssueStatusesForHold(f.companyId, root.linkedIssueId!, cancel.hold.id)).updatedIssueIds.sort())
|
||||
.toEqual([root.linkedIssueId, child.linkedIssueId].sort());
|
||||
await projected(root.id, "failed"); await projected(child.id, "failed"); await quiet();
|
||||
const restore = await tree.createHold(f.companyId, root.linkedIssueId!, { mode: "restore", actor });
|
||||
const restored = await tree.restoreIssueStatusesForHold(f.companyId, root.linkedIssueId!, restore.hold.id, { actor });
|
||||
expect(restored.updatedIssues).toEqual(expect.arrayContaining([
|
||||
expect.objectContaining({ id: root.linkedIssueId, status: "todo" }),
|
||||
expect.objectContaining({ id: child.linkedIssueId, status: "todo" }),
|
||||
]));
|
||||
await projected(root.id, "issue_created"); await projected(child.id, "issue_created");
|
||||
expect(await saved(root.id)).toMatchObject({ completedAt: null, failureReason: null });
|
||||
expect(await saved(child.id)).toMatchObject({ completedAt: null, failureReason: null });
|
||||
expect(await db.select().from(heartbeatRuns)).toEqual([]);
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("serializes a direct status hook with later task changes before projecting the routine", async () => {
|
||||
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, run.linkedIssueId!));
|
||||
let entered!: () => void, release!: () => void;
|
||||
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
|
||||
const releaseGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const holding = db.transaction(async tx => {
|
||||
await tx.select().from(routineRuns).where(eq(routineRuns.id, run.id)).for("update");
|
||||
entered(); await releaseGate;
|
||||
});
|
||||
let direct: Promise<unknown> | undefined, changing: Promise<unknown> | undefined;
|
||||
let directPid: number | undefined;
|
||||
try {
|
||||
await enteredGate; direct = f.service.syncRunStatusForIssue(run.linkedIssueId!);
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await db.execute(sql`select pid from pg_stat_activity
|
||||
where datname = current_database() and wait_event_type = 'Lock'
|
||||
and (query like 'update "routine_runs"%' or query like 'select %"routine_runs"%')`);
|
||||
expect(rows).toHaveLength(1); directPid = Number(rows[0]!.pid);
|
||||
}, { timeout: 5000 });
|
||||
await expect(db.transaction(tx => tx.select({ id: issues.id }).from(issues)
|
||||
.where(eq(issues.id, run.linkedIssueId!)).for("update", { noWait: true })))
|
||||
.rejects.toMatchObject({ cause: expect.objectContaining({ code: "55P03" }) });
|
||||
changing = issueService(db).update(run.linkedIssueId!, { status: "blocked" });
|
||||
await vi.waitFor(async () => {
|
||||
const rows = await db.execute(sql`select count(*)::int as count from pg_stat_activity
|
||||
where datname = current_database() and wait_event_type = 'Lock'
|
||||
and ${directPid}::int = any(pg_blocking_pids(pid))`);
|
||||
expect(Number(rows[0]?.count)).toBeGreaterThan(0);
|
||||
}, { timeout: 5000 });
|
||||
release(); await holding; await direct; await changing;
|
||||
await projected(run.id, "failed"); await quiet(); expect(errors).toEqual([]);
|
||||
} finally { release(); await holding; await direct; await changing; }
|
||||
});
|
||||
it("waits for the real outer transaction and forgets a rolled-back nested status write", async () => {
|
||||
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
|
||||
let release!: () => void, entered!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const written = new Promise<void>(resolve => { entered = resolve; });
|
||||
const writing = db.transaction(async tx => {
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "done" }, tx);
|
||||
entered(); await gate;
|
||||
});
|
||||
try { await written; expect((await saved(run.id)).status).toBe("issue_created"); }
|
||||
finally { release(); await writing; }
|
||||
await projected(run.id, "completed"); await quiet();
|
||||
const other = await dispatch(f);
|
||||
await expect(db.transaction(async tx => {
|
||||
await issueService(db).update(other.linkedIssueId!, { status: "cancelled" }, tx);
|
||||
throw new Error("outer rollback");
|
||||
})).rejects.toThrow("outer rollback");
|
||||
await settled(); expect((await saved(other.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("keeps the first write during drain and reconstructs on resume without passive SQL", async () => {
|
||||
const f = await fixture(), run = await dispatch(f); await start(); await quiet();
|
||||
draining = true;
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "cancelled" });
|
||||
await settled(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created");
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "failed");
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it.each(["standby", "disabled"])("does no owner SQL while %s and reconstructs when enabled", async mode => {
|
||||
const f = await fixture(), run = await dispatch(f);
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "done" });
|
||||
if (mode === "standby") standby = true; else enabled = false;
|
||||
await start(); await quiet(); expect((await saved(run.id)).status).toBe("issue_created");
|
||||
standby = false; enabled = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
await projected(run.id, "completed"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it("retries failed exact owners, retains the retry across drain and reconstructs after restart", async () => {
|
||||
const f = await fixture(), run = await dispatch(f), service = routineExecutionStatusService(db);
|
||||
const settle = vi.spyOn(service, "settle").mockRejectedValueOnce(new Error("projection unavailable"));
|
||||
await start(service);
|
||||
await issueService(db).update(run.linkedIssueId!, { status: "done" });
|
||||
await vi.waitFor(() => expect(errors).toHaveLength(1));
|
||||
expect((await saved(run.id)).status).toBe("issue_created");
|
||||
await vi.advanceTimersByTimeAsync(29_000); expect(settle).toHaveBeenCalledTimes(1);
|
||||
draining = true; await vi.advanceTimersByTimeAsync(60_000); await quiet();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await projected(run.id, "completed"); await quiet();
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
const other = await dispatch(f); await issueService(db).update(other.linkedIssueId!, { status: "blocked" });
|
||||
await start(); await projected(other.id, "failed"); await quiet();
|
||||
});
|
||||
it("preserves notifications during a sweep, coalesces duplicate hints and never overlaps", async () => {
|
||||
const f = await fixture(), first = await dispatch(f), second = await dispatch(f), service = routineExecutionStatusService(db);
|
||||
let entered!: () => void, release!: () => void;
|
||||
const enteredGate = new Promise<void>(resolve => { entered = resolve; });
|
||||
const releaseGate = new Promise<void>(resolve => { release = resolve; });
|
||||
const load = service.load;
|
||||
vi.spyOn(service, "load").mockImplementationOnce(async issueId => { const rows = await load(issueId); entered(); await releaseGate; return rows; });
|
||||
const starting = start(service); await enteredGate;
|
||||
await issueService(db).update(second.linkedIssueId!, { status: "done" });
|
||||
await db.transaction(async tx => { await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); await notifyRoutineExecutionStatus(tx, second.linkedIssueId!); });
|
||||
release(); await starting;
|
||||
await projected(second.id, "completed"); expect((await saved(first.id)).status).toBe("issue_created");
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
it.each(["succeeded", "failed", "cancelled", "interrupted", "timed_out"])("uses real %s run lifecycle changes for activeIssue without an open-issue retry", async status => {
|
||||
const f = await fixture(), run = await dispatch(f); await start();
|
||||
const [execution] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId,
|
||||
status: "running", contextSnapshot: { issueId: run.linkedIssueId } }).returning();
|
||||
await db.update(issues).set({ executionRunId: execution.id }).where(eq(issues.id, run.linkedIssueId!));
|
||||
expect((await f.service.getDetail(f.routine.id))?.activeIssue?.id).toBe(run.linkedIssueId);
|
||||
const deps = {
|
||||
...createHeartbeatRunState(db), ...createHeartbeatRunPreparation(db),
|
||||
issuesSvc: { addComment: vi.fn(), listReviewAttention: vi.fn() },
|
||||
recovery: { escalateStrandedAssignedIssue: vi.fn() }, companySkills: { completeTestRunForIssue: vi.fn() },
|
||||
budgets: { getInvocationBlock: vi.fn(async () => null) }, treeControlSvc: { getActivePauseHoldGate: vi.fn(async () => null) },
|
||||
enqueueWakeup: vi.fn(), getCurrentUserRedactionOptions: vi.fn(async () => ({})),
|
||||
getAgentInvokability: vi.fn(async () => ({ invokable: true })), emitTerminalAgentTaskRun: vi.fn(), budgetHooks: {},
|
||||
} satisfies HeartbeatLifecycleDependencies;
|
||||
await createHeartbeatLifecycle(db, deps).setRunStatusIfRunning(execution.id, status, { finishedAt: new Date() });
|
||||
expect((await f.service.getDetail(f.routine.id))?.activeIssue).toBeNull();
|
||||
expect((await saved(run.id)).status).toBe("issue_created"); await quiet(); expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { RoutineExecutionStatusScope } from "../services/routine-execution-status-notifications.js";
|
||||
import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js";
|
||||
import { afterEach, beforeEach, expect, it, vi } from "vitest";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerRoutineExecutionStatusWork } from "../services/routine-execution-status-work.js";
|
||||
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
let canRun = true;
|
||||
const errors: unknown[] = [];
|
||||
const entry = { id: "run", companyId: "company", routineId: "routine", issueId: "issue" };
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers(); canRun = true; errors.length = 0;
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) });
|
||||
coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => canRun, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
});
|
||||
afterEach(async () => { await coordinator.stop(); vi.useRealTimers(); });
|
||||
|
||||
it("loads actual debt once and leaves no recurring owner work", async () => {
|
||||
const service = { load: vi.fn(async () => []), settle: vi.fn(async () => true) };
|
||||
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
|
||||
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
expect(coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.load).toHaveBeenCalledOnce(); expect(service.settle).not.toHaveBeenCalled(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("reconstructs only an invalidated issue and coalesces duplicate requests", async () => {
|
||||
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
service.load.mockResolvedValueOnce([entry]);
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" })); worker.invalidate(JSON.stringify({ issueId: "issue" }));
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(service.load).toHaveBeenLastCalledWith({ issueId: "issue" }, undefined); expect(service.load).toHaveBeenCalledTimes(2);
|
||||
expect(service.settle).toHaveBeenCalledExactlyOnceWith(expect.objectContaining(entry));
|
||||
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("retries only the failed owner at backoff and disarms after success", async () => {
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("locked")) };
|
||||
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
|
||||
expect(errors).toHaveLength(1);
|
||||
const retryAt = coordinator.nextWakeAt(); expect(retryAt).toBe(Date.now() + 30_000);
|
||||
await vi.advanceTimersByTimeAsync(29_999); expect(service.settle).toHaveBeenCalledOnce();
|
||||
await vi.advanceTimersByTimeAsync(1); expect(service.settle).toHaveBeenCalledTimes(2);
|
||||
expect(service.load).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000); expect(service.settle).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("retains startup and first work during drain without passive owner calls", async () => {
|
||||
canRun = false;
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" }));
|
||||
await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.load).not.toHaveBeenCalled(); expect(service.settle).not.toHaveBeenCalled();
|
||||
canRun = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps a failed owner across drain and retries after resume", async () => {
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValueOnce(new Error("write failed")) };
|
||||
await registerRoutineExecutionStatusWork(coordinator, service, () => canRun).ready;
|
||||
canRun = false; await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(service.load).toHaveBeenCalledOnce();
|
||||
canRun = true; await vi.advanceTimersByTimeAsync(30_000);
|
||||
expect(service.settle).toHaveBeenCalledTimes(2); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
it("retains a notification that arrives while the startup reconstruction is in flight", async () => {
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>(resolve => { release = resolve; });
|
||||
const service = { load: vi.fn(async (_scope?: RoutineExecutionStatusScope, _afterId?: string) => [] as typeof entry[]), settle: vi.fn(async () => true) };
|
||||
service.load.mockImplementationOnce(async () => { await gate; return []; });
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun);
|
||||
await vi.waitFor(() => expect(service.load).toHaveBeenCalledOnce());
|
||||
worker.invalidate(JSON.stringify({ issueId: "issue" })); service.load.mockResolvedValueOnce([entry]); release(); await worker.ready;
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("cancels retries and subscriptions at shutdown", async () => {
|
||||
const service = { load: vi.fn(async () => [entry]), settle: vi.fn(async () => true).mockRejectedValue(new Error("offline")) };
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
await worker.stop(); worker.invalidate(JSON.stringify({ issueId: "issue" })); await vi.advanceTimersByTimeAsync(20 * 60_000);
|
||||
expect(service.settle).toHaveBeenCalledOnce(); expect(coordinator.nextWakeAt()).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it("reconstructs a backlog in bounded keyset pages while retaining a scoped hint behind the cursor", async () => {
|
||||
const firstPage = Array.from({ length: 50 }, (_, i) => ({ ...entry, id: `run-${String(i).padStart(3, "0")}` }));
|
||||
const late = { ...entry, id: "run-late", issueId: "late-issue" };
|
||||
const service = {
|
||||
load: vi.fn(async (scope?: RoutineExecutionStatusScope, afterId?: string) => scope?.issueId ? [late]
|
||||
: afterId ? [{ ...entry, id: "run-050" }] : firstPage),
|
||||
settle: vi.fn(async () => true),
|
||||
};
|
||||
const worker = registerRoutineExecutionStatusWork(coordinator, service, () => canRun); await worker.ready;
|
||||
expect(service.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
worker.invalidate(JSON.stringify({ issueId: "late-issue" }));
|
||||
await vi.advanceTimersByTimeAsync(3000);
|
||||
expect(service.load).toHaveBeenCalledWith(undefined, "run-049");
|
||||
expect(service.load).toHaveBeenCalledWith({ issueId: "late-issue" }, undefined);
|
||||
expect(service.settle).toHaveBeenCalledTimes(52);
|
||||
expect(service.settle).toHaveBeenCalledWith(expect.objectContaining(late));
|
||||
expect(coordinator.nextWakeAt()).toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
@@ -5,11 +6,12 @@ import { companies, createDb, issues, routineRuns, routines } from "@paperclipai
|
||||
import { routineService } from "../services/routines.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_ROUTINE_STATUS_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
if (!support.supported) console.warn(`Skipping routine status database tests: ${support.reason}`);
|
||||
(support.supported ? describe : describe.skip)("routine status reconciliation", () => {
|
||||
let db: ReturnType<typeof createDb>, database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(database.connectionString); }, 60_000);
|
||||
beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("routine-status-"); db = createDb(external ?? database.connectionString, { maxConnections: 6 }); }, 60_000);
|
||||
afterEach(async () => { vi.restoreAllMocks(); await db.execute(sql`SET client_min_messages = warning; TRUNCATE companies CASCADE`); });
|
||||
afterAll(async () => { await db?.$client.end(); await database?.cleanup(); });
|
||||
async function seed(status = "blocked") {
|
||||
@@ -36,11 +38,10 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
|
||||
expect((await db.select().from(routineRuns))[0]).toMatchObject({ status: "issue_created", completedAt: null });
|
||||
}
|
||||
});
|
||||
it.each(["active_issue", "wrong_origin", "wrong_routine", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
|
||||
it.each(["active_issue", "wrong_origin", "wrong_run", "wrong_company", "wrong_routine_company", "different_status"])("does not settle an unsupported handoff: %s", async mode => {
|
||||
const f = await seed();
|
||||
if (mode === "active_issue") await db.update(issues).set({ status: "in_progress" }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_origin") await db.update(issues).set({ originKind: "manual" }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_routine") await db.update(issues).set({ originId: randomUUID() }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_run") await db.update(issues).set({ originRunId: randomUUID() }).where(eq(issues.id, f.issue.id));
|
||||
if (mode === "wrong_company") { const other = await seed("done"); await db.update(routineRuns).set({ companyId: other.companyId }).where(eq(routineRuns.id, f.run.id)); }
|
||||
if (mode === "wrong_routine_company") { const other = await seed("done");
|
||||
@@ -53,6 +54,20 @@ if (!support.supported) console.warn(`Skipping routine status database tests: ${
|
||||
expect((await db.select().from(routineRuns)).find(row => row.id === f.run.id)).toEqual(before.find(row => row.id === f.run.id));
|
||||
if (mode === "wrong_routine_company") await expect(routineService(db).syncRunStatusForIssue(f.issue.id)).resolves.toBeNull();
|
||||
});
|
||||
it("uses reciprocal durable custody when a retained origin no longer names the current routine declaration", async () => {
|
||||
const f = await seed("done");
|
||||
await db.update(issues).set({ originId: "previous-declaration-origin" }).where(eq(issues.id, f.issue.id));
|
||||
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 1, settled: 1, deferred: 0 });
|
||||
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("completed");
|
||||
});
|
||||
it.each(["unknown-status", "other-code"])("preserves a failed invocation with an unrelated transient marker: %s", async mode => {
|
||||
const f = await seed("in_progress");
|
||||
await db.update(routineRuns).set({ status: "failed", failureReason: null,
|
||||
triggerPayload: { transientFailure: { code: mode === "other-code" ? "unrelated_failure" : "execution_issue_status",
|
||||
status: mode === "unknown-status" ? "unrecognized" : "blocked" } } }).where(eq(routineRuns.id, f.run.id));
|
||||
expect(await routineService(db).reconcileExecutionRunStatuses()).toEqual({ scanned: 0, settled: 0, deferred: 0 });
|
||||
expect((await db.select().from(routineRuns).where(eq(routineRuns.id, f.run.id)))[0].status).toBe("failed");
|
||||
});
|
||||
it.each(["resume", "retarget", "routine_transfer"])("rechecks %s after candidate selection and before repair", async mode => {
|
||||
const f = await seed(); const transaction = db.transaction.bind(db);
|
||||
vi.spyOn(db, "transaction").mockImplementationOnce(async callback => {
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { routineExecutionStatusService } from "../services/routine-run-status.js";
|
||||
import { attentionRetentionService } from "../services/attention-retention.js";
|
||||
import { statusCardEvaluationService, registerStatusCardEvaluationWork } from "../services/status-card-evaluation.js";
|
||||
import { nativeHarnessBackupExpirationService } from "../services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { skillVersionRetentionService } from "../services/skill-version-retention.js";
|
||||
import { approvalQueueSeedingService } from "../services/approval-queue-seeding-work.js";
|
||||
@@ -141,6 +144,9 @@ const {
|
||||
})),
|
||||
reconcileHotRestartAdoption: vi.fn(async () => ({ mode: "none" })),
|
||||
reapOrphanedRuns: vi.fn(async () => ({ reaped: 0, runIds: [] })),
|
||||
registerIssueStateRecoveryWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerDeferredReceiptWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRunDispatchWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerScheduledRetryCancellationWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerStoppedInstructionCollectionWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve() })),
|
||||
registerRetainedNativeSessionCleanupWork: vi.fn((_coordinator: unknown, _canRun: () => boolean) => ({ ready: Promise.resolve(), stop: vi.fn() })),
|
||||
@@ -187,6 +193,8 @@ const {
|
||||
const heartbeatServiceFactoryMock = vi.fn(() => heartbeatServiceMock);
|
||||
const issueThreadInteractionServiceMock = {
|
||||
sweepSupersededPendingRequestConfirmations: vi.fn(async () => ({ expired: 0 })),
|
||||
mergedPullRequestConfirmationPage: vi.fn(async () => ({ rows: [], complete: true })),
|
||||
mergedPullRequestConfirmationCandidates: vi.fn(async () => []),
|
||||
sweepMergedPullRequestConfirmations: vi.fn(async () => ({
|
||||
checked: 0,
|
||||
candidates: 0,
|
||||
@@ -214,7 +222,8 @@ const {
|
||||
};
|
||||
const executionWorkspaceServiceFactoryMock = vi.fn(() => executionWorkspaceServiceMock);
|
||||
const externalObjectsServiceMock = {
|
||||
refreshDueObjectsForActiveCompanies: vi.fn(async () => ({ companies: 0, checked: 0, refreshed: 0 })),
|
||||
registerRefreshWork: vi.fn((coordinator: { register: typeof deliveryWorkRegisterMock }, _canRun: () => boolean) =>
|
||||
coordinator.register("external-object-refresh", { run: async () => undefined, hasPending: async () => false })),
|
||||
};
|
||||
const externalObjectsServiceFactoryMock = vi.fn(() => externalObjectsServiceMock);
|
||||
const routineServiceMock = {
|
||||
@@ -384,6 +393,11 @@ vi.mock("../realtime/live-events-ws.js", () => ({
|
||||
setupLiveEventsWebSocketServer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/attention-retention.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/attention-retention.js")>(),
|
||||
attentionRetentionService: vi.fn(() => ({ load: vi.fn(async () => ({ rows: [], complete: true })), check: vi.fn() })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/join-request-retention.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/join-request-retention.js")>(),
|
||||
joinRequestRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
|
||||
@@ -399,6 +413,11 @@ vi.mock("../services/approval-retention.js", async importOriginal => ({
|
||||
approvalRetentionService: vi.fn(() => ({ sweep: vi.fn(async () => ({ archived: 0 })), next: vi.fn(async () => null) })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/routine-run-status.js", async importOriginal => ({
|
||||
...await importOriginal<typeof import("../services/routine-run-status.js")>(),
|
||||
routineExecutionStatusService: vi.fn(() => ({ load: vi.fn(async () => []), settle: vi.fn(async () => false) })),
|
||||
}));
|
||||
|
||||
vi.mock("../services/index.js", () => ({
|
||||
backfillLegacyToolOAuthTokens: vi.fn(async () => ({
|
||||
scannedConnections: 0,
|
||||
@@ -537,6 +556,13 @@ vi.mock("../services/native-runtime/native-question-bridge.js", () => ({
|
||||
validateNativeQuestionResponseInput: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../services/status-card-evaluation.js", () => ({
|
||||
statusCardEvaluationService: vi.fn(() => ({ run: vi.fn(async () => {}), next: vi.fn(async () => null), changed: vi.fn() })),
|
||||
registerStatusCardEvaluationWork: vi.fn((_coordinator, service, canRun) => ({
|
||||
ready: canRun() ? service.run(new AbortController().signal) : Promise.resolve(), wake: vi.fn(), stop: vi.fn(async () => {}),
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../services/secret-proposals.js", () => ({
|
||||
createSecretProposalsService: vi.fn(() => ({
|
||||
expiryDeadlines: vi.fn(async () => []),
|
||||
@@ -576,6 +602,8 @@ vi.mock("../services/native-runtime/native-harness-backup-expiration.js", () =>
|
||||
tick: vi.fn(async () => ({ checked: 0, candidates: 0, evidenceCandidates: 0, failed: 0, truncated: false })),
|
||||
noteEvidence: vi.fn(), nextEvidenceAt: vi.fn(() => null),
|
||||
tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn(),
|
||||
backupScopes: vi.fn(async () => ({ names: [], complete: true })),
|
||||
reconcileBackup: vi.fn(async () => null), expireBackup: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
@@ -751,9 +779,12 @@ describe("startServer feedback export wiring", () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ nativeHarnessBackupExpirationEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toHaveLength(enabled ? 1 : 0);
|
||||
expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "evidence"] : ["backups"]);
|
||||
expect(vi.mocked(nativeHarnessBackupExpirationService).mock.calls.map(([, options]) => options?.duty)).toEqual(enabled ? ["backups", "backups", "evidence"] : ["backups"]);
|
||||
const backups = vi.mocked(nativeHarnessBackupExpirationService).mock.results[0]!.value;
|
||||
expect(backups.registerCheckpointExpiration).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toHaveLength(enabled ? 1 : 0);
|
||||
expect(backups.tick).not.toHaveBeenCalled();
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeHarnessBackupExpiration)).toEqual([]);
|
||||
});
|
||||
|
||||
it("tracks held evidence reconstruction after HTTP readiness", async () => {
|
||||
@@ -761,6 +792,9 @@ describe("startServer feedback export wiring", () => {
|
||||
const pending = Promise.withResolvers<{ truncated: boolean }>();
|
||||
vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(), noteEvidence: vi.fn(), nextEvidenceAt: () => null,
|
||||
tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never);
|
||||
vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({
|
||||
backupScopes: vi.fn(async () => ({ names: [], complete: true })), reconcileBackup: vi.fn(), expireBackup: vi.fn(),
|
||||
} as never);
|
||||
vi.mocked(nativeHarnessBackupExpirationService).mockReturnValueOnce({ tick: vi.fn(() => pending.promise), noteEvidence: vi.fn(), nextEvidenceAt: () => null,
|
||||
tickCheckpoints: vi.fn(), registerCheckpointExpiration: vi.fn() } as never);
|
||||
try {
|
||||
@@ -800,6 +834,28 @@ describe("startServer feedback export wiring", () => {
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.skillVersionRetention)).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers status-card evaluation only with maintenance and never ticks it from the heartbeat loop (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 1000 }));
|
||||
let tick: (() => void) | undefined;
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
|
||||
if (interval === 1000) tick = callback;
|
||||
return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(registerStatusCardEvaluationWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(statusCardEvaluationService).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
if (enabled) {
|
||||
const service = vi.mocked(statusCardEvaluationService).mock.results.at(-1)!.value;
|
||||
expect(service.run).toHaveBeenCalledOnce();
|
||||
expect(tick).toBeDefined();
|
||||
for (let i = 0; i < 3; i++) { tick!(); await settleStartupHousekeeping(); }
|
||||
expect(service.run).toHaveBeenCalledOnce();
|
||||
}
|
||||
expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.statusCardEvaluation)).toEqual([]);
|
||||
} finally { timer.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each([false, true])("registers response recovery and its shutdown subscription only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -933,6 +989,21 @@ describe("startServer feedback export wiring", () => {
|
||||
if (enabled) expect(heartbeatServiceMock.registerCapturedInstructionCommitWork.mock.calls[0][1]()).toBe(true);
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry and queued dispatch only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.registerIssueStateRecoveryWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerRunDispatchWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.registerDeferredReceiptWork).toHaveBeenCalledTimes(enabled ? 1 : 0);
|
||||
expect(heartbeatServiceMock.resumeQueuedRuns).not.toHaveBeenCalled();
|
||||
if (enabled) {
|
||||
const [coordinator, admitted] = heartbeatServiceMock.registerRunDispatchWork.mock.calls[0];
|
||||
const app = await createAppMock.mock.results.at(-1)!.value;
|
||||
expect(coordinator).toBe(app.locals.deliveryWork);
|
||||
expect(admitted()).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("registers retry cancellation recovery only when maintenance is enabled (%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
@@ -984,7 +1055,7 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([false, true])("registers archive delivery separately from retention sweeps (scheduler %s)", async enabled => {
|
||||
it.each([false, true])("registers source retention separately from archive delivery without tick scans (scheduler %s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled, heartbeatSchedulerIntervalMs: 30000 }));
|
||||
let tick: (() => void) | undefined;
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, interval: number) => {
|
||||
@@ -996,12 +1067,17 @@ describe("startServer feedback export wiring", () => {
|
||||
const registration = deliveryWorkRegisterMock.mock.calls.find(([queue]) => queue === DELIVERY_QUEUES.decisionArchiveDelivery);
|
||||
if (!enabled) {
|
||||
expect(registration).toBeUndefined();
|
||||
expect(attentionRetentionService).not.toHaveBeenCalled();
|
||||
expect(decisionRetentionService).not.toHaveBeenCalled();
|
||||
expect(tick).toBeDefined(); tick!(); await settleStartupHousekeeping();
|
||||
expect(decisionRetentionService).not.toHaveBeenCalled();
|
||||
return;
|
||||
}
|
||||
expect(registration).toBeDefined();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.attentionRetention)).toBe(true);
|
||||
expect(attentionRetentionService).toHaveBeenCalledOnce();
|
||||
const source = vi.mocked(attentionRetentionService).mock.results[0]!.value;
|
||||
expect(source.load).toHaveBeenCalledOnce();
|
||||
const retention = vi.mocked(decisionRetentionService).mock.results[0]!.value;
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
|
||||
expect(retention.nextArchiveNotificationAt).toHaveBeenCalledOnce();
|
||||
@@ -1009,9 +1085,10 @@ describe("startServer feedback export wiring", () => {
|
||||
const companyReads = () => vi.mocked(db.select).mock.results.filter(({ value }) =>
|
||||
vi.mocked(value.from).mock.calls.some(([table]) => table === companies)).length;
|
||||
const before = companyReads();
|
||||
expect(before).toBeGreaterThan(0); expect(tick).toBeDefined();
|
||||
expect(before).toBe(0); expect(tick).toBeDefined();
|
||||
tick!(); await settleStartupHousekeeping();
|
||||
expect(companyReads()).toBeGreaterThan(before);
|
||||
expect(companyReads()).toBe(before);
|
||||
expect(source.load).toHaveBeenCalledOnce();
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledOnce();
|
||||
await registration![1].run(new AbortController().signal);
|
||||
expect(retention.deliverNotifications).toHaveBeenCalledTimes(2);
|
||||
@@ -1357,13 +1434,14 @@ describe("startServer feedback export wiring", () => {
|
||||
await startServer();
|
||||
await settleStartupHousekeeping();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(intervalCallback).not.toBeNull();
|
||||
intervalCallback?.();
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).toHaveBeenCalledTimes(2);
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.sweepStaleIssueLocks).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.reapOrphanedRuns).toHaveBeenLastCalledWith({ staleThresholdMs: 300_000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true });
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(heartbeatServiceMock.promoteDueScheduledRetries).not.toHaveBeenCalled();
|
||||
expect(retiredDetector).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
delete (runtime as Partial<typeof runtime>).reconcileProductivityReviews;
|
||||
@@ -1402,6 +1480,27 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([{ cloud: false, enabled: true }, { cloud: true, enabled: false }, { cloud: true, enabled: true }])(
|
||||
"registers routine status only for enabled Cloud scheduling ($cloud/$enabled)", async ({ cloud, enabled }) => {
|
||||
if (cloud) vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "startup-test-token");
|
||||
else vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", undefined);
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
const intervalCallbacks: Array<() => void> = [];
|
||||
const intervals = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void) => {
|
||||
intervalCallbacks.push(callback); return 1 as unknown as ReturnType<typeof setInterval>;
|
||||
}) as typeof setInterval);
|
||||
try {
|
||||
await startServer(); await settleStartupHousekeeping();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.routineExecutionStatus)).toHaveLength(cloud && enabled ? 1 : 0);
|
||||
const service = vi.mocked(routineExecutionStatusService).mock.results.at(-1)?.value;
|
||||
if (cloud && enabled) expect(service?.load).toHaveBeenCalledExactlyOnceWith(undefined, undefined);
|
||||
for (let index = 0; index < 4; index++) for (const callback of intervalCallbacks) callback();
|
||||
await settleStartupHousekeeping();
|
||||
if (cloud && enabled) expect(service?.load).toHaveBeenCalledOnce();
|
||||
} finally { intervals.mockRestore(); vi.unstubAllEnvs(); }
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps routine ticks and setup cleanup active when heartbeat scheduling is suppressed", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: true,
|
||||
@@ -1435,8 +1534,12 @@ describe("startServer feedback export wiring", () => {
|
||||
await Promise.resolve();
|
||||
|
||||
expect(heartbeatServiceMock.tickAgentTimers).toHaveBeenCalledTimes(1);
|
||||
expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1);
|
||||
expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).toHaveBeenCalledTimes(1);
|
||||
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1);
|
||||
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function));
|
||||
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true);
|
||||
expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalledTimes(1);
|
||||
expect(issueThreadInteractionServiceMock.sweepMergedPullRequestConfirmations).not.toHaveBeenCalled();
|
||||
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.mergedPullRequestConfirmations)).toBe(true);
|
||||
expect(executionWorkspaceServiceMock.sweepTerminalWorkspaces).toHaveBeenCalledTimes(1);
|
||||
expect(routineServiceMock.tickScheduledTriggers).toHaveBeenCalledTimes(1);
|
||||
const routineWorkerIndex = deliveryWorkRegisterMock.mock.calls.findIndex(([queue]) => queue === DELIVERY_QUEUES.routineSchedules);
|
||||
@@ -1458,6 +1561,15 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it.each([true, false])("does not wait for external-object provider work before readiness (scheduler=%s)", async enabled => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled }));
|
||||
externalObjectsServiceMock.registerRefreshWork.mockReturnValueOnce({
|
||||
ready: new Promise<void>(() => {}), wake: vi.fn(), stop: vi.fn(async () => {}),
|
||||
});
|
||||
await startServer();
|
||||
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("keeps external object refresh active when heartbeat scheduling is disabled", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({
|
||||
heartbeatSchedulerEnabled: false,
|
||||
@@ -1485,7 +1597,9 @@ describe("startServer feedback export wiring", () => {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
expect(externalObjectsServiceMock.refreshDueObjectsForActiveCompanies).toHaveBeenCalledTimes(1);
|
||||
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledTimes(1);
|
||||
expect(externalObjectsServiceMock.registerRefreshWork).toHaveBeenCalledWith(expect.any(Object), expect.any(Function));
|
||||
expect(deliveryWorkRegisterMock.mock.calls.some(([queue]) => queue === DELIVERY_QUEUES.externalObjectRefresh)).toBe(true);
|
||||
expect(routineServiceMock.tickScheduledTriggers).not.toHaveBeenCalled();
|
||||
expect(heartbeatServiceMock.reconcileCostAccounting).toHaveBeenCalledTimes(1);
|
||||
expect(environmentCustomImagesServiceMock.cleanupExpiredSetupSessions).not.toHaveBeenCalled();
|
||||
@@ -1523,6 +1637,20 @@ describe("startServer feedback export wiring", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("becomes ready without waiting for the merge-confirmation observation", async () => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true }));
|
||||
let release!: () => void;
|
||||
const blocked = new Promise<{ rows: never[]; complete: boolean }>(resolve => { release = () => resolve({ rows: [], complete: true }); });
|
||||
issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage.mockReturnValueOnce(blocked);
|
||||
const timer = vi.spyOn(globalThis, "setInterval").mockImplementation(() => 1 as unknown as ReturnType<typeof setInterval>);
|
||||
let started = false;
|
||||
const starting = startServer().then(server => { started = true; return server; });
|
||||
try {
|
||||
await vi.waitFor(() => expect(issueThreadInteractionServiceMock.mergedPullRequestConfirmationPage).toHaveBeenCalled());
|
||||
await settleStartupHousekeeping(); expect(started).toBe(true); expect(getStartupRecoveryState().phase).toBe("ready");
|
||||
} finally { release(); await starting; timer.mockRestore(); }
|
||||
});
|
||||
|
||||
it.each(["retained cleanup", "workspace finalization", "environment settlement", "native run projection", "native result commit", "native recovery notices", "external conversation settlement"] as const)("becomes ready while %s is held but joins it at shutdown", async owner => {
|
||||
loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: true }));
|
||||
let releaseCleanup!: () => void;
|
||||
|
||||
@@ -0,0 +1,656 @@
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { documentService } from "../services/documents.js";
|
||||
import * as companySearch from "../services/company-search.js";
|
||||
import { setImmediate as immediate } from "node:timers/promises";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { activityLog, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, costEvents, companies, companyMemberships, createDb, documents, withDedicatedDbConnection,
|
||||
heartbeatRuns, instanceSettings, issueComments, issues, statusCards, statusCardUpdates, documentRevisions } from "@paperclipai/db";
|
||||
import { companySearchQuerySchema, statusCardRefreshPolicySchema } from "@paperclipai/shared";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "../services/status-card-evaluation.js";
|
||||
import { statusCardService } from "../services/status-cards.js";
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARD_EVALUATION_TEST_DATABASE_URL");
|
||||
const support = external ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
(support.supported ? describe : describe.skip)("status-card evaluation work", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
let db: ReturnType<typeof createDb>;
|
||||
let coordinator: ReturnType<typeof createDeliveryWorkCoordinator> | undefined;
|
||||
let draining = false;
|
||||
let companyId: string, agentId: string;
|
||||
const errors: unknown[] = [];
|
||||
const evaluations = new Set<Promise<unknown>>();
|
||||
let compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
|
||||
const actor = { agentId: null, userId: "board-user" };
|
||||
const policy = (mode: "manual" | "reactive" | "interval" = "reactive") => statusCardRefreshPolicySchema.parse({
|
||||
mode, debounceSeconds: 60, intervalMinutes: 15,
|
||||
});
|
||||
// Controlled provider boundary: assignment persists a queued wake, and tests
|
||||
// explicitly run the authorized summarizer writeback. No model/provider runs.
|
||||
const wakeup = vi.fn(async (id: string, options: { payload?: Record<string, unknown> | null; reason?: string | null }) => {
|
||||
const [agent] = await db.select({ companyId: agents.companyId }).from(agents).where(eq(agents.id, id));
|
||||
return db.insert(agentWakeupRequests).values({ companyId: agent!.companyId, agentId: id, source: "assignment",
|
||||
reason: options.reason, payload: options.payload, status: "queued" }).returning();
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
if (!external) temporary = await startEmbeddedPostgresTestDatabase("status-card-evaluation-");
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
}, 30_000);
|
||||
beforeEach(async () => {
|
||||
companyId = randomUUID(); agentId = randomUUID(); draining = false; errors.length = 0; wakeup.mockReset();
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"] });
|
||||
await db.insert(companies).values({ id: companyId, name: "Status", issuePrefix: `S${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
|
||||
await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`)));
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
});
|
||||
afterEach(async () => {
|
||||
await coordinator?.stop(); coordinator = undefined;
|
||||
vi.useRealTimers(); vi.restoreAllMocks();
|
||||
await db.delete(budgetIncidents); await db.delete(budgetPolicies); await db.delete(approvals); await db.delete(costEvents);
|
||||
await db.delete(agentWakeupRequests); await db.delete(statusCardUpdates); await db.delete(statusCards);
|
||||
await db.delete(documentRevisions); await db.delete(documents); await db.delete(activityLog);
|
||||
await db.delete(issueComments); await db.delete(issues); await db.delete(heartbeatRuns);
|
||||
await db.delete(instanceSettings); await db.delete(agents); await db.delete(companyMemberships); await db.delete(companies);
|
||||
});
|
||||
afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); });
|
||||
async function start(service = statusCardEvaluationService(db, { wakeup })) {
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !draining, canReconcile: () => true,
|
||||
onError: error => errors.push(error) });
|
||||
await registerStatusCardEvaluationWork(coordinator, { ...service, run: signal => {
|
||||
const work = service.run(signal);
|
||||
evaluations.add(work);
|
||||
void work.then(() => evaluations.delete(work), () => evaluations.delete(work));
|
||||
return work;
|
||||
} }, () => !draining).ready;
|
||||
}
|
||||
async function settled() {
|
||||
// Do not leap across PostgreSQL connect timers while real I/O is pending.
|
||||
// Await actual domain work, then require several settled event-loop turns
|
||||
// so a transient zero hold cannot stand in for transaction reconciliation.
|
||||
for (let turn = 0; turn < 3; turn++) {
|
||||
await vi.waitFor(async () => {
|
||||
await Promise.allSettled([...evaluations]);
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await Promise.allSettled([...evaluations]);
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
}, { timeout: 5000 });
|
||||
await immediate();
|
||||
}
|
||||
}
|
||||
async function quiet() {
|
||||
await settled();
|
||||
expect(coordinator!.nextWakeAt()).toBeNull();
|
||||
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "update"), executes = vi.spyOn(db, "execute");
|
||||
const calls = wakeup.mock.calls.length;
|
||||
await vi.advanceTimersByTimeAsync(10 * 60_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(executes).not.toHaveBeenCalled();
|
||||
expect(wakeup).toHaveBeenCalledTimes(calls);
|
||||
reads.mockRestore(); writes.mockRestore(); executes.mockRestore();
|
||||
}
|
||||
async function writeback(cardId: string, generationIssueId: string, compile = false) {
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
|
||||
await db.update(issues).set({ checkoutRunId: run!.id }).where(eq(issues.id, generationIssueId));
|
||||
const writer = { agentId, runId: run!.id };
|
||||
const cards = statusCardService(db);
|
||||
if (compile) await cards.writeQuery(cardId, { queries: [compiledQuery],
|
||||
title: "Blockers", changeSummary: "Compiled", generationIssueId }, writer);
|
||||
await cards.writeSummary(cardId, { markdown: "Current blockers", changeSummary: "Updated", generationIssueId }, writer);
|
||||
await issueService(db).update(generationIssueId, { status: "done" });
|
||||
}
|
||||
async function card(mode: "manual" | "reactive" | "interval" = "reactive", refreshPolicy = policy(mode)) {
|
||||
const cards = statusCardService(db);
|
||||
const created = await cards.create(companyId, { interestPrompt: "Blockers", titlePinned: false, agentId, refreshPolicy }, actor);
|
||||
const compile = await cards.requestCompile(created.id, actor);
|
||||
await writeback(created.id, compile.generatingIssue.id, true);
|
||||
return cards.getById(created.id).then(row => row!);
|
||||
}
|
||||
|
||||
it("settles empty, disabled, configured manual and unchanged reactive history without recurring SQL", async () => {
|
||||
await start(); await quiet();
|
||||
await card("manual"); await card("reactive"); await quiet();
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("ordinary issue change schedules debounce, assigns one durable wake, writes a result and becomes quiet", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
const changed = await issueService(db).create(companyId, { title: "New blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
await vi.waitFor(async () => {
|
||||
expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
}, { timeout: 5000 });
|
||||
const pending = await statusCardService(db).getById(created.id);
|
||||
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(pending?.nextEvalAt).toBeInstanceOf(Date);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled();
|
||||
const generating = await statusCardService(db).getById(created.id);
|
||||
expect(generating?.generatingIssueId).toBeTruthy();
|
||||
const durable = await db.select().from(agentWakeupRequests);
|
||||
expect(durable).toHaveLength(1); expect(durable[0]?.payload?.issueId).toBe(generating?.generatingIssueId);
|
||||
expect(changed.companyId).toBe(companyId);
|
||||
await writeback(created.id, generating!.generatingIssueId!); await quiet();
|
||||
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === generating?.generatingIssueId)))
|
||||
.toMatchObject([{ status: "ok", finishedAt: expect.any(Date) }]);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps card deletion scoped to its company and a missing deletion quiet", async () => {
|
||||
const removed = await card();
|
||||
await card();
|
||||
const ownCompany = companyId, ownAgent = agentId;
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const other = await card();
|
||||
companyId = ownCompany; agentId = ownAgent;
|
||||
const factory = companySearch.companySearchService;
|
||||
const searchedCompanies: string[] = [];
|
||||
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
|
||||
const search = factory(owner), actual = search.search;
|
||||
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
|
||||
return search;
|
||||
});
|
||||
await start(); await quiet(); searchedCompanies.length = 0;
|
||||
const cards = statusCardService(db);
|
||||
expect(await cards.remove(removed.id)).toMatchObject({ id: removed.id, companyId: ownCompany });
|
||||
await settled();
|
||||
// Deleting a card only reconstructs deadlines; it is not an issue change
|
||||
// that warrants re-running any unchanged reactive card's saved search.
|
||||
expect(searchedCompanies).toEqual([]);
|
||||
expect(await cards.getById(other.id)).toMatchObject({ nextEvalAt: null, generatingIssueId: null, pendingChangeCount: 0 });
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
searchedCompanies.length = 0;
|
||||
expect(await cards.remove(removed.id)).toBeNull();
|
||||
await quiet();
|
||||
expect(searchedCompanies).toEqual([]);
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("coalesces one hundred committed source writes without delaying another company's saved deadline", async () => {
|
||||
const reactive = await card();
|
||||
const originalCompany = companyId, originalAgent = agentId;
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `O${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const other = await card("interval");
|
||||
const otherIssue = await issueService(db).create(companyId, { title: "Other interval blocker", status: "blocked", priority: "medium" });
|
||||
const otherReactive = await card();
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
const originalRun = service.run;
|
||||
let running: Promise<unknown> | undefined;
|
||||
const run = vi.spyOn(service, "run").mockImplementation(signal => {
|
||||
const result = originalRun(signal); running = result; return result;
|
||||
});
|
||||
await start(service); await settled();
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
// Give each committed producer and any actual domain work time to settle.
|
||||
// This prevents incidental overlap from hiding per-write reconstruction.
|
||||
const flush = async () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await running; await immediate();
|
||||
}
|
||||
};
|
||||
await flush();
|
||||
const select = vi.spyOn(db, "select");
|
||||
const factory = companySearch.companySearchService;
|
||||
const searchedCompanies: string[] = [];
|
||||
vi.spyOn(companySearch, "companySearchService").mockImplementation(owner => {
|
||||
const search = factory(owner);
|
||||
const actual = search.search;
|
||||
search.search = async (...args) => { searchedCompanies.push(args[0]); return actual(...args); };
|
||||
return search;
|
||||
});
|
||||
run.mockClear();
|
||||
const burstStarted = Date.now();
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const issue = await issueService(db).create(companyId, { title: `Burst blocker ${i}`, status: "blocked", priority: "medium" });
|
||||
await flush();
|
||||
await issueService(db).addComment(issue.id, `Human update ${i}`, { userId: "board-user" });
|
||||
await flush();
|
||||
}
|
||||
await flush();
|
||||
// One eager reconciliation after idle and one bounded batched continuation;
|
||||
// the producers themselves are not counted as background SQL.
|
||||
const dueReads = () => select.mock.calls.filter(([fields]) => fields && "card" in fields && fields.card === statusCards).length;
|
||||
const nextReads = () => select.mock.calls.filter(([fields]) => fields && "at" in fields && fields.at === statusCards.nextEvalAt).length;
|
||||
const elapsedMs = Date.now() - burstStarted;
|
||||
const maximum = Math.ceil(elapsedMs / 1000) + 2;
|
||||
console.info("status-card burst", { elapsedMs, maximum, runs: run.mock.calls.length, due: dueReads(), next: nextReads(), searches: searchedCompanies.length });
|
||||
expect(run.mock.calls.length).toBeLessThanOrEqual(maximum);
|
||||
expect(dueReads()).toBeLessThanOrEqual(maximum); expect(nextReads()).toBeLessThanOrEqual(maximum);
|
||||
expect(searchedCompanies.length).toBeLessThanOrEqual(maximum);
|
||||
expect(searchedCompanies.length).toBeGreaterThan(0);
|
||||
expect(new Set(searchedCompanies)).toEqual(new Set([originalCompany]));
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
// Re-register with an existing durable deadline inside the coalescing
|
||||
// window. A company hint must not make that real deadline wait one second.
|
||||
await coordinator!.stop(); coordinator = undefined;
|
||||
const dueAt = Date.now() + 500;
|
||||
await db.update(statusCards).set({ nextEvalAt: new Date(dueAt), nextEvalReason: "refresh" }).where(eq(statusCards.id, other.id));
|
||||
await start(service); await flush();
|
||||
await issueService(db).addComment(otherIssue.id, "Another company's coalesced human update", { userId: "board-user" });
|
||||
await issueService(db).addComment((await db.select({ id: issues.id }).from(issues).where(eq(issues.companyId, originalCompany)))[0]!.id,
|
||||
"More changes while another company is due", { userId: "board-user" });
|
||||
await flush();
|
||||
await vi.advanceTimersByTimeAsync(500); await flush();
|
||||
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
|
||||
expect(Date.now()).toBeLessThan(dueAt + 500);
|
||||
expect(wakeup.mock.calls[0]![0]).not.toBe(originalAgent);
|
||||
const generated = (await statusCardService(db).getById(other.id))!;
|
||||
expect(generated.generatingIssueId).toBeTruthy();
|
||||
expect((await statusCardService(db).getById(reactive.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect((await statusCardService(db).getById(otherReactive.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(searchedCompanies).toContain(other.companyId);
|
||||
// Complete all ordinary results, then prove there is no recurring batch
|
||||
// timer once the requested work has settled.
|
||||
companyId = other.companyId; agentId = other.agentId!;
|
||||
await writeback(other.id, generated.generatingIssueId!);
|
||||
await statusCardService(db).update((await statusCardService(db).getById(other.id))!, { archived: true }, actor);
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
await vi.advanceTimersByTimeAsync(120_000); await settled(); await flush();
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(reactive.id))!.generatingIssueId).toBeTruthy());
|
||||
const pending = (await statusCardService(db).getById(reactive.id))!;
|
||||
expect(pending.generatingIssueId).toBeTruthy();
|
||||
await writeback(reactive.id, pending.generatingIssueId!);
|
||||
const otherPending = (await statusCardService(db).getById(otherReactive.id))!;
|
||||
expect(otherPending.generatingIssueId).toBeTruthy();
|
||||
companyId = other.companyId; agentId = other.agentId!;
|
||||
await writeback(otherReactive.id, otherPending.generatingIssueId!);
|
||||
companyId = originalCompany; agentId = originalAgent;
|
||||
select.mockRestore();
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
}, 30_000);
|
||||
|
||||
it("bounds active-hours conversion work across twenty cards and repeated ordinary company writes", async () => {
|
||||
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
|
||||
const hoursPolicy = { ...policy(), activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } };
|
||||
const cards = [];
|
||||
for (let i = 0; i < 20; i++) cards.push(await card("reactive", hoursPolicy));
|
||||
await start(); await quiet();
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
const actualParts = RealFormatter.prototype.formatToParts;
|
||||
let constructions = 0, conversions = 0;
|
||||
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
if (args[0] === "en-GB" && args[1]?.hourCycle === "h23") constructions++;
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
const parts = vi.spyOn(RealFormatter.prototype, "formatToParts").mockImplementation(function (this: Intl.DateTimeFormat, date) {
|
||||
if (this.resolvedOptions().locale === "en-GB" && this.resolvedOptions().hourCycle === "h23") conversions++;
|
||||
return actualParts.call(this, date);
|
||||
});
|
||||
const target = await issueService(db).create(companyId, { title: "Closed-hours blocker", status: "blocked", priority: "medium" });
|
||||
await settled();
|
||||
expect(constructions).toBeLessThanOrEqual(1);
|
||||
expect(conversions).toBeLessThanOrEqual(1000);
|
||||
const cold = { constructions, conversions };
|
||||
constructions = 0; conversions = 0;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await issueService(db).addComment(target.id, `Closed-hours human update ${i}`, { userId: "board-user" });
|
||||
await settled();
|
||||
}
|
||||
expect(constructions).toBe(0);
|
||||
expect(conversions).toBeGreaterThanOrEqual(200);
|
||||
expect(conversions).toBeLessThanOrEqual(800);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
const opening = new Date("2026-11-03T09:00:00Z");
|
||||
for (const created of cards) expect((await statusCardService(db).getById(created.id)))
|
||||
.toMatchObject({ state: "paused_hours", pendingChangeCount: 1, nextEvalAt: opening });
|
||||
process.stdout.write(JSON.stringify({ measurement: "status-card company active-hours workload", cards: 20,
|
||||
sourceWrites: 11, cold, warm: { constructions, conversions } }) + "\n");
|
||||
constructor.mockRestore(); parts.mockRestore();
|
||||
await vi.advanceTimersByTimeAsync(opening.getTime() - Date.now()); await settled();
|
||||
expect(wakeup).toHaveBeenCalledTimes(20);
|
||||
for (const created of cards) {
|
||||
const generated = (await statusCardService(db).getById(created.id))!;
|
||||
expect(generated.generatingIssueId).toBeTruthy();
|
||||
await writeback(created.id, generated.generatingIssueId!);
|
||||
}
|
||||
await quiet(); expect(errors).toEqual([]);
|
||||
}, 60_000);
|
||||
|
||||
it("cancels archived or disabled future work and replaces an earlier interval deadline", async () => {
|
||||
await start(); let created = await card("interval"); await settled();
|
||||
const old = coordinator!.nextWakeAt();
|
||||
created = await statusCardService(db).update(created, { refreshPolicy: { ...policy("interval"), intervalMinutes: 1 } }, actor);
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await Promise.allSettled([...evaluations]);
|
||||
expect(coordinator!.nextWakeAt()).toBeLessThan(old!);
|
||||
});
|
||||
await statusCardService(db).update(created, { archived: true }, actor); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("restores overdue debt after the feature is disabled across its saved deadline", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
await issueService(db).create(companyId, { title: "Disabled blocker", status: "blocked", priority: "medium" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: false }); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
|
||||
await instanceSettingsService(db).updateExperimental({ enableStatusCards: true });
|
||||
await vi.waitFor(() => expect(wakeup).toHaveBeenCalledOnce());
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!); await quiet();
|
||||
});
|
||||
|
||||
it("does not react to rolled-back or nested uncommitted card configuration", async () => {
|
||||
await start(); const created = await card("manual"); await quiet();
|
||||
await expect(db.transaction(async tx => {
|
||||
await statusCardService(tx as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
await vi.advanceTimersByTimeAsync(30_000); expect(wakeup).not.toHaveBeenCalled();
|
||||
throw new Error("rollback");
|
||||
})).rejects.toThrow("rollback");
|
||||
await quiet(); expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
|
||||
await db.transaction(async tx => {
|
||||
await tx.transaction(async nested => {
|
||||
await statusCardService(nested as unknown as typeof db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
});
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull();
|
||||
expect((await statusCardService(db).getById(created.id))?.nextEvalAt).toBeNull();
|
||||
});
|
||||
await settled(); expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("serializes competing connections when one source event preempts a future evaluation", async () => {
|
||||
const created = await card();
|
||||
await db.update(statusCards).set({ nextEvalAt: new Date(Date.now() + 3_600_000) }).where(eq(statusCards.id, created.id));
|
||||
await issueService(db).create(companyId, { title: "Concurrent blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
const first = statusCardEvaluationService(db, { wakeup });
|
||||
const second = statusCardEvaluationService(db, { wakeup });
|
||||
first.changed(new Set([`changes:${companyId}`])); second.changed(new Set([`changes:${companyId}`]));
|
||||
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
|
||||
const pending = await statusCardService(db).getById(created.id);
|
||||
expect(pending?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect(pending?.nextEvalAt?.getTime()).toBeLessThan(Date.now() + 3_600_000);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(60_000);
|
||||
await Promise.all([first.run(new AbortController().signal), second.run(new AbortController().signal)]);
|
||||
const updated = await statusCardService(db).getById(created.id);
|
||||
expect(updated?.generatingIssueId).toBeTruthy(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
expect(await db.select().from(statusCardUpdates).then(rows => rows.filter(row => row.generationIssueId === updated?.generatingIssueId)))
|
||||
.toHaveLength(1);
|
||||
});
|
||||
|
||||
it("finishes generation and writeback with a one-connection pool and releases dedicated locks", async () => {
|
||||
vi.useRealTimers();
|
||||
const previousDb = db;
|
||||
db = createDb(external ?? temporary!.connectionString, { maxConnections: 1, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 });
|
||||
try {
|
||||
const created = await card("reactive", { ...policy(), debounceSeconds: 1 });
|
||||
const countLocks = async () => Number((await db.execute<{ count: string }>(sql`
|
||||
select count(*) from pg_stat_activity where datname = current_database()
|
||||
and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count);
|
||||
const baseline = await countLocks();
|
||||
await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const holder = withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
|
||||
acquired.resolve(); await release.promise;
|
||||
}));
|
||||
await acquired.promise;
|
||||
try {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
expect(await service.next()).toBeGreaterThan(Date.now());
|
||||
} finally { release.resolve(); await holder; }
|
||||
await service.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
await vi.waitFor(async () => {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
}, { timeout: 5000, interval: 100 });
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
|
||||
await service.run(new AbortController().signal);
|
||||
expect(await service.next()).toBeNull();
|
||||
expect(idleWorkSnapshot().active).toBe(0);
|
||||
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
|
||||
// Failure exits also release the physical connection and advisory lock.
|
||||
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
|
||||
await service.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry");
|
||||
await vi.waitFor(async () => expect(await countLocks()).toBe(baseline));
|
||||
await withDedicatedDbConnection(db, connection => connection.transaction(async tx => {
|
||||
expect((await tx.execute<{ acquired: boolean }>(sql`select pg_try_advisory_xact_lock(hashtextextended(${created.id}, 0)) as acquired`))[0]!.acquired).toBe(true);
|
||||
}));
|
||||
} finally { await db.$client.end({ timeout: 1 }); db = previousDb; }
|
||||
}, 15_000);
|
||||
|
||||
it.each([false, true])("recovers a lost committed hint before a future membership deadline (legacy=%s)", async legacy => {
|
||||
await issueService(db).create(companyId, { title: "Original blocker", status: "blocked", priority: "medium" });
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1d" });
|
||||
const created = await card();
|
||||
expect(created.nextEvalReason).toBe("membership");
|
||||
const oldExpiry = created.nextEvalAt!.getTime();
|
||||
// No coordinator exists to receive this committed source hint. A fresh
|
||||
// service has no in-memory hint and must reconstruct from durable truth.
|
||||
await issueService(db).create(companyId, { title: "Committed before crash", status: "blocked", priority: "medium" });
|
||||
if (legacy) await db.update(statusCards).set({ nextEvalReason: null }).where(eq(statusCards.id, created.id));
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(1);
|
||||
expect(pending.nextEvalReason).toBe("policy");
|
||||
expect(pending.nextEvalAt!.getTime()).toBeLessThan(oldExpiry);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
await writeback(created.id, (await statusCardService(db).getById(created.id))!.generatingIssueId!);
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("membership");
|
||||
});
|
||||
|
||||
it("preserves a legacy ambiguous five-minute claim without extending it on restart", async () => {
|
||||
const created = await card();
|
||||
const retry = new Date(Date.now() + 240_000);
|
||||
await db.update(statusCards).set({ nextEvalAt: retry, nextEvalReason: null }).where(eq(statusCards.id, created.id));
|
||||
await issueService(db).create(companyId, { title: "Unconsumed source", status: "blocked", priority: "medium" });
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(retry.getTime());
|
||||
vi.setSystemTime(retry); await restarted.run(new AbortController().signal);
|
||||
expect((await statusCardService(db).getById(created.id))!.pendingChangeCount).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it.each(["debounce", "hours", "rate", "budget"] as const)("restores pending %s eligibility after restart without advancing it or acting early", async gate => {
|
||||
vi.setSystemTime(new Date("2026-11-02T18:00:00Z"));
|
||||
const refreshPolicy = { ...policy(), ...(gate === "hours" ? { activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } } : {}),
|
||||
...(gate === "rate" ? { maxUpdatesPerHour: 1 } : {}), ...(gate === "budget" ? { dailyTokenCap: 100 } : {}) };
|
||||
const created = await card("reactive", refreshPolicy);
|
||||
if (gate === "rate" || gate === "budget") await db.insert(statusCardUpdates).values({ cardId: created.id, kind: "full", trigger: "reactive",
|
||||
status: "ok", startedAt: new Date(Date.now() - (gate === "rate" ? 30 * 60_000 : 60_000)), finishedAt: new Date(), inputTokens: gate === "budget" ? 100 : 0 });
|
||||
await issueService(db).create(companyId, { title: "Saved blocker", status: "blocked", priority: "medium", createdByUserId: "board-user" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBeGreaterThan(0); expect(pending.nextEvalAt!.getTime()).toBeGreaterThan(Date.now());
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(pending.nextEvalAt!.getTime());
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toEqual(pending.nextEvalAt);
|
||||
vi.setSystemTime(pending.nextEvalAt!.getTime() - 1); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("unions startup reconstruction with a scoped source change in another company", async () => {
|
||||
const firstCompany = companyId;
|
||||
const first = await card();
|
||||
await issueService(db).create(companyId, { title: "First blocker", status: "blocked", priority: "medium" });
|
||||
companyId = randomUUID(); agentId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Other", issuePrefix: `X${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Other summarizer", role: "general", status: "idle", adapterType: "process", adapterConfig: {} });
|
||||
const second = await card();
|
||||
await issueService(db).create(companyId, { title: "Second blocker", status: "blocked", priority: "medium" });
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
service.changed(new Set([`changes:${companyId}`]));
|
||||
await service.run(new AbortController().signal);
|
||||
const cards = statusCardService(db);
|
||||
expect((await cards.getById(first.id))?.companyId).toBe(firstCompany);
|
||||
expect((await cards.getById(first.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
expect((await cards.getById(second.id))?.pendingChangeCount).toBeGreaterThan(0);
|
||||
await vi.advanceTimersByTimeAsync(60_000); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledTimes(2);
|
||||
expect(new Set(await db.select().from(agentWakeupRequests).then(rows => rows.map(row => row.companyId))))
|
||||
.toEqual(new Set([firstCompany, companyId]));
|
||||
});
|
||||
|
||||
it("keeps ignored issue activity quiet and reacts to human comments and document search membership", async () => {
|
||||
await start(); await card(); await quiet();
|
||||
await issueService(db).create(companyId, { title: "Ignored todo", status: "todo", priority: "medium" });
|
||||
await quiet(); expect(wakeup).not.toHaveBeenCalled();
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "all", q: "launch" });
|
||||
const searchCard = await card(); await quiet();
|
||||
const target = await issueService(db).create(companyId, { title: "Ordinary task", status: "todo", priority: "medium" });
|
||||
await quiet();
|
||||
await documentService(db).upsertIssueDocument({ issueId: target.id, key: "plan", format: "markdown", body: "launch evidence", createdByUserId: "board-user" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
await writeback(searchCard.id, (await statusCardService(db).getById(searchCard.id))!.generatingIssueId!); await quiet();
|
||||
await issueService(db).addComment(target.id, "A human launch update", { userId: "board-user" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(searchCard.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
});
|
||||
|
||||
it("retains relative query membership expiry across restart and emits the eventual removal", async () => {
|
||||
const now = new Date();
|
||||
const target = await issueService(db).create(companyId, { title: "Expiring blocker", status: "blocked", priority: "medium" });
|
||||
await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, target.id));
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h" });
|
||||
const created = await card();
|
||||
const expires = now.getTime() + 3_600_001;
|
||||
expect(created.nextEvalAt?.getTime()).toBe(expires);
|
||||
const restarted = statusCardEvaluationService(db, { wakeup });
|
||||
await restarted.run(new AbortController().signal);
|
||||
expect(await restarted.next()).toBe(expires);
|
||||
vi.setSystemTime(expires - 1); await restarted.run(new AbortController().signal); expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(expires); await restarted.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(1); expect(pending.nextEvalAt!.getTime()).toBe(expires + 60_000);
|
||||
vi.setSystemTime(pending.nextEvalAt!); await restarted.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("uses expiry of an off-page candidate to refresh offset-ranked query membership", async () => {
|
||||
const now = new Date();
|
||||
const tasks = await Promise.all((["critical", "high", "medium"] as const).map(priority => issueService(db).create(companyId, {
|
||||
title: `${priority} blocker`, priority, status: "blocked",
|
||||
})));
|
||||
await db.update(issues).set({ updatedAt: new Date(now.getTime() - 59 * 60_000) }).where(eq(issues.id, tasks[0]!.id));
|
||||
for (const task of tasks.slice(1)) await db.update(issues).set({ updatedAt: now }).where(eq(issues.id, task.id));
|
||||
compiledQuery = companySearchQuerySchema.parse({ scope: "issues", status: ["blocked"], updatedWithin: "1h", sort: "priority", limit: 1, offset: 1 });
|
||||
const created = await card();
|
||||
expect(Object.keys(created.fingerprint!)).toEqual([tasks[1]!.id]);
|
||||
const expires = now.getTime() + 60_001;
|
||||
expect(created.nextEvalAt?.getTime()).toBe(expires);
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
vi.setSystemTime(expires); await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBe(2); expect(wakeup).not.toHaveBeenCalled();
|
||||
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
const [update] = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.generationIssueId, (await statusCardService(db).getById(created.id))!.generatingIssueId!));
|
||||
expect(new Set(update!.changes.map(change => change.issueId))).toEqual(new Set([tasks[1]!.id, tasks[2]!.id]));
|
||||
});
|
||||
|
||||
it("retains a scoped retry while another evaluator owns the card lock", async () => {
|
||||
const created = await card();
|
||||
await issueService(db).create(companyId, { title: "Locked blocker", status: "blocked", priority: "medium" });
|
||||
const acquired = Promise.withResolvers<void>(), release = Promise.withResolvers<void>();
|
||||
const holder = db.transaction(async tx => {
|
||||
await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${created.id}, 0))`);
|
||||
acquired.resolve(); await release.promise;
|
||||
});
|
||||
await acquired.promise;
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
service.changed(new Set([`changes:${companyId}`]));
|
||||
try {
|
||||
await service.run(new AbortController().signal);
|
||||
expect(await service.next()).toBeGreaterThan(Date.now());
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
} finally { release.resolve(); await holder; }
|
||||
await vi.advanceTimersByTimeAsync(30_000); await service.run(new AbortController().signal);
|
||||
const pending = (await statusCardService(db).getById(created.id))!;
|
||||
expect(pending.pendingChangeCount).toBeGreaterThan(0);
|
||||
vi.setSystemTime(pending.nextEvalAt!); await service.run(new AbortController().signal);
|
||||
expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("retains the durable five-minute retry after evaluation failure across restart and drain", async () => {
|
||||
const created = await card();
|
||||
await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id));
|
||||
const service = statusCardEvaluationService(db, { wakeup });
|
||||
await service.run(new AbortController().signal);
|
||||
const failed = (await statusCardService(db).getById(created.id))!;
|
||||
const retryAt = failed.nextEvalAt!.getTime();
|
||||
expect(retryAt).toBeGreaterThan(Date.now() + 299_000); expect(wakeup).not.toHaveBeenCalled();
|
||||
await start(); expect(coordinator!.nextWakeAt()).toBe(retryAt);
|
||||
draining = true; await vi.advanceTimersByTimeAsync(5 * 60_000); await settled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
// Repair the controlled malformed stored-query fault; no tested result is precreated.
|
||||
await db.update(statusCards).set({ queries: [compiledQuery] }).where(eq(statusCards.id, created.id));
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await quiet();
|
||||
expect((await statusCardService(db).getById(created.id))!.nextEvalAt).toBeNull();
|
||||
});
|
||||
|
||||
it("cancels failed assignments without rerunning a completed generation and settles after retry", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
wakeup.mockRejectedValueOnce(new Error("assignment unavailable"));
|
||||
await issueService(db).create(companyId, { title: "Assignment blocker", status: "blocked", priority: "medium" });
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled();
|
||||
await vi.waitFor(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
expect((await statusCardService(db).getById(created.id))!.state).toBe("error");
|
||||
});
|
||||
const failed = (await statusCardService(db).getById(created.id))!;
|
||||
expect(failed).toMatchObject({ state: "error", generatingIssueId: null, nextEvalAt: null, nextEvalReason: null });
|
||||
expect(errors.length).toBeGreaterThan(0);
|
||||
const history = await db.select().from(statusCardUpdates).where(eq(statusCardUpdates.cardId, created.id));
|
||||
const ledger = history.find(row => row.status === "failed")!;
|
||||
expect(ledger.finishedAt).toBeInstanceOf(Date);
|
||||
expect((await db.select().from(issues).where(eq(issues.id, ledger.generationIssueId!)))[0]?.status).toBe("cancelled");
|
||||
const count = wakeup.mock.calls.length; await quiet(); expect(wakeup).toHaveBeenCalledTimes(count);
|
||||
// An explicit ordinary refresh is still the user's supported retry action.
|
||||
const retry = await statusCardService(db).requestRefresh(created.id, { trigger: "manual", actor });
|
||||
expect(retry.generatingIssue).toBeTruthy(); await writeback(created.id, retry.generatingIssue!.id); await quiet();
|
||||
expect((await statusCardService(db).getById(created.id))!.state).toBe("active");
|
||||
});
|
||||
|
||||
it("budget-owned company resume reactivates reactive work committed while paused", async () => {
|
||||
await start(); const created = await card(); await quiet();
|
||||
const budgets = budgetService(db);
|
||||
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 100, hardStopEnabled: true, notifyEnabled: false }, "board-user");
|
||||
const [cost] = await db.insert(costEvents).values({ companyId, agentId, provider: "openai", biller: "openai", billingType: "metered_api", model: "test",
|
||||
costCents: 200, occurredAt: new Date() }).returning();
|
||||
await budgets.evaluateCostEvent(cost!);
|
||||
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("paused");
|
||||
await quiet();
|
||||
await issueService(db).create(companyId, { title: "Paused blocker", status: "blocked", priority: "medium" }); await quiet();
|
||||
expect(wakeup).not.toHaveBeenCalled();
|
||||
await budgets.upsertPolicy(companyId, { scopeType: "company", scopeId: companyId, amount: 1000 }, "board-user");
|
||||
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]?.status).toBe("active");
|
||||
await vi.waitFor(async () => expect((await statusCardService(db).getById(created.id))?.pendingChangeCount).toBeGreaterThan(0));
|
||||
await vi.advanceTimersByTimeAsync(60_000); await settled(); expect(wakeup).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("preserves the first write during drain until resume", async () => {
|
||||
await start(); const created = await card("manual"); await quiet();
|
||||
draining = true;
|
||||
await statusCardService(db).update(created, { refreshPolicy: policy("interval") }, actor);
|
||||
await settled(); expect(wakeup).not.toHaveBeenCalled();
|
||||
draining = false; await vi.advanceTimersByTimeAsync(30_000); await settled();
|
||||
expect(coordinator!.nextWakeAt()).not.toBeNull(); expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { installDatabaseWorkSignals, subscribeDatabaseWork } from "../../../packages/db/src/work-signals.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { notifyExecutionRunWork, writeExecutionControl } from "../services/execution-control-notifications.js";
|
||||
|
||||
it.each([false, true])("keeps execution queue consumers intact and emits only a known status-card company scope (%s)", async scoped => {
|
||||
const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []), transaction: async (fn: (tx: object) => Promise<unknown>) =>
|
||||
fn({ execute: async () => [{ xid: "1" }] }) });
|
||||
const queues = [DELIVERY_QUEUES.executionStatus, DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.executionReplacement,
|
||||
DELIVERY_QUEUES.executionDisposition, DELIVERY_QUEUES.issueMonitors];
|
||||
const settled: Array<{ queue: string; hints?: string[] }> = [];
|
||||
const unsubscribe = [...queues, DELIVERY_QUEUES.statusCardEvaluation, DELIVERY_QUEUES.workspaceRecovery].map(queue => subscribeDatabaseWork(owner, queue, (event, hints) => {
|
||||
if (event === "settled") settled.push({ queue, hints: hints ? [...hints] : undefined });
|
||||
}));
|
||||
try {
|
||||
const scope = scoped ? { companyId: "changed-company", runId: "changed-run" } : undefined;
|
||||
await owner.transaction(async tx => {
|
||||
await notifyExecutionRunWork(tx, scope);
|
||||
expect(settled).toEqual([]);
|
||||
});
|
||||
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([...queues, ...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
|
||||
expect(settled.find(row => row.queue === DELIVERY_QUEUES.statusCardEvaluation)?.hints)
|
||||
.toEqual(scoped ? ["changes:changed-company"] : undefined);
|
||||
settled.length = 0;
|
||||
const write = vi.fn(async () => "written");
|
||||
expect(await writeExecutionControl(owner as never, DELIVERY_QUEUES.workspaceRecovery, write, scope)).toBe("written");
|
||||
expect(write).toHaveBeenCalledOnce();
|
||||
expect(new Set(settled.map(row => row.queue))).toEqual(new Set([DELIVERY_QUEUES.issueMonitors, DELIVERY_QUEUES.workspaceRecovery,
|
||||
...(scoped ? [DELIVERY_QUEUES.statusCardEvaluation] : [])]));
|
||||
} finally { for (const stop of unsubscribe) stop(); }
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { statusCardRefreshPolicySchema } from "@paperclipai/shared";
|
||||
import {
|
||||
chooseStatusCardUpdateKind,
|
||||
@@ -8,10 +8,12 @@ import {
|
||||
filterStatusCardChanges,
|
||||
isWithinStatusCardActiveHours,
|
||||
nextStatusCardEvaluationAt,
|
||||
nextStatusCardPolicyEligibility,
|
||||
statusCardChangesHash,
|
||||
} from "../services/status-card-update-engine.js";
|
||||
|
||||
describe("status card update engine", () => {
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
const defaultPolicy = statusCardRefreshPolicySchema.parse({ mode: "interval", intervalMinutes: 15 });
|
||||
|
||||
it("retains non-terminal and terminal status transitions plus membership changes", () => {
|
||||
@@ -78,6 +80,149 @@ describe("status card update engine", () => {
|
||||
expect(evaluateStatusCardPolicy({ policy: hours, now: new Date("2026-07-23T18:00:00.000Z"), lastChangeAt: null, updatesLastHour: 0, tokensToday: 0, manual: false }).action).toBe("pause_hours");
|
||||
});
|
||||
|
||||
it("disarms unchanged reactive evaluation while retaining exact pending eligibility", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 90, maxUpdatesPerHour: 2,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
|
||||
const now = new Date("2026-07-23T16:59:30Z");
|
||||
expect(nextStatusCardEvaluationAt(policy, now)).toBeNull();
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 }))
|
||||
.toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
const open = new Date("2026-07-23T14:00:00Z");
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: new Date("2026-07-23T13:00:00Z"),
|
||||
completedUpdateTimes: [new Date("2026-07-23T13:30:00Z"), new Date("2026-07-23T13:45:00Z")], tokensToday: 0 }))
|
||||
.toEqual(new Date("2026-07-23T14:30:00.001Z"));
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now: open, lastChangeAt: open, completedUpdateTimes: [], tokensToday: 100_000 }))
|
||||
.toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
});
|
||||
|
||||
it("retains empty-window pending changes without a recurring eligibility scan", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "09:00", timezone: "UTC" } });
|
||||
const now = new Date("2026-07-23T14:00:00Z");
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
|
||||
});
|
||||
|
||||
it("bounds empty-window formatter work before any timezone conversion", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "09:00", timezone: "Pacific/Chatham" } });
|
||||
const formatter = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function () {
|
||||
throw new Error("Empty active hours must not construct a formatter");
|
||||
});
|
||||
const now = new Date("2026-07-23T14:00:00Z");
|
||||
expect(isWithinStatusCardActiveHours(policy, now)).toBe(false);
|
||||
expect(nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: now, completedUpdateTimes: [], tokensToday: 0 })).toBeNull();
|
||||
expect(formatter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds formatter construction and reuses the next opening across cards and source writes", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "America/St_Johns" } });
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
let constructions = 0;
|
||||
vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
if (++constructions > 1) throw new Error("Card evaluation repeatedly constructed timezone formatters");
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
const conversions = vi.spyOn(RealFormatter.prototype, "formatToParts");
|
||||
const now = new Date("2026-07-23T22:00:00Z");
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
const started = performance.now();
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("2026-07-24T11:30:00Z"));
|
||||
const coldMs = performance.now() - started;
|
||||
const coldConversions = conversions.mock.calls.length;
|
||||
expect(coldConversions).toBeLessThanOrEqual(2881);
|
||||
conversions.mockClear();
|
||||
const warmStarted = performance.now();
|
||||
for (let write = 0; write < 100; write++) for (let card = 0; card < 20; card++) {
|
||||
expect(nextStatusCardPolicyEligibility({ ...input,
|
||||
policy: { ...policy, activeHours: { ...policy.activeHours! } },
|
||||
now: new Date(now.getTime() + write * 1000),
|
||||
})).toEqual(new Date("2026-07-24T11:30:00Z"));
|
||||
}
|
||||
expect(constructions).toBeLessThanOrEqual(1);
|
||||
expect(conversions.mock.calls.length).toBeLessThanOrEqual(2000);
|
||||
process.stdout.write(JSON.stringify({ measurement: "status-card active-hours cost", coldMs, coldConversions, constructions,
|
||||
warmMs: performance.now() - warmStarted, warmConversions: conversions.mock.calls.length, evaluations: 2000 }) + "\n");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["America/New_York", "02:15", "03:30", "2026-03-08T06:50:00Z", "2026-03-08T07:00:00Z"],
|
||||
["America/New_York", "02:15", "02:45", "2026-03-08T06:50:00Z", "2026-03-09T06:15:00Z"],
|
||||
["America/New_York", "02:15", "02:45", "2026-03-07T07:45:00Z", "2026-03-09T06:15:00Z"],
|
||||
["America/New_York", "01:15", "01:45", "2026-11-01T05:50:00Z", "2026-11-01T06:15:00Z"],
|
||||
["America/New_York", "23:30", "01:30", "2026-11-01T05:40:00Z", "2026-11-01T06:00:00Z"],
|
||||
["Australia/Lord_Howe", "02:15", "02:45", "2026-10-03T15:20:00Z", "2026-10-03T15:30:00Z"],
|
||||
["Australia/Lord_Howe", "01:35", "01:45", "2026-04-04T14:50:00Z", "2026-04-04T15:05:00Z"],
|
||||
["Asia/Kathmandu", "09:00", "17:00", "2026-07-23T12:00:00Z", "2026-07-24T03:15:00Z"],
|
||||
["Pacific/Apia", "00:10", "00:20", "2011-12-30T09:50:00Z", "2011-12-30T10:10:00Z"],
|
||||
])("preserves the first real opening in %s %s-%s after %s", (timezone, start, end, time, expected) => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1, activeHours: { start, end, timezone } });
|
||||
const now = new Date(time);
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date(expected));
|
||||
// A later closed instant reuses the opening, while the exact opening itself
|
||||
// remains eligible without rounding it into the next minute or day.
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(now.getTime() + 1000) })).toEqual(new Date(expected));
|
||||
const opening = new Date(expected);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: opening })).toEqual(opening);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: new Date(opening.getTime() + 1) })).toEqual(new Date(opening.getTime() + 1));
|
||||
});
|
||||
|
||||
it("invalidates an upcoming opening for clock rollback and changed policy values", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone: "UTC" } });
|
||||
const calculate = (time: string, activeHours = policy.activeHours!) => {
|
||||
const now = new Date(time);
|
||||
return nextStatusCardPolicyEligibility({ policy: { ...policy, activeHours }, now,
|
||||
lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
|
||||
};
|
||||
expect(calculate("2026-07-23T18:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T08:00:00Z")).toEqual(new Date("2026-07-23T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T17:00:00Z")).toEqual(new Date("2026-07-24T09:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, start: "10:00" })).toEqual(new Date("2026-07-24T10:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, end: "19:00" })).toEqual(new Date("2026-07-23T18:00:00Z"));
|
||||
expect(calculate("2026-07-23T18:00:00Z", { ...policy.activeHours!, timezone: "Asia/Tokyo" })).toEqual(new Date("2026-07-24T00:00:00Z"));
|
||||
});
|
||||
|
||||
it("checks the precise instant before a cached minute opening for historical second offsets", () => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "10:00", timezone: "Africa/Monrovia" } });
|
||||
const now = new Date("1970-01-01T08:00:00Z");
|
||||
const input = { policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 };
|
||||
expect(nextStatusCardPolicyEligibility(input)).toEqual(new Date("1970-01-01T09:45:00Z"));
|
||||
const inside = new Date("1970-01-01T09:44:45Z");
|
||||
expect(isWithinStatusCardActiveHours(policy, inside)).toBe(true);
|
||||
expect(nextStatusCardPolicyEligibility({ ...input, now: inside })).toEqual(inside);
|
||||
});
|
||||
|
||||
it("bounds formatter and upcoming-opening cache retention", () => {
|
||||
const now = new Date("2026-07-25T00:00:00Z");
|
||||
const timezonePolicy = (timezone: string) => statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: "09:00", end: "17:00", timezone } });
|
||||
const zones = Intl.supportedValuesOf("timeZone").slice(0, 34);
|
||||
const first = timezonePolicy(zones[0]!);
|
||||
isWithinStatusCardActiveHours(first, now);
|
||||
for (const zone of zones.slice(1)) isWithinStatusCardActiveHours(timezonePolicy(zone), now);
|
||||
const RealFormatter = Intl.DateTimeFormat;
|
||||
const constructor = vi.spyOn(Intl, "DateTimeFormat").mockImplementation(function (...args: ConstructorParameters<typeof Intl.DateTimeFormat>) {
|
||||
return new RealFormatter(...args);
|
||||
} as unknown as typeof Intl.DateTimeFormat);
|
||||
isWithinStatusCardActiveHours(first, now);
|
||||
expect(constructor).toHaveBeenCalledTimes(1);
|
||||
constructor.mockRestore();
|
||||
const time = (minute: number) => `${String(Math.floor(minute / 60)).padStart(2, "0")}:${String(minute % 60).padStart(2, "0")}`;
|
||||
const calculate = (minute: number) => {
|
||||
const policy = statusCardRefreshPolicySchema.parse({ mode: "reactive", debounceSeconds: 1,
|
||||
activeHours: { start: time(minute), end: time(minute + 1), timezone: "UTC" } });
|
||||
return nextStatusCardPolicyEligibility({ policy, now, lastChangeAt: new Date(now.getTime() - 60_000), completedUpdateTimes: [], tokensToday: 0 });
|
||||
};
|
||||
calculate(1);
|
||||
for (let minute = 2; minute <= 258; minute++) calculate(minute);
|
||||
const conversions = vi.spyOn(Intl.DateTimeFormat.prototype, "formatToParts");
|
||||
expect(calculate(1)).toEqual(new Date("2026-07-25T00:01:00Z"));
|
||||
expect(conversions.mock.calls.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("selects full rebuilds for bounded drift rules and incremental otherwise", () => {
|
||||
const base = { hasDocument: true, changeCount: 2, queryVersion: 3, lastUpdateQueryVersion: 3, incrementalCount: 2, configurationChanged: false };
|
||||
expect(chooseStatusCardUpdateKind(base)).toBe("incremental");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
@@ -37,7 +38,8 @@ import {
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_STATUS_CARDS_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
type Db = ReturnType<typeof createDb>;
|
||||
@@ -79,8 +81,8 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-status-cards-");
|
||||
db = createDb(externalDatabaseUrl ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -100,6 +102,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
@@ -189,6 +192,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
.send({ title: "Launch health", titlePinned: true });
|
||||
expect(patched.status).toBe(200);
|
||||
expect(patched.body).toMatchObject({ title: "Launch health", titlePinned: true });
|
||||
expect(patched.body).not.toHaveProperty("nextEvalReason");
|
||||
|
||||
const scheduled = await request(app)
|
||||
.patch(`/api/status-cards/${created.body.id}`)
|
||||
@@ -1149,6 +1153,7 @@ describeEmbeddedPostgres("status card routes", () => {
|
||||
});
|
||||
expect(summaryWrite.status).toBe(200);
|
||||
expect(summaryWrite.body.card).toMatchObject({ state: "active", queryVersion: 1, generatingIssueId: null });
|
||||
expect(summaryWrite.body.card).not.toHaveProperty("nextEvalReason");
|
||||
expect(summaryWrite.body.document.latestBody).toContain("**Decide:**");
|
||||
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "full"))).toMatchObject({ status: "ok", finishedAt: expect.any(Date), inputTokens: 5200, outputTokens: 980 });
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { PgDialect } from "drizzle-orm/pg-core";
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
enrichWorkProductMetadataWithDiff,
|
||||
@@ -33,6 +35,18 @@ function createWorkProductRow(overrides: Partial<Record<string, unknown>> = {})
|
||||
};
|
||||
}
|
||||
|
||||
function retentionReads(insideTransaction: () => boolean) {
|
||||
return vi.fn(async (query: SQL) => {
|
||||
const statement = new PgDialect().sqlToQuery(query);
|
||||
expect(insideTransaction()).toBe(true);
|
||||
expect(statement.params).toContain("company-1");
|
||||
expect(statement.params).toContain("issue-1");
|
||||
if (statement.sql.includes("origin_kind")) return [{ id: "issue-1", originKind: null, originId: null }];
|
||||
if (statement.sql.includes("with recursive affected")) return [{ issueId: "issue-1" }];
|
||||
throw new Error(`Unexpected work-product fixture query: ${statement.sql}`);
|
||||
});
|
||||
}
|
||||
|
||||
describe("workProductService", () => {
|
||||
it("extracts runner totals and enriches work-product metadata", () => {
|
||||
const summary = workProductDiffSummaryFromEventPayload({
|
||||
@@ -158,11 +172,16 @@ describe("workProductService", () => {
|
||||
const insertValues = vi.fn(() => ({ returning: insertReturning }));
|
||||
const txInsert = vi.fn(() => ({ values: insertValues }));
|
||||
|
||||
let transactionOpen = false;
|
||||
const tx = {
|
||||
execute: retentionReads(() => transactionOpen),
|
||||
update: txUpdate,
|
||||
insert: txInsert,
|
||||
};
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
|
||||
transactionOpen = true;
|
||||
try { return await callback(tx); } finally { transactionOpen = false; }
|
||||
});
|
||||
|
||||
const svc = workProductService({ transaction } as any);
|
||||
const result = await svc.createForIssue("issue-1", "company-1", {
|
||||
@@ -194,11 +213,16 @@ describe("workProductService", () => {
|
||||
const updateSet = vi.fn(() => ({ where: updateWhere }));
|
||||
const txUpdate = vi.fn(() => ({ set: updateSet }));
|
||||
|
||||
let transactionOpen = false;
|
||||
const tx = {
|
||||
execute: retentionReads(() => transactionOpen),
|
||||
select: txSelect,
|
||||
update: txUpdate,
|
||||
};
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => await callback(tx));
|
||||
const transaction = vi.fn(async (callback: (input: typeof tx) => Promise<unknown>) => {
|
||||
transactionOpen = true;
|
||||
try { return await callback(tx); } finally { transactionOpen = false; }
|
||||
});
|
||||
|
||||
const svc = workProductService({ transaction } as any);
|
||||
const result = await svc.update("work-product-1", {
|
||||
|
||||
+34
-203
@@ -1,4 +1,9 @@
|
||||
import { registerGitHubConnectionEventWork, registerGitHubConnectionContinuityWork } from "./services/github-connection-work.js";
|
||||
import { registerNativeHarnessBackupExpirationWork } from "./services/native-runtime/native-harness-backup-expiration-work.js";
|
||||
import { registerMergeConfirmationWork } from "./services/merged-pr-confirmation-work.js";
|
||||
import { registerRoutineExecutionStatusWork } from "./services/routine-execution-status-work.js";
|
||||
import { routineExecutionStatusService } from "./services/routine-run-status.js";
|
||||
import { attentionRetentionService, registerAttentionRetentionWork } from "./services/attention-retention.js";
|
||||
import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js";
|
||||
import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js";
|
||||
import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js";
|
||||
@@ -33,8 +38,6 @@ import {
|
||||
skillVersionRetentionService,
|
||||
} from "./services/skill-version-retention.js";
|
||||
import {
|
||||
NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS,
|
||||
NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS,
|
||||
nativeHarnessBackupExpirationService,
|
||||
} from "./services/native-runtime/native-harness-backup-expiration.js";
|
||||
import { registerNativeRunnerEvidenceRetentionWork } from "./services/native-runtime/native-runner-evidence-retention-work.js";
|
||||
@@ -110,7 +113,6 @@ import { createApp } from "./app.js";
|
||||
import { initializeCloudRuntimeConfig } from "./services/cloud-runtime-config.js";
|
||||
import { loadCloudBootstrapConfig, loadConfig, type Config } from "./config.js";
|
||||
import { logger } from "./middleware/logger.js";
|
||||
import { createIncompleteTaskWatchdogScanReporter } from "./services/task-watchdog-diagnostics.js";
|
||||
import { setStartupRecoveryPhase } from "./startup-recovery-state.js";
|
||||
import {
|
||||
StartupRefusalError,
|
||||
@@ -128,7 +130,6 @@ import { cloudActorHeaderSourceFromHeaders, resolveCloudTenantActor } from "./mi
|
||||
import {
|
||||
feedbackService,
|
||||
applyManagedEnvironments,
|
||||
attentionService,
|
||||
backfillPrincipalAccessCompatibility,
|
||||
backfillLegacyToolOAuthTokens,
|
||||
bootstrapExecutionPolicyFromEnv,
|
||||
@@ -146,13 +147,12 @@ import {
|
||||
reconcileCodexLocalManagedHomesOnStartup,
|
||||
reconcilePersistedRuntimeServicesOnStartup,
|
||||
routineService,
|
||||
statusCardService,
|
||||
toolAccessService,
|
||||
workspaceOperationService,
|
||||
} from "./services/index.js";
|
||||
import { questionResponseDeliveryService } from "./services/question-response-delivery.js";
|
||||
import { deliverNativeQuestionResponse } from "./services/native-runtime/native-question-bridge.js";
|
||||
import { queueIssueAssignmentWakeup } from "./services/issue-assignment-wakeup.js";
|
||||
import { registerStatusCardEvaluationWork, statusCardEvaluationService } from "./services/status-card-evaluation.js";
|
||||
import { createSecretProposalsService } from "./services/secret-proposals.js";
|
||||
import { environmentRuntimeService } from "./services/environment-runtime.js";
|
||||
import { historicalCleanupVerificationService } from "./services/historical-cleanup-verification.js";
|
||||
@@ -974,9 +974,6 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
};
|
||||
const pluginWorkerManager = createPluginWorkerManager();
|
||||
const reportIncompleteWatchdogScans = createIncompleteTaskWatchdogScanReporter(
|
||||
(details, message) => logger.warn(details, message),
|
||||
);
|
||||
const heartbeat = config.heartbeatSchedulerEnabled
|
||||
? heartbeatService(db as any, { pluginWorkerManager })
|
||||
: null;
|
||||
@@ -1296,47 +1293,10 @@ async function startServerWithDatabaseTeardown(
|
||||
evidenceRetentionDays: config.nativeRunnerEvidenceRetentionDays,
|
||||
shouldContinue: () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(),
|
||||
});
|
||||
let nativeHarnessBackupExpirationInFlight = false;
|
||||
let nativeHarnessBackupExpirationContinuation: ReturnType<typeof setTimeout> | undefined;
|
||||
const sweepNativeHarnessBackups = () => {
|
||||
if (!config.nativeHarnessBackupExpirationEnabled || nativeHarnessBackupExpirationInFlight || isIdleTaskDrainActive() || heartbeatSchedulerStopped || isWarmStandby()) return;
|
||||
nativeHarnessBackupExpirationInFlight = true;
|
||||
trackHeartbeatSchedulerWork(nativeHarnessBackupExpiration.tick()
|
||||
.then((result) => {
|
||||
if (result.candidates > 0 || result.evidenceCandidates > 0 || result.failed > 0) {
|
||||
logger.info({ ...result, dryRun: config.nativeHarnessBackupExpirationDryRun }, "native harness backup expiration");
|
||||
}
|
||||
if (result.truncated && !heartbeatSchedulerStopped) {
|
||||
nativeHarnessBackupExpirationContinuation = setTimeout(sweepNativeHarnessBackups, 1000);
|
||||
nativeHarnessBackupExpirationContinuation.unref?.();
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "native harness backup expiration failed");
|
||||
})
|
||||
.finally(() => { nativeHarnessBackupExpirationInFlight = false; }));
|
||||
};
|
||||
const nativeHarnessBackupExpirationInterval = setInterval(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_INTERVAL_MS);
|
||||
nativeHarnessBackupExpirationInterval.unref?.();
|
||||
const nativeHarnessBackupExpirationStartup = setTimeout(sweepNativeHarnessBackups, NATIVE_HARNESS_BACKUP_EXPIRATION_STARTUP_DELAY_MS);
|
||||
nativeHarnessBackupExpirationStartup.unref?.();
|
||||
const externalObjects = externalObjectService(db as any, {
|
||||
pluginWorkerManager,
|
||||
enabled: async () => (await instanceSettingsService(db).getExperimental()).enableExternalObjects === true,
|
||||
});
|
||||
const scheduleExternalObjectRefreshSweep = (now = new Date()) => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork(externalObjects
|
||||
.refreshDueObjectsForActiveCompanies(50, now)
|
||||
.then((result) => {
|
||||
if (result.checked > 0 || result.refreshed > 0) {
|
||||
logger.info({ ...result }, "external-object scheduler tick refreshed due objects");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "external-object scheduler tick failed");
|
||||
}));
|
||||
};
|
||||
|
||||
// The retry backstop for orphan sandboxes. An acquire that rejects a
|
||||
// foreign-company insert tears the provisioned sandbox down. If that teardown
|
||||
@@ -1363,8 +1323,16 @@ async function startServerWithDatabaseTeardown(
|
||||
resolveNativeQuestion: (interaction) => deliverNativeQuestionResponse(db as any, interaction),
|
||||
});
|
||||
const deliveryWork = app.locals.deliveryWork as ReturnType<typeof createDeliveryWorkCoordinator>;
|
||||
externalObjects.registerRefreshWork(deliveryWork,
|
||||
() => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby());
|
||||
if (config.nativeHarnessBackupExpirationEnabled) {
|
||||
nativeHarnessBackupExpiration.registerCheckpointExpiration(deliveryWork);
|
||||
const backupExpirationWork = registerNativeHarnessBackupExpirationWork(deliveryWork, db as any, {
|
||||
dryRun: config.nativeHarnessBackupExpirationDryRun,
|
||||
closedForDays: config.nativeHarnessBackupExpirationDays,
|
||||
}, () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby());
|
||||
trackHeartbeatSchedulerWork(backupExpirationWork.ready);
|
||||
server.on("close", backupExpirationWork.stop);
|
||||
}
|
||||
const taskDrainExpiryWork = registerTaskDrainExpiryWork(deliveryWork, () => {
|
||||
if (heartbeatSchedulerStopped || isWarmStandby()) return;
|
||||
@@ -1543,16 +1511,7 @@ async function startServerWithDatabaseTeardown(
|
||||
heartbeat.drainActiveRunExecutions();
|
||||
prepareHotRestartShutdown = heartbeat.prepareHotRestartShutdown;
|
||||
const routines = routineService(db as any, { pluginWorkerManager });
|
||||
let nextRoutineStatusReconcile = 0;
|
||||
const scheduleRoutineStatusReconcile = () => {
|
||||
if (!isCloudManagedInstance() || isWarmStandby() || isIdleTaskDrainActive() || heartbeatSchedulerStopped ||
|
||||
Date.now() < nextRoutineStatusReconcile) return;
|
||||
nextRoutineStatusReconcile = Date.now() + 5 * 60_000;
|
||||
trackHeartbeatSchedulerWork(routines.reconcileExecutionRunStatuses().then(result => {
|
||||
if (result.settled > 0) logger.info(result, "reconciled routine execution issue status");
|
||||
}).catch(() => logger.warn("routine execution status reconciliation deferred")));
|
||||
};
|
||||
const statusCards = statusCardService(db as any);
|
||||
const statusCardEvaluation = statusCardEvaluationService(db as any, heartbeat);
|
||||
const issues = issueService(db as any);
|
||||
const mergedPullRequestConfirmations = issueThreadInteractionService(db as any, {
|
||||
wakeup: heartbeat.wakeup,
|
||||
@@ -1583,19 +1542,8 @@ async function startServerWithDatabaseTeardown(
|
||||
: config.workspaceCompactionDelayHours * 60 * 60 * 1000,
|
||||
terminalWorkspaceInspectionBackoffMs: TERMINAL_WORKSPACE_RETRY_MS,
|
||||
});
|
||||
const scheduleMergedPullRequestConfirmationSweep = () => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork(mergedPullRequestConfirmations
|
||||
.sweepMergedPullRequestConfirmations()
|
||||
.then((result) => {
|
||||
if (result.accepted > 0) {
|
||||
logger.info(result, "accepted merge confirmations for merged pull requests");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "merged pull-request confirmation sweep failed");
|
||||
}));
|
||||
};
|
||||
registerMergeConfirmationWork(deliveryWork, mergedPullRequestConfirmations,
|
||||
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby());
|
||||
const gitRestorePointWorker = registerGitRestorePointWork(deliveryWork, gitRestorePointRetentionService(db as any, {
|
||||
storage: storageService,
|
||||
retentionDays: config.gitRestorePointRetentionDays,
|
||||
@@ -1756,8 +1704,6 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
}
|
||||
|
||||
const promotion = await timeStartupRecovery("scheduled_retries", () => heartbeat.promoteDueScheduledRetries());
|
||||
await timeStartupRecovery("queued_runs", () => heartbeat.resumeQueuedRuns());
|
||||
const recoveredGoalActions = await timeStartupRecovery("goal_actions", () => heartbeat.recoverPendingSessionGoalActions());
|
||||
if (
|
||||
recoveredGoalActions.enqueued > 0 ||
|
||||
@@ -1775,48 +1721,10 @@ async function startServerWithDatabaseTeardown(
|
||||
"startup session-goal recovery resumed durable agent goals",
|
||||
);
|
||||
}
|
||||
const reconciled = await heartbeat.reconcileStrandedAssignedIssues();
|
||||
if (
|
||||
promotion.promoted > 0 ||
|
||||
reconciled.assignmentDispatched > 0 ||
|
||||
reconciled.dispatchRequeued > 0 ||
|
||||
reconciled.continuationRequeued > 0 ||
|
||||
reconciled.successfulRunHandoffEscalated > 0 ||
|
||||
reconciled.successfulRunHandoffRetried > 0 ||
|
||||
reconciled.escalated > 0
|
||||
) {
|
||||
logger.warn(
|
||||
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
|
||||
"startup heartbeat recovery changed assigned issue state",
|
||||
);
|
||||
}
|
||||
|
||||
const dependencyWakesReconciled = await heartbeat.reconcileResolvedDependencyWakes();
|
||||
if (dependencyWakesReconciled.healed > 0) {
|
||||
logger.warn(
|
||||
{ ...dependencyWakesReconciled },
|
||||
"startup dependency-wake reconciliation restored task execution paths",
|
||||
);
|
||||
}
|
||||
|
||||
const taskWatchdogsReconciled = await heartbeat.reconcileTaskWatchdogs();
|
||||
reportIncompleteWatchdogScans(taskWatchdogsReconciled, "startup");
|
||||
if (taskWatchdogsReconciled.triggered > 0) {
|
||||
logger.warn(
|
||||
{ ...taskWatchdogsReconciled },
|
||||
"startup task-watchdog reconciliation triggered watchdog work",
|
||||
);
|
||||
}
|
||||
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "startup active-run output watchdog created review work");
|
||||
}
|
||||
|
||||
const swept = await heartbeat.sweepStaleIssueLocks();
|
||||
if (swept.cleared > 0) {
|
||||
logger.warn({ ...swept }, "startup stale-lock sweeper cleared issue locks");
|
||||
}
|
||||
})().catch((err) => {
|
||||
logger.error({ err }, "startup heartbeat recovery failed");
|
||||
throw err;
|
||||
@@ -1825,6 +1733,12 @@ async function startServerWithDatabaseTeardown(
|
||||
await startupHeartbeatRecovery;
|
||||
}
|
||||
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerIssueStateRecoveryWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerDeferredReceiptWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerRunDispatchWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready;
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerScheduledRetryCancellationWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
|
||||
|
||||
@@ -1904,23 +1818,6 @@ async function startServerWithDatabaseTeardown(
|
||||
if (config.heartbeatSchedulerEnabled) await heartbeat.registerStoppedInstructionCollectionWork(deliveryWork,
|
||||
() => !isIdleTaskDrainActive() && !heartbeatSchedulerStopped && !isWarmStandby()).ready;
|
||||
|
||||
const runRetentionSweep = async () => {
|
||||
const activeCompanies = await db.select({ id: companies.id }).from(companies).where(eq(companies.status, "active"));
|
||||
let archived = 0;
|
||||
for (const company of activeCompanies) {
|
||||
// Cursor pagination rebuilds the whole feed for every page; one
|
||||
// unscoped all-items build keeps this sweep at a single feed build
|
||||
// per company per tick.
|
||||
const page = await attentionService(db as any).list(company.id, {
|
||||
includeDismissed: true,
|
||||
excludeSources: ["approval", "join_request"],
|
||||
all: true,
|
||||
allowUnscopedAll: true,
|
||||
});
|
||||
archived += await retentionExecutor.autoArchive({ companyId: company.id, items: page.items });
|
||||
}
|
||||
return { archived };
|
||||
};
|
||||
// Startup housekeeping: provider and connection sweeps that every timer tick
|
||||
// repeats and that neither serving requests nor run recovery depends on.
|
||||
// Readiness does not wait for it. It is tracked as scheduler work, so idle
|
||||
@@ -1963,7 +1860,10 @@ async function startServerWithDatabaseTeardown(
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("join_request_retention", () => registerJoinRequestRetentionWork(deliveryWork, joinRequestRetentionService(db as any),
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("retention_sweep", runRetentionSweep);
|
||||
await step("status_card_evaluation", () => registerStatusCardEvaluationWork(deliveryWork, statusCardEvaluation,
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
await step("attention_retention", () => registerAttentionRetentionWork(deliveryWork, attentionRetentionService(db as any),
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped).ready);
|
||||
};
|
||||
|
||||
const createHeartbeatSchedulerTick = (scheduled: ReturnType<typeof registerScheduledWork>) => () => {
|
||||
@@ -1972,10 +1872,6 @@ async function startServerWithDatabaseTeardown(
|
||||
// captures the authoritative set of running heartbeat rows.
|
||||
trackHeartbeatSchedulerWork((async () => {
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
scheduleRoutineStatusReconcile();
|
||||
trackHeartbeatSchedulerWork(runRetentionSweep().catch((err: unknown) => {
|
||||
logger.error({ err }, "decision retention sweep failed");
|
||||
}));
|
||||
const sweptRuntimeStatuses = heartbeat.sweepExpiredRuntimeStatuses();
|
||||
if (sweptRuntimeStatuses > 0) {
|
||||
logger.info(
|
||||
@@ -1985,90 +1881,24 @@ async function startServerWithDatabaseTeardown(
|
||||
}
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
scheduleExternalObjectRefreshSweep(new Date());
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
scheduleMergedPullRequestConfirmationSweep();
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
trackHeartbeatSchedulerWork((async () => {
|
||||
const experimental = await instanceSettingsService(db).getExperimental();
|
||||
if (experimental.enableStatusCards !== true) return;
|
||||
const result = await statusCards.tickDueStatusCards(new Date());
|
||||
await Promise.all(result.enqueued.map(async ({ cardId, generatingIssue }) => {
|
||||
try {
|
||||
await queueIssueAssignmentWakeup({
|
||||
heartbeat,
|
||||
issue: generatingIssue,
|
||||
reason: "status_card_update_assigned",
|
||||
mutation: "status_card.scheduler_update_requested",
|
||||
contextSource: "status_card_scheduler",
|
||||
requestedByActorType: "system",
|
||||
taskKey: `status-card:${cardId}`,
|
||||
rethrowOnError: true,
|
||||
});
|
||||
} catch (err) {
|
||||
await issues.update(generatingIssue.id, { status: "cancelled" });
|
||||
throw err;
|
||||
}
|
||||
}));
|
||||
if (result.evaluated > 0 || result.enqueued.length > 0) {
|
||||
logger.info({ evaluated: result.evaluated, enqueued: result.enqueued.length }, "status-card scheduler tick complete");
|
||||
}
|
||||
})().catch((err) => {
|
||||
logger.error({ err }, "status-card scheduler tick failed");
|
||||
}));
|
||||
|
||||
|
||||
if (isIdleTaskDrainActive() || heartbeatSchedulerStopped) return;
|
||||
if (!(await heartbeat.resolveSchedulingSuppression()).suppressed) {
|
||||
// Periodically reap orphaned runs (5-min staleness threshold) and make sure
|
||||
// persisted queued work is still being driven forward.
|
||||
runPeriodicRecovery(() => heartbeat
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(() => heartbeat.promoteDueScheduledRetries())
|
||||
.then(async (promotion) => {
|
||||
await heartbeat.resumeQueuedRuns();
|
||||
const reconciled = await heartbeat.reconcileStrandedAssignedIssues({ skipExternalConversationSettlement: true });
|
||||
if (
|
||||
promotion.promoted > 0 ||
|
||||
reconciled.assignmentDispatched > 0 ||
|
||||
reconciled.dispatchRequeued > 0 ||
|
||||
reconciled.continuationRequeued > 0 ||
|
||||
reconciled.successfulRunHandoffEscalated > 0 ||
|
||||
reconciled.successfulRunHandoffRetried > 0 ||
|
||||
reconciled.escalated > 0
|
||||
) {
|
||||
logger.warn(
|
||||
{ promotedScheduledRetries: promotion.promoted, promotedScheduledRetryRunIds: promotion.runIds, ...reconciled },
|
||||
"periodic heartbeat recovery changed assigned issue state",
|
||||
);
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const reconciled = await heartbeat.reconcileResolvedDependencyWakes();
|
||||
if (reconciled.healed > 0) {
|
||||
logger.warn({ ...reconciled }, "periodic dependency-wake reconciliation restored task execution paths");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const reconciled = await heartbeat.reconcileTaskWatchdogs();
|
||||
reportIncompleteWatchdogScans(reconciled, "periodic");
|
||||
if (reconciled.triggered > 0) {
|
||||
logger.warn({ ...reconciled }, "periodic task-watchdog reconciliation triggered watchdog work");
|
||||
}
|
||||
})
|
||||
.reapOrphanedRuns({ staleThresholdMs: 5 * 60 * 1000, skipNativeStatusWakeIntents: true, skipScheduledRetryCancellation: true, skipCapturedInstructionCommit: true, skipInstructionDirectoryCleanup: true, skipStoppedInstructionCollection: true, skipCancelledWorkspacePreservation: true, skipObsoletePolicyReviews: true, skipAutomaticCompletionReviews: true, skipUnsafeWorkspaceExports: true, skipNativePendingEffects: true, skipNativeResponsePresentation: true, skipNativeReviewResponse: true, skipNativeStatusReassessment: true, skipRetainedNativeSessionCleanup: true, skipNativeWorkspaceFinalization: true, skipNativeResultCommit: true, skipNativeRunProjection: true, skipNativeRecoveryNotices: true, skipExternalConversationSettlement: true, skipExternalAgentSettlement: true, skipNativeEnvironmentSettlement: true })
|
||||
.then(async () => {
|
||||
const scanned = await heartbeat.scanSilentActiveRuns();
|
||||
if (scanned.created > 0 || scanned.escalated > 0) {
|
||||
logger.warn({ ...scanned }, "periodic active-run output watchdog created review work");
|
||||
}
|
||||
})
|
||||
.then(async () => {
|
||||
const swept = await heartbeat.sweepStaleIssueLocks();
|
||||
if (swept.cleared > 0) {
|
||||
logger.warn({ ...swept }, "periodic stale-lock sweeper cleared issue locks");
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
logger.error({ err }, "periodic heartbeat recovery failed");
|
||||
}));
|
||||
@@ -2078,6 +1908,11 @@ async function startServerWithDatabaseTeardown(
|
||||
}));
|
||||
};
|
||||
const startHeartbeatScheduler = () => {
|
||||
if (isCloudManagedInstance()) {
|
||||
const status = registerRoutineExecutionStatusWork(deliveryWork, routineExecutionStatusService(db as any),
|
||||
() => !isIdleTaskDrainActive() && !isWarmStandby() && !heartbeatSchedulerStopped);
|
||||
trackHeartbeatSchedulerWork(status.ready);
|
||||
}
|
||||
const scheduled = registerScheduledWork(deliveryWork, {
|
||||
retryMs: config.heartbeatSchedulerIntervalMs,
|
||||
tickAgentTimers: async signal => {
|
||||
@@ -2109,7 +1944,6 @@ async function startServerWithDatabaseTeardown(
|
||||
// as the enabled path.
|
||||
await startEnvironmentCleanup();
|
||||
startHeartbeatSchedulerInterval(() => {
|
||||
scheduleExternalObjectRefreshSweep(new Date());
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2214,9 +2048,6 @@ async function startServerWithDatabaseTeardown(
|
||||
heartbeatSchedulerStopped = true;
|
||||
skillVersionRetentionWorker?.unsubscribe();
|
||||
await taskDrainExpiryWork.stop();
|
||||
clearInterval(nativeHarnessBackupExpirationInterval);
|
||||
clearTimeout(nativeHarnessBackupExpirationStartup);
|
||||
clearTimeout(nativeHarnessBackupExpirationContinuation);
|
||||
nativeRunnerEvidenceRetentionWorker?.unsubscribe();
|
||||
await skillSourceSync.stop();
|
||||
unsubscribeChatCompletions();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../../../services/delivery-work-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
@@ -385,6 +386,7 @@ export function createPostgresWatchdogAdapter(db: Db): WatchdogRunReader & Watch
|
||||
));
|
||||
}
|
||||
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -40,7 +40,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
|
||||
let companyId: string;
|
||||
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
|
||||
beforeAll(async () => {
|
||||
if (external) { await applyPendingMigrations(external); db = createDb(external); }
|
||||
if (external) { await applyPendingMigrations(external); db = createDb(external, { maxConnections: 6 }); }
|
||||
else { database = await startEmbeddedPostgresTestDatabase("paperclip-agent-lifecycle-"); db = createDb(database.connectionString); }
|
||||
}, 90_000);
|
||||
afterAll(async () => {
|
||||
|
||||
@@ -80,22 +80,20 @@ export function agentRecords(db: Db, effects: LifecycleEffects) {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
.update(agents)
|
||||
.set({
|
||||
status: "idle",
|
||||
pauseReason: null,
|
||||
pausedAt: null,
|
||||
errorReason: null,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready")))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
const updated = await effects.transaction(db, existing.companyId, async tx => {
|
||||
const [current] = await tx.select({ status: agents.status, lifecycleState: agents.lifecycleState })
|
||||
.from(agents).where(and(eq(agents.id, id), eq(agents.companyId, existing.companyId))).for("no key update");
|
||||
if (current?.status !== "error" || current.lifecycleState !== "ready") {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
await effects.beforeStatusChange(tx, existing.companyId, id, "error", "idle");
|
||||
const [saved] = await tx.update(agents).set({
|
||||
status: "idle", pauseReason: null, pausedAt: null, errorReason: null, updatedAt: new Date(),
|
||||
}).where(and(eq(agents.id, id), eq(agents.status, "error"), eq(agents.lifecycleState, "ready"))).returning();
|
||||
if (!saved) throw conflict("Only agents in error status can have their error cleared");
|
||||
return saved;
|
||||
});
|
||||
|
||||
if (!updated) {
|
||||
throw conflict("Only agents in error status can have their error cleared");
|
||||
}
|
||||
return getById(updated.id);
|
||||
},
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { hasRequiredWorkspaceRecovery } from "../../../services/workspace-restore-recovery-state.js";
|
||||
import { readNativeWorkspaceSyncReference } from "../../../services/native-runtime/native-workspace-sync.js";
|
||||
import { isPreDispatchExecutionWaitVerified } from "../../../services/pre-dispatch-execution-wait.js";
|
||||
@@ -511,6 +512,15 @@ export function createPostgresRunDispatchAdapter(
|
||||
return decideScheduledRetryGate(factsResult.facts, input.now);
|
||||
}
|
||||
|
||||
async function nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null> {
|
||||
const [row] = await db.select({ at: heartbeatRuns.scheduledRetryAt }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
|
||||
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined))
|
||||
.orderBy(asc(heartbeatRuns.scheduledRetryAt)).limit(1);
|
||||
return row?.at?.getTime() ?? null;
|
||||
}
|
||||
|
||||
async function listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]> {
|
||||
const rows = await db
|
||||
.select()
|
||||
@@ -518,6 +528,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
.where(
|
||||
and(
|
||||
eq(heartbeatRuns.status, "scheduled_retry"),
|
||||
sql`jsonb_typeof(${heartbeatRuns.resultJson}->'startupCancellation') is distinct from 'object'`,
|
||||
lte(heartbeatRuns.scheduledRetryAt, input.now),
|
||||
input.cutoff ? gte(heartbeatRuns.createdAt, input.cutoff) : undefined,
|
||||
),
|
||||
@@ -986,6 +997,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
recordedCommentId && authors.get(recordedCommentId) === actorId ? [recordedCommentId] : [];
|
||||
}));
|
||||
if (commentIds.some(id => !recordedCommentIds.has(id))) return null;
|
||||
await notifyDeferredExecutionWork(tx, { companyId: run.companyId, requestId: wake.id });
|
||||
const [deferred] = await tx.update(agentWakeupRequests).set({
|
||||
status: "deferred_issue_execution", runId: null, claimedAt: null, finishedAt: null, error: null,
|
||||
payload: withQueuedCommentIdsInWakePayload({ ...payload, _paperclipWakeContext: context,
|
||||
@@ -1414,6 +1426,7 @@ export function createPostgresRunDispatchAdapter(
|
||||
|
||||
return {
|
||||
evaluateScheduledRetryGate,
|
||||
nextScheduledRetryAt,
|
||||
listDueRetries,
|
||||
cancelStaleQueuedRun,
|
||||
dispatchResolvedInteractionIfCurrent,
|
||||
|
||||
@@ -24,6 +24,7 @@ export type EvaluateScheduledRetryGateInput = {
|
||||
|
||||
/** Read-only scheduled-retry operations exposed to application use cases. */
|
||||
export interface ScheduledRetryReader {
|
||||
nextScheduledRetryAt(input: { cutoff: Date | null }): Promise<number | null>;
|
||||
evaluateScheduledRetryGate(input: EvaluateScheduledRetryGateInput): Promise<GateDecision>;
|
||||
listDueRetries(input: ListDueRetriesInput): Promise<DueRetryRun[]>;
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ function fakeReader(dueRuns: DueRetryRun[] = []): ScheduledRetryReader & { evalu
|
||||
const evaluateCalls: unknown[] = [];
|
||||
return {
|
||||
evaluateCalls,
|
||||
async nextScheduledRetryAt() { return null; },
|
||||
async evaluateScheduledRetryGate(input) {
|
||||
evaluateCalls.push(input);
|
||||
return { allowed: true };
|
||||
|
||||
@@ -67,6 +67,7 @@ export function createRunDispatch(db: Db, deps: RunDispatchDeps = {}) {
|
||||
});
|
||||
|
||||
return {
|
||||
nextScheduledRetryAt: (input: { cutoff: Date | null }) => adapter.nextScheduledRetryAt(input),
|
||||
evaluateScheduledRetryGate: createEvaluateScheduledRetryGate({ reader: adapter }),
|
||||
promoteScheduledRetry,
|
||||
promoteDueScheduledRetries: createPromoteDueScheduledRetries({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
|
||||
import { DELIVERY_QUEUES } from "../../../services/delivery-work-notifications.js";
|
||||
import { instanceSettingsService } from "../../../services/instance-settings.js";
|
||||
import { randomBytes, randomUUID } from "node:crypto";
|
||||
@@ -37,7 +38,8 @@ import type { TransactionScope } from "../application/ports.js";
|
||||
// a foreign-company row is invisible to a read, and a deferred-status
|
||||
// compare-and-set that affects no row leaves no other trace. The decision
|
||||
// branching itself is proven against plain facts in `domain/policy.test.ts`.
|
||||
const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
|
||||
const external = resolveExternalTestDatabaseUrl("PAPERCLIP_WAKE_QUEUE_TEST_DATABASE_URL");
|
||||
const embeddedPostgresSupport = external ? { supported: true, reason: undefined } : await getEmbeddedPostgresTestSupport();
|
||||
const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
|
||||
|
||||
if (!embeddedPostgresSupport.supported) {
|
||||
@@ -57,8 +59,8 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(tempDb.connectionString);
|
||||
if (!external) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-wake-queue-postgres-adapter-");
|
||||
db = createDb(external ?? tempDb!.connectionString, { maxConnections: 6 });
|
||||
}, 20_000);
|
||||
|
||||
afterEach(async () => {
|
||||
@@ -78,6 +80,7 @@ describeEmbeddedPostgres("wake-queue postgres adapter", () => {
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end({ timeout: 1 });
|
||||
await tempDb?.cleanup();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { notifyDeferredExecutionWork } from "../../../services/deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "../../../services/run-dispatch-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "../../../services/external-conversation-settlement-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -610,6 +613,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
},
|
||||
|
||||
async queueReviewParticipantRecoveryRun({ companyId, issue, finishingRun, recoveryAgent, sessionBefore, now }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const executionState = parseIssueExecutionState(issue.executionState);
|
||||
const wakeupRequest = await tx
|
||||
.insert(agentWakeupRequests)
|
||||
@@ -637,6 +641,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
@@ -715,6 +720,7 @@ function buildTransaction(tx: Db, deps: WakeQueuePostgresAdapterDeps, db: Db, ru
|
||||
.returning()
|
||||
.then((rows) => rows[0]);
|
||||
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
const queuedRun = await tx
|
||||
.insert(heartbeatRuns)
|
||||
.values({
|
||||
@@ -1007,6 +1013,11 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
)).for("update");
|
||||
if (!prior) throw new Error("wake-queue: the deferred wake to merge into was not found for this company");
|
||||
const priorPayload = parseObject(prior.payload), issueId = readNonEmptyString(priorPayload.issueId);
|
||||
const previousComments = new Set(queuedCommentIdsFromWakePayload(priorPayload));
|
||||
const newComments = queuedCommentIdsFromWakePayload(input.mergedPayload).some(id => !previousComments.has(id));
|
||||
const newInteraction = readNonEmptyString(input.mergedPayload.interactionId);
|
||||
if (newComments || newInteraction && newInteraction !== readNonEmptyString(priorPayload.interactionId))
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId, requestId: prior.id });
|
||||
const priorCommentId = readNonEmptyString(priorPayload.commentId), priorActor = readNonEmptyString(prior.requestedByActorId);
|
||||
const [priorComment] = prior.requestedByActorType === "user" && priorActor && issueId && isUuidLike(issueId) &&
|
||||
priorCommentId && isUuidLike(priorCommentId) ? await tx.select({ id: issueComments.id, body: issueComments.body }).from(issueComments).where(and(
|
||||
@@ -1078,6 +1089,7 @@ export function createWakeAdmissionWriter(): WakeAdmissionWriter {
|
||||
|
||||
async insertNewDeferredWake(scope, input) {
|
||||
const tx = requireAdmissionTx(scope, input.companyId);
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.insert(agentWakeupRequests).values({
|
||||
...input.durableReceipt,
|
||||
companyId: input.companyId,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../../../services/status-card-notifications.js";
|
||||
import { notifyExternalAgentSettlement } from "../../external-agents/index.js";
|
||||
import { notifyNativeStatusReassessment } from "../../../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../../../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -55,6 +56,7 @@ export type QueuedCommentQueuePostgresAdapterDeps = {
|
||||
function buildTransaction(tx: Db, companyId: string, issueId: string, deps: QueuedCommentQueuePostgresAdapterDeps): QueuedCommentQueueTransaction {
|
||||
return {
|
||||
async updateCommentBody({ issueId, commentId, body, updatedAt }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const updated = await tx
|
||||
.update(issueComments)
|
||||
.set({ body, updatedAt })
|
||||
@@ -65,6 +67,7 @@ function buildTransaction(tx: Db, companyId: string, issueId: string, deps: Queu
|
||||
},
|
||||
|
||||
async touchIssueUpdatedAt({ issueId, updatedAt }) {
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(issues).set({ updatedAt }).where(and(eq(issues.id, issueId), eq(issues.companyId, companyId)));
|
||||
},
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyStatusCardIssueChange } from "../services/status-card-notifications.js";
|
||||
import { removePluginWorkspace } from "../services/plugin-workspaces.js";
|
||||
import { canActorReadWorkspaceOperation } from "../services/heartbeat-run-privacy.js";
|
||||
import { canActorReadExecutionWorkspace, executionWorkspaceReadSqlCondition } from "../services/authorization.js";
|
||||
@@ -1245,7 +1246,9 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
await runtimeLeases.release({ executionWorkspaceId: existing.id, force: true });
|
||||
|
||||
if (existing.mode === "shared_workspace") {
|
||||
await db
|
||||
await db.transaction(async tx => {
|
||||
await notifyStatusCardIssueChange(tx, existing.companyId);
|
||||
await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
executionWorkspaceId: null,
|
||||
@@ -1257,6 +1260,7 @@ export function executionWorkspaceRoutes(db: Db, opts: { pluginWorkerManager?: P
|
||||
eq(issues.executionWorkspaceId, existing.id),
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../services/deferred-execution-notifications.js";
|
||||
import { notifyNativeRecoveryNotices } from "../services/native-recovery-notice-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "../services/native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "../services/unsafe-workspace-export-notifications.js";
|
||||
@@ -16674,6 +16675,7 @@ export function issueRoutes(
|
||||
}
|
||||
// The click is durable fresh user intent, including when the message
|
||||
// predates a failed run's stop. Keep its content and original attribution.
|
||||
await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: locked.wake.id });
|
||||
await tx.update(agentWakeupRequests).set({
|
||||
payload: { ...readObject(locked.wake.payload), queuedCommentInterrupt: {
|
||||
actorId: actor.actorId, requestedAt: new Date().toISOString(),
|
||||
|
||||
@@ -16,6 +16,12 @@ import { accessService, heartbeatService, instanceSettingsService, issueService,
|
||||
import { queueIssueAssignmentWakeup, type IssueAssignmentWakeupDeps } from "../services/issue-assignment-wakeup.js";
|
||||
import { assertCompanyAccess, getAccessibleResource, getActorInfo, hasCompanyAccess } from "./authz.js";
|
||||
|
||||
// Deadline provenance belongs to scheduling, not the public card contract.
|
||||
function publicCard<T extends { nextEvalReason?: unknown }>(card: T) {
|
||||
const { nextEvalReason: _reason, ...result } = card;
|
||||
return result;
|
||||
}
|
||||
|
||||
export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWakeupDeps } = {}) {
|
||||
const router = Router();
|
||||
const access = accessService(db);
|
||||
@@ -144,9 +150,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
assertCompanyAccess(req, companyId);
|
||||
await assertStatusCardsEnabled();
|
||||
const query = listStatusCardsQuerySchema.parse(req.query);
|
||||
res.json(await service.list(companyId, query.archived, {
|
||||
res.json((await service.list(companyId, query.archived, {
|
||||
issueReadCondition: await issueReadSqlCondition(db, req.actor),
|
||||
}));
|
||||
})).map(publicCard));
|
||||
});
|
||||
|
||||
router.post("/companies/:companyId/status-cards", validate(createStatusCardSchema), async (req, res) => {
|
||||
@@ -162,7 +168,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
try {
|
||||
const compile = await enqueueCompile(req, card.id);
|
||||
await logMutation(req, companyId, "status_card.created", card.id, { state: card.state });
|
||||
res.status(201).json(compile.card);
|
||||
res.status(201).json(publicCard(compile.card));
|
||||
} catch (error) {
|
||||
await service.remove(card.id);
|
||||
throw error;
|
||||
@@ -173,9 +179,9 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
await assertStatusCardsEnabled();
|
||||
const card = await getAccessibleResource(req, res, service.getById(req.params.id as string), "Status card not found");
|
||||
if (!card) return;
|
||||
res.json(await service.hydrate(card, {
|
||||
res.json(publicCard(await service.hydrate(card, {
|
||||
issueReadCondition: await issueReadSqlCondition(db, req.actor),
|
||||
}));
|
||||
})));
|
||||
});
|
||||
|
||||
router.patch("/status-cards/:id", validate(patchStatusCardSchema), async (req, res) => {
|
||||
@@ -197,7 +203,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
fields: Object.keys(req.body),
|
||||
archived: Boolean(updated.archivedAt),
|
||||
});
|
||||
res.json(compile?.card ?? restore?.card ?? updated);
|
||||
res.json(publicCard(compile?.card ?? restore?.card ?? updated));
|
||||
});
|
||||
|
||||
router.delete("/status-cards/:id", async (req, res) => {
|
||||
@@ -234,7 +240,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
generatingIssueId: result.generatingIssue.id,
|
||||
alreadyGenerating: result.alreadyGenerating,
|
||||
});
|
||||
res.status(result.alreadyGenerating ? 200 : 202).json(result);
|
||||
res.status(result.alreadyGenerating ? 200 : 202).json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
router.post("/status-cards/:id/refresh", validate(refreshStatusCardSchema), async (req, res) => {
|
||||
@@ -249,7 +255,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
alreadyGenerating: result.alreadyGenerating,
|
||||
enqueued: result.enqueued,
|
||||
});
|
||||
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json(result);
|
||||
res.status(result.enqueued && !result.alreadyGenerating ? 202 : 200).json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
router.get("/status-cards/:id/dry-run", async (req, res) => {
|
||||
@@ -289,7 +295,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
generationIssueId: req.body.generationIssueId,
|
||||
changeSummary: req.body.changeSummary,
|
||||
});
|
||||
res.json(updated);
|
||||
res.json(publicCard(updated));
|
||||
});
|
||||
|
||||
router.put("/status-cards/:id/summary", validate(writeStatusCardSummarySchema), async (req, res) => {
|
||||
@@ -309,7 +315,7 @@ export function statusCardRoutes(db: Db, opts: { heartbeat?: IssueAssignmentWake
|
||||
documentId: result.document.id,
|
||||
changeSummary: req.body.changeSummary,
|
||||
});
|
||||
res.json(result);
|
||||
res.json({ ...result, card: publicCard(result.card) });
|
||||
});
|
||||
|
||||
return router;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyNativeStatusReassessment } from "./native-status-reassessment-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -743,8 +745,10 @@ export function accessService(db: Db) {
|
||||
sql`${issues.status} not in ('done', 'cancelled')`,
|
||||
);
|
||||
await notifyNativeStatusReassessment(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
const resetInProgress = await tx
|
||||
.update(issues)
|
||||
.set({
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
|
||||
@@ -75,6 +76,8 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
|
||||
if (!reason.trim()) throw conflict("A repair reason is required");
|
||||
const before = await inspect(tx, companyId);
|
||||
if (before.fingerprint !== fingerprint) throw conflict("Accounting changed since inspection; inspect again before repairing");
|
||||
if (before.findings.some(f => f.repairable && f.kind === "agent_projection"))
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
for (const finding of before.findings.filter(f => f.repairable)) {
|
||||
if (finding.kind === "company_projection") await tx.update(companies).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(eq(companies.id, companyId));
|
||||
if (finding.kind === "agent_projection") await tx.update(agents).set({ spentMonthlyCents: sql`${finding.expected.cents}::numeric`, updatedAt: new Date() }).where(and(eq(agents.id, finding.entityId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
@@ -16,6 +19,7 @@ import { clearPrimaryAgent } from "./primary-agent.js";
|
||||
import { budgetServiceInTransaction } from "./budgets.js";
|
||||
|
||||
export async function prepareAgentConfiguration(db: Db, existing: AgentConfigurationRecord, data: AgentConfigurationPatch, options?: UpdateAgentOptions): Promise<AgentConfigurationPatch> {
|
||||
await notifyAttentionRetentionWork(db, { companyId: existing.companyId });
|
||||
const id = existing.id;
|
||||
const { ensureManager, assertNoCycle, assertCompanyShortnameAvailable, assertBuiltInAgentMetadataMutationAllowed } = agentRecordQueries(db);
|
||||
if (existing.status === "pending_approval" && !options?.allowPendingApprovalConfigUpdate) {
|
||||
@@ -83,6 +87,8 @@ export async function prepareAgentConfiguration(db: Db, existing: AgentConfigura
|
||||
existing.adapterConfig,
|
||||
);
|
||||
}
|
||||
await notifyRunDispatch(db, { companyId: existing.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: existing.companyId });
|
||||
if (normalizedPatch.runtimeConfig !== undefined || normalizedPatch.reportsTo !== undefined) {
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyIssueRecovery } from "./issue-recovery-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
import { FAST_RESPONSE_AGENT_GUIDANCE, fastResponseHistoryBody } from "@paperclipai/shared";
|
||||
@@ -269,6 +271,7 @@ export async function prepareConversationTurn(
|
||||
context.conversationReplayThroughCommentId = replayThrough?.id ?? commentId;
|
||||
}
|
||||
}
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await tx
|
||||
@@ -389,6 +392,7 @@ export async function settleConversationTurn(
|
||||
issue.conversationState === conversationState
|
||||
)
|
||||
return true;
|
||||
await notifyStatusCardIssueChange(tx, issue.companyId);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
if (status === "in_review") await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: issue.companyId, issueId: issue.id });
|
||||
@@ -544,6 +548,7 @@ export async function resumeConversationForReset(db: Db, comment: typeof issueCo
|
||||
eq(issues.id, comment.issueId), eq(issues.companyId, comment.companyId),
|
||||
)).for("update");
|
||||
if (!isConversation(issue) || comment.conversationSessionGeneration != null) return;
|
||||
await notifyIssueRecovery(tx, { companyId: issue.companyId });
|
||||
const released = await tx.update(issueTreeHolds).set({
|
||||
status: "released", releasedAt: new Date(), updatedAt: new Date(),
|
||||
releasedByActorType: "user", releasedByUserId: comment.authorUserId,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js";
|
||||
import { notifyExternalConversationSettlement } from "./external-conversation-settlement-notifications.js";
|
||||
import { notifyUnsafeWorkspaceExportRecovery } from "./unsafe-workspace-export-notifications.js";
|
||||
@@ -34,10 +36,12 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
|
||||
if (classifierHold) throw conflict("Wait for active prompt injection checks or resolve their unknown charges in Costs before deleting this agent", {
|
||||
code: "agent_prompt_injection_accounting_pending",
|
||||
});
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
await notifyExternalConversationSettlement(tx, { companyId });
|
||||
await issueThreadInteractionService(tx).cancelPendingForDeletedAddressee(companyId, id);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.issueMonitors);
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyStatusCardIssueChange(tx, companyId);
|
||||
await tx.update(agents).set({ reportsTo: null }).where(eq(agents.reportsTo, id));
|
||||
await tx
|
||||
.update(issues)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifySkillVersionRetention, notifySkillVersionPinChange } from "./skill-version-retention-notifications.js";
|
||||
import { agentAppearanceSchema, randomAgentAppearance } from "@paperclipai/shared";
|
||||
import { normalizePaperclipRunnerAdapterConfig } from "@paperclipai/adapter-utils/server-utils";
|
||||
@@ -16,6 +17,7 @@ import { initializePrimaryAgent } from "./primary-agent.js";
|
||||
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
|
||||
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
|
||||
await notifyAttentionRetentionWork(db, { companyId });
|
||||
const { runner, ...data } = input;
|
||||
const selection = options?.runnerResolved
|
||||
? { adapterType: data.adapterType ?? "process", adapterConfig: isPlainRecord(data.adapterConfig) ? data.adapterConfig : {} }
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyNativeReviewResponse } from "./native-review-response-notifications.js";
|
||||
import { isAgentStatusInvokable } from "@paperclipai/shared";
|
||||
import { notifyNativeResponsePresentation } from "./native-response-presentation-notifications.js";
|
||||
@@ -38,7 +41,12 @@ export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): li
|
||||
},
|
||||
notifyWork: tx => notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle),
|
||||
async beforeStatusChange(tx, companyId, agentId, before, after) {
|
||||
if (before !== after) await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyAttentionRetentionWork(tx, { companyId });
|
||||
if (before !== after) {
|
||||
await notifyRunDispatch(tx, { companyId });
|
||||
await notifyDeferredExecutionEligibility(tx, { companyId });
|
||||
await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentTimers);
|
||||
}
|
||||
if (!isAgentStatusInvokable(before) && isAgentStatusInvokable(after)) {
|
||||
await notifyNativeResponsePresentation(tx, { companyId, agentId });
|
||||
await notifyNativeReviewResponse(tx, { companyId, agentId });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { and, eq, inArray } from "drizzle-orm";
|
||||
import { agents, projects, type Db } from "@paperclipai/db";
|
||||
import { trustAuthorizationPolicySchema, type AgentProjectAccess } from "@paperclipai/shared";
|
||||
@@ -55,6 +56,7 @@ export async function addAgentProjectAccess(db: Db, actor: AuthorizationActor, c
|
||||
const added = requested.filter(id => !previous.includes(id));
|
||||
const next = [...previous, ...added];
|
||||
if (added.length) {
|
||||
await notifyAttentionRetentionWork(tx, { companyId, sourceKind: "agent_error_alert" });
|
||||
await tx.update(agents).set({ permissions: { ...agent.permissions, authorizationPolicy: {
|
||||
...policy, trustBoundary: { ...boundary, mode: "low_trust_review", companyId, projectIds: next },
|
||||
} }, updatedAt: new Date() }).where(and(eq(agents.id, agentId), eq(agents.companyId, companyId)));
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyAttentionRetentionWork } from "./retention-work-notifications.js";
|
||||
import { type BudgetServiceHooks } from "./budgets.js";
|
||||
|
||||
import { and, desc, eq, inArray } from "drizzle-orm";
|
||||
@@ -52,7 +53,9 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
const updated = await db
|
||||
const updated = await db.transaction(async tx => {
|
||||
await notifyAttentionRetentionWork(tx, { companyId: existing.companyId });
|
||||
return tx
|
||||
.update(agents)
|
||||
.set({
|
||||
permissions: normalizeAgentPermissions({ ...existing.permissions, ...permissions }),
|
||||
@@ -61,6 +64,7 @@ export function agentService(db: Db, budgetHooks: BudgetServiceHooks = {}) {
|
||||
.where(eq(agents.id, id))
|
||||
.returning()
|
||||
.then((rows) => rows[0] ?? null);
|
||||
});
|
||||
|
||||
return updated ? getById(updated.id) : null;
|
||||
},
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
import { and, eq, getTableColumns, inArray, isNull, sql, type SQL } from "drizzle-orm";
|
||||
import { agents, decisionRetention, decisions, heartbeatRuns,
|
||||
issueApprovals, issueRecoveryActions, issueThreadInteractions, issues, approvals, type Db } from "@paperclipai/db";
|
||||
import { evaluateAgentInvokability } from "./agent-invokability.js";
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, listFailedRunsWithNewerRun, readRunIssueId, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { decisionRetentionService, DEFAULT_DECISION_ARCHIVE_DAYS } from "./decision-retention.js";
|
||||
import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js";
|
||||
import { DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
import { attentionIssueCondition } from "./issue-visibility.js";
|
||||
import { RETRY_EXHAUSTION_MESSAGE_PREFIX } from "./heartbeat-run-events.js";
|
||||
import { issueService } from "./issues.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { registerScopedDeadlineWork } from "./scoped-deadline-work.js";
|
||||
import type { AttentionRetentionScope } from "./retention-work-notifications.js";
|
||||
|
||||
const archiveDelay = DEFAULT_DECISION_ARCHIVE_DAYS * 86_400_000;
|
||||
const kinds = ["decision", "issue_thread_interaction", "recovery_action", "blocker_attention", "review",
|
||||
"failed_run", "budget_alert", "agent_error_alert"] as const;
|
||||
type Kind = typeof kinds[number];
|
||||
type Scope = AttentionRetentionScope;
|
||||
type Candidate = { sourceKind: Kind; sourceId: string; companyId: string; activityAt: string; deadlineAt: number; issueId: string | null };
|
||||
type Entry = Candidate & { id: string; dueAt: number; rootId: string; variant?: "owner" | "terminal" };
|
||||
type Cursor = { sourceKind: string; sourceId: string };
|
||||
|
||||
type PageIssue = typeof issues.$inferSelect & { activityAt: string; deadlineAt: number;
|
||||
blockedActivityAt: string | null; blockedDeadlineAt: number | null };
|
||||
type PageEvidence = {
|
||||
decisions: Map<string, { originIssueId: string }>;
|
||||
recoveries: Map<string, typeof issueRecoveryActions.$inferSelect>;
|
||||
runs: Map<string, typeof heartbeatRuns.$inferSelect>;
|
||||
agents: Set<string>;
|
||||
visibleIssues: Map<string, PageIssue>;
|
||||
newerRuns: Set<string>;
|
||||
deferredRoots: Set<string>;
|
||||
blockers: Awaited<ReturnType<ReturnType<typeof issueService>["listBlockerAttention"]>>;
|
||||
};
|
||||
|
||||
const sourceTables: Record<Kind, SQL> = {
|
||||
decision: sql`decisions`, issue_thread_interaction: sql`issue_thread_interactions`,
|
||||
recovery_action: sql`issue_recovery_actions`, blocker_attention: sql`issues`, review: sql`issues`,
|
||||
failed_run: sql`heartbeat_runs`, budget_alert: sql`budget_incidents`, agent_error_alert: sql`agents`,
|
||||
};
|
||||
|
||||
const sources: Record<Kind, SQL> = {
|
||||
decision: sql`select 'decision' as source_kind, d.id::text as source_id, d.company_id, d.updated_at as activity_at, d.origin_issue_id::text as issue_id
|
||||
from decisions d where d.status = 'open'`,
|
||||
issue_thread_interaction: sql`select 'issue_thread_interaction', i.id::text, i.company_id, i.updated_at, i.issue_id::text
|
||||
from issue_thread_interactions i where i.status = 'pending' and i.addressee_user_id is null`,
|
||||
recovery_action: sql`select 'recovery_action', r.id::text, r.company_id, r.updated_at, r.source_issue_id::text
|
||||
from issue_recovery_actions r where r.status in ('active', 'escalated') and r.owner_type in (${sql.join(HUMAN_RECOVERY_OWNER_TYPES.map(owner => sql`${owner}`), sql`, `)})`,
|
||||
blocker_attention: sql`select 'blocker_attention', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'blocked' and i.hidden_at is null and i.harness_kind is null`,
|
||||
review: sql`select 'review', i.id::text, i.company_id, i.updated_at, i.id::text
|
||||
from issues i where i.status = 'in_review' and i.hidden_at is null and i.harness_kind is null`,
|
||||
failed_run: sql`select 'failed_run', r.id::text, r.company_id, coalesce(r.finished_at, r.updated_at, r.created_at), coalesce(r.context_snapshot->>'issueId', r.context_snapshot->>'taskId')
|
||||
from heartbeat_runs r where r.status in ('failed', 'timed_out') and exists (
|
||||
select 1 from heartbeat_run_events e where e.run_id = r.id and e.company_id = r.company_id
|
||||
and e.event_type = 'lifecycle' and e.message like ${`${RETRY_EXHAUSTION_MESSAGE_PREFIX}%`})`,
|
||||
budget_alert: sql`select 'budget_alert', b.id::text, b.company_id, b.updated_at, null::text from budget_incidents b
|
||||
where b.status = 'open' and (b.threshold_type = 'hard' or (b.amount_limit > 0
|
||||
and round(b.amount_observed::numeric / b.amount_limit * 10000) / 100 >= ${ATTENTION_BUDGET_WARNING_PERCENT}))`,
|
||||
agent_error_alert: sql`select 'agent_error_alert', a.id::text, a.company_id, a.updated_at, null::text from agents a where a.status = ${ATTENTION_AGENT_ERROR_STATUS}`,
|
||||
};
|
||||
|
||||
export function attentionRetentionService(db: Db) {
|
||||
const cursors = new Map<string, Cursor>();
|
||||
async function candidates(database: Db, scope: Scope = {}, cursor?: Cursor, sourceId?: string): Promise<Candidate[]> {
|
||||
const issueIds = scope.issueIds ?? (scope.issueId ? [scope.issueId] : []);
|
||||
const selected = kinds.filter(kind => (!scope.sourceKind || scope.sourceKind === kind)
|
||||
&& (!issueIds.length || !["budget_alert", "agent_error_alert"].includes(kind)));
|
||||
if (!selected.length) return [];
|
||||
const rows = await database.execute(sql`with source_candidates(source_kind, source_id, company_id, activity_at, issue_id) as (${sql.join(selected.map(kind => sources[kind]), sql` union all `)})
|
||||
select s.source_kind as "sourceKind", s.source_id as "sourceId", s.company_id as "companyId",
|
||||
s.issue_id as "issueId", s.activity_at::text as "activityAt", ceil(extract(epoch from s.activity_at) * 1000 + ${archiveDelay}) as "deadlineAt"
|
||||
from source_candidates s inner join companies c on c.id = s.company_id
|
||||
left join decision_retention r on r.company_id = s.company_id and r.source_kind = s.source_kind and r.source_id = s.source_id
|
||||
where c.status = 'active'
|
||||
${issueIds.length ? sql`and s.issue_id in (${sql.join(issueIds.map(id => sql`${id}`), sql`, `)})` : sql``}
|
||||
and (${scope.companyId ?? null}::uuid is null or s.company_id = ${scope.companyId ?? null}::uuid)
|
||||
and (${sourceId ?? scope.sourceId ?? null}::text is null or s.source_id = ${sourceId ?? scope.sourceId ?? null})
|
||||
and (s.source_kind = 'blocker_attention' or ((r.id is null or r.keep = false) and r.archived_at is null))
|
||||
${cursor ? sql`and (s.source_kind, s.source_id) > (${cursor.sourceKind}, ${cursor.sourceId})` : sql``}
|
||||
order by s.source_kind, s.source_id limit 100`) as unknown as Candidate[];
|
||||
return rows.map(row => ({ ...row, deadlineAt: Number(row.deadlineAt) }));
|
||||
}
|
||||
|
||||
async function visibleIssue(database: Db, companyId: string, issueId: string, conversations = false) {
|
||||
const [row] = await database.select().from(issues).where(and(eq(issues.companyId, companyId), eq(issues.id, issueId),
|
||||
attentionIssueCondition(conversations)));
|
||||
return row;
|
||||
}
|
||||
|
||||
async function interactionEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "issue_thread_interaction") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.id, ids),
|
||||
eq(issueThreadInteractions.status, "pending"), isNull(issueThreadInteractions.addresseeUserId)));
|
||||
if (!rows.length) continue;
|
||||
const visible = await database.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, rows.map(row => row.issueId)), attentionIssueCondition(true)));
|
||||
const visibleIds = new Set(visible.map(row => row.id));
|
||||
const roster = await database.select().from(agents).where(eq(agents.companyId, companyId));
|
||||
const boardAddressed = (row: typeof issueThreadInteractions.$inferSelect) => !row.addresseeAgentId
|
||||
|| !evaluateAgentInvokability(roster.find(agent => agent.id === row.addresseeAgentId), roster).invokable;
|
||||
const confirmationIssueIds = rows.filter(row => row.kind === "request_confirmation").map(row => row.issueId);
|
||||
const siblings = confirmationIssueIds.length ? await database.select().from(issueThreadInteractions).where(and(
|
||||
eq(issueThreadInteractions.companyId, companyId), inArray(issueThreadInteractions.issueId, confirmationIssueIds),
|
||||
eq(issueThreadInteractions.status, "pending"), eq(issueThreadInteractions.kind, "request_confirmation"),
|
||||
isNull(issueThreadInteractions.addresseeUserId))) : [];
|
||||
const newest = new Set(collapsePendingConfirmationsToNewest(siblings.filter(boardAddressed)).map(row => row.id));
|
||||
for (const row of rows) result.set(row.id, visibleIds.has(row.issueId) && boardAddressed(row)
|
||||
&& (row.kind !== "request_confirmation" || newest.has(row.id)));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function reviewEligibility(database: Db, candidates: Candidate[]) {
|
||||
const result = new Map<string, boolean>();
|
||||
const byCompany = new Map<string, string[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.sourceKind !== "review") continue;
|
||||
const ids = byCompany.get(candidate.companyId) ?? [];
|
||||
ids.push(candidate.sourceId);
|
||||
byCompany.set(candidate.companyId, ids);
|
||||
}
|
||||
for (const [companyId, ids] of byCompany) {
|
||||
const rows = await database.select().from(issues).where(and(eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids), attentionIssueCondition(false)));
|
||||
const pendingApprovals = await database.select({ issueId: issueApprovals.issueId }).from(issueApprovals)
|
||||
.innerJoin(approvals, eq(approvals.id, issueApprovals.approvalId)).where(and(
|
||||
eq(issueApprovals.companyId, companyId), eq(approvals.companyId, companyId),
|
||||
inArray(issueApprovals.issueId, ids), eq(approvals.status, "pending")));
|
||||
const approved = new Set(pendingApprovals.map(row => row.issueId));
|
||||
const stalled = await issueService(database).listReviewAttention(companyId, rows);
|
||||
for (const row of rows) {
|
||||
const state = parseIssueExecutionState(row.executionState);
|
||||
result.set(row.id, requiresReviewAttention({
|
||||
hasHumanParticipant: state?.status === "pending" && state.currentParticipant?.type === "user",
|
||||
hasUserAssignee: Boolean(row.assigneeUserId), hasPendingApproval: approved.has(row.id),
|
||||
stalled: stalled.get(row.id)?.state === "stalled",
|
||||
}));
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function pageEvidence(database: Db, candidates: Candidate[]) {
|
||||
const byCompany = new Map<string, Candidate[]>();
|
||||
for (const candidate of candidates) {
|
||||
if (!["decision", "recovery_action", "failed_run", "blocker_attention"].includes(candidate.sourceKind)) continue;
|
||||
const rows = byCompany.get(candidate.companyId) ?? [];
|
||||
rows.push(candidate);
|
||||
byCompany.set(candidate.companyId, rows);
|
||||
}
|
||||
const result = new Map<string, PageEvidence>();
|
||||
for (const [companyId, rows] of byCompany) {
|
||||
const ids = (kind: Kind) => rows.filter(row => row.sourceKind === kind).map(row => row.sourceId);
|
||||
const decisionIds = ids("decision"), recoveryIds = ids("recovery_action"), runIds = ids("failed_run"), blockerIds = ids("blocker_attention");
|
||||
const decisionRows = decisionIds.length ? await database.select({ id: decisions.id, originIssueId: decisions.originIssueId })
|
||||
.from(decisions).where(and(eq(decisions.companyId, companyId), inArray(decisions.id, decisionIds), eq(decisions.status, "open"))) : [];
|
||||
const recoveryRows = recoveryIds.length ? await database.select().from(issueRecoveryActions).where(and(
|
||||
eq(issueRecoveryActions.companyId, companyId), inArray(issueRecoveryActions.id, recoveryIds))) : [];
|
||||
const runRows = runIds.length ? await database.select().from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId), inArray(heartbeatRuns.id, runIds))) : [];
|
||||
const runAgents = runRows.length ? await database.select({ id: agents.id }).from(agents).where(and(
|
||||
eq(agents.companyId, companyId), inArray(agents.id, [...new Set(runRows.map(row => row.agentId))]),
|
||||
sql`${agents.status} <> 'terminated'`)) : [];
|
||||
const readIssues = async (issueIds: string[]) => issueIds.length ? database.select({ ...getTableColumns(issues),
|
||||
activityAt: sql<string>`${issues.updatedAt}::text`,
|
||||
deadlineAt: sql<number>`ceil(extract(epoch from ${issues.updatedAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
blockedActivityAt: sql<string | null>`${issues.blockedTransitionAt}::text`,
|
||||
blockedDeadlineAt: sql<number | null>`ceil(extract(epoch from ${issues.blockedTransitionAt}) * 1000 + ${archiveDelay})`.mapWith(Number),
|
||||
}).from(issues).where(and(eq(issues.companyId, companyId), inArray(issues.id, [...new Set(issueIds)]),
|
||||
attentionIssueCondition(false))) : [];
|
||||
const issueIds = [...blockerIds, ...decisionRows.map(row => row.originIssueId), ...recoveryRows.map(row => row.sourceIssueId),
|
||||
...runRows.map(row => readRunIssueId(row.contextSnapshot))].filter((id): id is string => Boolean(id));
|
||||
const visibleRows = await readIssues(issueIds);
|
||||
const visible = new Map(visibleRows.map(row => [row.id, row]));
|
||||
const blockerRoots = visibleRows.filter(row => blockerIds.includes(row.id));
|
||||
const blockerBatch = blockerRoots.length ? await issueService(database).listBlockerAttentionBatch(companyId, blockerRoots)
|
||||
: { attention: new Map(), deferredRootIds: new Set<string>() };
|
||||
const blockers: PageEvidence["blockers"] = blockerBatch.attention;
|
||||
const terminalIds = [...blockers.values()].filter(terminalBlockerRequiresAttention)
|
||||
.map(attention => attention.terminalBlockerIssueId).filter((id): id is string => id != null && !visible.has(id));
|
||||
for (const row of await readIssues(terminalIds)) visible.set(row.id, row);
|
||||
result.set(companyId, {
|
||||
decisions: new Map(decisionRows.map(row => [row.id, row])), recoveries: new Map(recoveryRows.map(row => [row.id, row])),
|
||||
runs: new Map(runRows.map(row => [row.id, row])), agents: new Set(runAgents.map(row => row.id)), visibleIssues: visible,
|
||||
newerRuns: await listFailedRunsWithNewerRun(database, companyId,
|
||||
runRows.map(run => ({ ...run, issueId: readRunIssueId(run.contextSnapshot) }))), blockers, deferredRoots: blockerBatch.deferredRootIds,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function eligible(database: Db, candidate: Candidate, variant?: "owner" | "terminal", lock = false, reviewAttention?: Map<string, boolean>, interactionAttention?: Map<string, boolean>, evidence?: PageEvidence): Promise<Entry | null> {
|
||||
if (candidate.sourceKind === "blocker_attention" && variant === "terminal" && evidence?.deferredRoots.has(candidate.sourceId)) {
|
||||
return { ...candidate, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:terminal`, dueAt: Date.now() };
|
||||
}
|
||||
let sourceId = candidate.sourceId;
|
||||
let activityAt = candidate.activityAt;
|
||||
let dueAt = candidate.deadlineAt;
|
||||
if (candidate.sourceKind === "decision") {
|
||||
const row = evidence ? evidence.decisions.get(sourceId) : (await database.select({ originIssueId: decisions.originIssueId }).from(decisions)
|
||||
.where(and(eq(decisions.companyId, candidate.companyId), eq(decisions.id, sourceId), eq(decisions.status, "open"))))[0];
|
||||
if (!row || row.originIssueId && !(evidence ? evidence.visibleIssues.has(row.originIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.originIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "issue_thread_interaction") {
|
||||
const attention = interactionAttention ?? await interactionEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "recovery_action") {
|
||||
const row = evidence ? evidence.recoveries.get(sourceId) : (await database.select().from(issueRecoveryActions).where(and(eq(issueRecoveryActions.companyId, candidate.companyId), eq(issueRecoveryActions.id, sourceId))))[0];
|
||||
if (!row || !(evidence ? evidence.visibleIssues.has(row.sourceIssueId)
|
||||
: await visibleIssue(database, candidate.companyId, row.sourceIssueId))) return null;
|
||||
} else if (candidate.sourceKind === "review") {
|
||||
const attention = reviewAttention ?? await reviewEligibility(database, [candidate]);
|
||||
if (!attention.get(sourceId)) return null;
|
||||
} else if (candidate.sourceKind === "blocker_attention") {
|
||||
const row = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!row) return null;
|
||||
if (variant === "owner") {
|
||||
if (row.unblockDescriptor?.owner !== "board" || !isProspectiveBlockedTransition(row)) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.blockedActivityAt!, due: evidence.visibleIssues.get(sourceId)!.blockedDeadlineAt! } : (await database.execute(sql`select blocked_transition_at::text as at,
|
||||
ceil(extract(epoch from blocked_transition_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${row.id}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
} else {
|
||||
const attention = evidence ? evidence.blockers.get(row.id) : (await issueService(database).listBlockerAttention(candidate.companyId, [row])).get(row.id);
|
||||
if (!terminalBlockerRequiresAttention(attention)) return null;
|
||||
sourceId = attention.terminalBlockerIssueId ?? row.id;
|
||||
if (lock && sourceId !== candidate.sourceId) await database.execute(sql`select id from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid for update`);
|
||||
const terminal = evidence ? evidence.visibleIssues.get(sourceId) : await visibleIssue(database, candidate.companyId, sourceId);
|
||||
if (!terminal) return null;
|
||||
const activity = evidence ? { at: evidence.visibleIssues.get(sourceId)!.activityAt, due: evidence.visibleIssues.get(sourceId)!.deadlineAt } : (await database.execute(sql`select updated_at::text as at,
|
||||
ceil(extract(epoch from updated_at) * 1000 + ${archiveDelay}) as due from issues
|
||||
where company_id = ${candidate.companyId}::uuid and id = ${sourceId}::uuid`) as unknown as Array<{ at: string; due: string }>)[0];
|
||||
activityAt = activity.at; dueAt = Number(activity.due);
|
||||
}
|
||||
const [retention] = await database.select().from(decisionRetention).where(and(eq(decisionRetention.companyId, candidate.companyId),
|
||||
eq(decisionRetention.sourceKind, candidate.sourceKind), eq(decisionRetention.sourceId, sourceId)));
|
||||
if (retention?.keep || retention?.archivedAt) return null;
|
||||
} else if (candidate.sourceKind === "failed_run") {
|
||||
const run = evidence ? evidence.runs.get(sourceId) : (await database.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, candidate.companyId), eq(heartbeatRuns.id, sourceId))))[0];
|
||||
if (!run) return null;
|
||||
const agent = evidence ? evidence.agents.has(run.agentId) : (await database.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, candidate.companyId),
|
||||
eq(agents.id, run.agentId), sql`${agents.status} <> 'terminated'`)))[0];
|
||||
const issueId = readRunIssueId(run.contextSnapshot);
|
||||
if (!agent || issueId && !(evidence ? evidence.visibleIssues.has(issueId)
|
||||
: await visibleIssue(database, candidate.companyId, issueId))) return null;
|
||||
if ((evidence ? evidence.newerRuns : await listFailedRunsWithNewerRun(database, candidate.companyId, [{ ...run, issueId }])).has(run.id)) return null;
|
||||
}
|
||||
return { ...candidate, sourceId, activityAt, rootId: candidate.sourceId, variant,
|
||||
id: `${candidate.sourceKind}:${candidate.sourceId}:${variant ?? "source"}`, dueAt };
|
||||
}
|
||||
|
||||
async function load(scope?: Scope, signal?: AbortSignal) {
|
||||
const key = JSON.stringify(scope ?? {});
|
||||
const rows = await candidates(db, scope, cursors.get(key));
|
||||
const entries: Entry[] = [];
|
||||
const reviews = await reviewEligibility(db, rows);
|
||||
const interactions = await interactionEligibility(db, rows);
|
||||
const evidence = await pageEvidence(db, rows);
|
||||
for (const row of rows) {
|
||||
if (signal?.aborted) return { rows: null, complete: true };
|
||||
for (const variant of row.sourceKind === "blocker_attention" ? ["owner", "terminal"] as const : [undefined]) {
|
||||
const entry = await eligible(db, row, variant, false, reviews, interactions, evidence.get(row.companyId));
|
||||
if (entry) entries.push(entry);
|
||||
}
|
||||
}
|
||||
if (rows.length === 100) {
|
||||
const last = rows[rows.length - 1]!;
|
||||
cursors.set(key, { sourceKind: last.sourceKind, sourceId: last.sourceId });
|
||||
} else cursors.delete(key);
|
||||
return { rows: entries, complete: rows.length < 100 };
|
||||
}
|
||||
|
||||
async function check(entry: Entry) {
|
||||
return db.transaction(async transaction => {
|
||||
const database = transaction as unknown as Db;
|
||||
await database.execute(sql`select id from ${sourceTables[entry.sourceKind]}
|
||||
where company_id = ${entry.companyId}::uuid and id = ${entry.rootId}::uuid for update`);
|
||||
const [candidate] = await candidates(database, { companyId: entry.companyId, sourceKind: entry.sourceKind }, undefined, entry.rootId);
|
||||
if (!candidate) return null;
|
||||
const current = await eligible(database, candidate, entry.variant, true);
|
||||
if (!current || current.dueAt > Date.now()) return current;
|
||||
const retention = decisionRetentionService(database);
|
||||
const item = { sourceKind: current.sourceKind, subject: { id: current.sourceId }, activityAt: current.activityAt };
|
||||
const state = (await retention.syncItems(current.companyId, [item])).get(`${current.sourceKind}:${current.sourceId}`)!;
|
||||
await retention.autoArchive({ companyId: current.companyId, items: [{ ...item,
|
||||
keep: state.keep, archivedAt: state.archivedAt?.toISOString() ?? null, retentionVersion: state.version }] });
|
||||
return null;
|
||||
});
|
||||
}
|
||||
return { load, check };
|
||||
}
|
||||
|
||||
export function registerAttentionRetentionWork(coordinator: ReturnType<typeof createDeliveryWorkCoordinator>,
|
||||
service: ReturnType<typeof attentionRetentionService>, canRun: () => boolean) {
|
||||
return registerScopedDeadlineWork(coordinator, {
|
||||
queue: DELIVERY_QUEUES.attentionRetention,
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint) as Scope,
|
||||
matches: (entry, scope) => (!scope?.companyId || scope.companyId === entry.companyId)
|
||||
&& (!scope?.sourceKind || scope.sourceKind === entry.sourceKind)
|
||||
&& (!scope?.sourceId || scope.sourceId === entry.sourceId)
|
||||
&& (!scope?.issueId || scope.issueId === entry.issueId)
|
||||
&& (!scope?.issueIds?.length || entry.issueId !== null && scope.issueIds.includes(entry.issueId)),
|
||||
load: service.load, check: service.check,
|
||||
}, canRun);
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import { and, eq, gt, inArray, sql, type SQL } from "drizzle-orm";
|
||||
import { heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
|
||||
export const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
export const ATTENTION_BUDGET_WARNING_PERCENT = 85;
|
||||
export const ATTENTION_AGENT_ERROR_STATUS = "error";
|
||||
|
||||
export function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
export function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
export async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
|
||||
export function requiresReviewAttention(input: {
|
||||
hasHumanParticipant: boolean; hasUserAssignee: boolean; hasPendingApproval: boolean; stalled: boolean;
|
||||
}) {
|
||||
return input.hasHumanParticipant || input.hasUserAssignee || input.hasPendingApproval || input.stalled;
|
||||
}
|
||||
|
||||
export function terminalBlockerRequiresAttention<T extends { state?: string; blockingTreeLive?: boolean }>(
|
||||
attention: T | null | undefined,
|
||||
): attention is T & { state: "stalled" | "needs_attention" } {
|
||||
return Boolean(attention && !attention.blockingTreeLive && ["stalled", "needs_attention"].includes(attention.state ?? ""));
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { HUMAN_RECOVERY_OWNER_TYPES, ATTENTION_BUDGET_WARNING_PERCENT, ATTENTION_AGENT_ERROR_STATUS, collapsePendingConfirmationsToNewest, readRunIssueId, listFailedRunsWithNewerRun, requiresReviewAttention, terminalBlockerRequiresAttention } from "./attention-source-eligibility.js";
|
||||
import { approvalExecutionIssueCondition, firstApprovalIssueId } from "./approval-visibility.js";
|
||||
import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, inArray, isNull, notInArray, or, sql, type SQL } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import {
|
||||
agents,
|
||||
@@ -58,7 +59,7 @@ import {
|
||||
BLOCKER_ATTENTION_MAX_NODES,
|
||||
issueService,
|
||||
} from "./issues.js";
|
||||
import { executionIssueCondition, visibleIssueCondition } from "./issue-visibility.js";
|
||||
import { attentionIssueCondition, executionIssueCondition } from "./issue-visibility.js";
|
||||
import { parseIssueExecutionState } from "./issue-execution-policy.js";
|
||||
import { isProspectiveBlockedTransition } from "./routable-blocked.js";
|
||||
import { issueReadSqlCondition, type AuthorizationActor } from "./authorization.js";
|
||||
@@ -107,7 +108,6 @@ const SOURCE_RANK: Record<AttentionSourceKind, number> = {
|
||||
|
||||
const PENDING_INTERACTION_STATUSES = ["pending"] as const;
|
||||
const OPEN_RECOVERY_STATUSES = ["active", "escalated"] as const;
|
||||
const HUMAN_RECOVERY_OWNER_TYPES = ["user", "board"] as const;
|
||||
const DETAIL_EXCERPT_LENGTH = 160;
|
||||
const DETAIL_IMAGE_LIMIT = 3;
|
||||
const OPEN_DECISION_DEFAULT_LIMIT = 500;
|
||||
@@ -766,37 +766,6 @@ export function interactionResolverAudience(
|
||||
};
|
||||
}
|
||||
|
||||
function collapsePendingConfirmationsToNewest<T extends {
|
||||
id: string;
|
||||
issueId: string;
|
||||
kind: string;
|
||||
createdAt: Date;
|
||||
payload: unknown;
|
||||
}>(rows: T[]) {
|
||||
// Quality gates bind independent outputs; a newer visual approval must not
|
||||
// hide a still-pending copy approval (or affect its badge count).
|
||||
const independent = (row: T) => Boolean(row.payload && typeof row.payload === "object" && "qualityReview" in row.payload && row.payload.qualityReview);
|
||||
const newestByGroup = new Map<string, T>();
|
||||
for (const row of rows) {
|
||||
if (row.kind !== "request_confirmation" || independent(row)) continue;
|
||||
const groupKey = `${row.issueId}:${row.kind}`;
|
||||
const newest = newestByGroup.get(groupKey);
|
||||
if (
|
||||
!newest
|
||||
|| row.createdAt.getTime() > newest.createdAt.getTime()
|
||||
|| (row.createdAt.getTime() === newest.createdAt.getTime() && row.id > newest.id)
|
||||
) {
|
||||
newestByGroup.set(groupKey, row);
|
||||
}
|
||||
}
|
||||
|
||||
return rows.filter((row) => (
|
||||
row.kind !== "request_confirmation"
|
||||
|| independent(row)
|
||||
|| newestByGroup.get(`${row.issueId}:${row.kind}`)?.id === row.id
|
||||
));
|
||||
}
|
||||
|
||||
function budgetObservedPercent(amountObserved: number, amountLimit: number) {
|
||||
return amountLimit > 0 ? Math.round((amountObserved / amountLimit) * 10_000) / 100 : 0;
|
||||
}
|
||||
@@ -866,9 +835,7 @@ async function issueSummaryMap(
|
||||
.where(and(
|
||||
eq(issues.companyId, companyId),
|
||||
inArray(issues.id, ids),
|
||||
includeConversations
|
||||
? or(executionIssueCondition(), and(visibleIssueCondition(), isNotNull(issues.conversationAgentId)))
|
||||
: executionIssueCondition(),
|
||||
attentionIssueCondition(includeConversations),
|
||||
readCondition,
|
||||
));
|
||||
return new Map(rows.map((row) => [row.id, {
|
||||
@@ -1108,70 +1075,6 @@ function resolveBlockingIssue(
|
||||
return null;
|
||||
}
|
||||
|
||||
function readRunIssueId(contextSnapshot: Record<string, unknown> | null) {
|
||||
const issueId = contextSnapshot?.issueId ?? contextSnapshot?.taskId;
|
||||
return typeof issueId === "string" && issueId.length > 0 ? issueId : null;
|
||||
}
|
||||
|
||||
type FailedRunProbe = { id: string; agentId: string; issueId: string | null; createdAt: Date };
|
||||
|
||||
const FAILED_RUN_PROBE_BATCH_SIZE = 1000;
|
||||
|
||||
async function listFailedRunsWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
const withNewerRun = new Set<string>();
|
||||
const keyed = failedRuns.filter(run => run.issueId !== null);
|
||||
for (let start = 0; start < keyed.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const batch = keyed.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE);
|
||||
for (const id of await listFailedRunBatchWithNewerRun(db, companyId, batch)) withNewerRun.add(id);
|
||||
}
|
||||
const unkeyed = failedRuns.filter(run => run.issueId === null);
|
||||
if (unkeyed.length === 0) return withNewerRun;
|
||||
const agentIds = [...new Set(unkeyed.map(run => run.agentId))];
|
||||
const oldestCreatedAt = unkeyed.reduce((oldest, run) => run.createdAt < oldest ? run.createdAt : oldest,
|
||||
unkeyed[0]!.createdAt);
|
||||
const latestByAgent = new Map<string, Date>();
|
||||
for (let start = 0; start < agentIds.length; start += FAILED_RUN_PROBE_BATCH_SIZE) {
|
||||
const rows = await db.select({
|
||||
agentId: heartbeatRuns.agentId,
|
||||
createdAt: sql<Date>`max(${heartbeatRuns.createdAt})`.mapWith(heartbeatRuns.createdAt),
|
||||
}).from(heartbeatRuns).where(and(
|
||||
eq(heartbeatRuns.companyId, companyId),
|
||||
inArray(heartbeatRuns.agentId, agentIds.slice(start, start + FAILED_RUN_PROBE_BATCH_SIZE)),
|
||||
gt(heartbeatRuns.createdAt, oldestCreatedAt),
|
||||
sql`coalesce(${heartbeatRuns.contextSnapshot} ->> 'issueId', ${heartbeatRuns.contextSnapshot} ->> 'taskId', '') = ''`,
|
||||
)).groupBy(heartbeatRuns.agentId);
|
||||
for (const row of rows) latestByAgent.set(row.agentId, row.createdAt);
|
||||
}
|
||||
for (const run of unkeyed) {
|
||||
if ((latestByAgent.get(run.agentId)?.getTime() ?? 0) > run.createdAt.getTime()) withNewerRun.add(run.id);
|
||||
}
|
||||
return withNewerRun;
|
||||
}
|
||||
|
||||
async function listFailedRunBatchWithNewerRun(db: Db, companyId: string, failedRuns: FailedRunProbe[]) {
|
||||
if (failedRuns.length === 0) return [];
|
||||
const failed = sql.join(
|
||||
failedRuns.map((run) => sql`(${run.id}::uuid, ${run.agentId}::uuid, ${run.issueId}::text, ${new Date(run.createdAt.getTime() + 1).toISOString()}::timestamptz)`),
|
||||
sql`, `,
|
||||
);
|
||||
const runIssueId = sql`(${heartbeatRuns.contextSnapshot} ->> 'issueId')`;
|
||||
const runTaskId = sql`(${heartbeatRuns.contextSnapshot} ->> 'taskId')`;
|
||||
const newerRunExists = (match: SQL) => sql`exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${companyId}
|
||||
and ${heartbeatRuns.agentId} = failed.agent_id
|
||||
and ${heartbeatRuns.createdAt} >= failed.newer_from
|
||||
and ${match}
|
||||
)`;
|
||||
const rows = await db.execute(sql`
|
||||
select failed.id
|
||||
from (values ${failed}) as failed (id, agent_id, issue_id, newer_from)
|
||||
where ${newerRunExists(sql`${runIssueId} = failed.issue_id`)}
|
||||
or ${newerRunExists(sql`${runTaskId} = failed.issue_id and ${runIssueId} is null`)}
|
||||
`) as unknown as Array<{ id: string }>;
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
export function attentionService(db: Db, serviceOptions: AttentionServiceOptions = {}) {
|
||||
const openDecisionLimit = Math.min(
|
||||
Math.max(Math.trunc(serviceOptions.openDecisionLimit ?? OPEN_DECISION_DEFAULT_LIMIT), 1),
|
||||
@@ -1636,8 +1539,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
}));
|
||||
}
|
||||
const blockerAttention = issue.blockerAttention;
|
||||
if (blockerAttention?.state !== "stalled" && blockerAttention?.state !== "needs_attention") continue;
|
||||
if (blockerAttention.blockingTreeLive) continue;
|
||||
if (!terminalBlockerRequiresAttention(blockerAttention)) continue;
|
||||
const issueSummary = blockedIssueSummaries.get(issue.id) ?? null;
|
||||
const terminalIssueId = blockerAttention.terminalBlockerIssueId ?? issue.id;
|
||||
const terminalSummary = terminalBlockerSummaries.get(terminalIssueId)
|
||||
@@ -1739,7 +1641,8 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const pendingApprovalId = pendingApprovalByIssueId.get(review.id) ?? null;
|
||||
const reviewAttention = reviewAttentionByIssueId.get(review.id);
|
||||
const stalled = reviewAttention?.state === "stalled";
|
||||
if (!hasHumanParticipant && !review.assigneeUserId && !pendingApprovalId && !stalled) continue;
|
||||
if (!requiresReviewAttention({ hasHumanParticipant, hasUserAssignee: Boolean(review.assigneeUserId),
|
||||
hasPendingApproval: Boolean(pendingApprovalId), stalled })) continue;
|
||||
const issue = reviewIssueMap.get(review.id);
|
||||
if (!issue) continue;
|
||||
const dedupKey = `review:${review.id}`;
|
||||
@@ -1858,7 +1761,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
const budgetOverview = await budgetService(db).overview(companyId);
|
||||
for (const incident of budgetOverview.activeIncidents) {
|
||||
const observedPercent = budgetObservedPercent(incident.amountObserved, incident.amountLimit);
|
||||
if (incident.thresholdType !== "hard" && observedPercent < 85) continue;
|
||||
if (incident.thresholdType !== "hard" && observedPercent < ATTENTION_BUDGET_WARNING_PERCENT) continue;
|
||||
const dedupKey = `budget:${incident.policyId}:${toIso(incident.windowStart)}:${incident.thresholdType}`;
|
||||
add(createItem({
|
||||
companyId,
|
||||
@@ -1921,7 +1824,7 @@ export function attentionService(db: Db, serviceOptions: AttentionServiceOptions
|
||||
updatedAt: agents.updatedAt,
|
||||
})
|
||||
.from(agents)
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, "error")))
|
||||
.where(and(eq(agents.companyId, companyId), eq(agents.status, ATTENTION_AGENT_ERROR_STATUS)))
|
||||
.orderBy(desc(agents.updatedAt), desc(agents.id));
|
||||
|
||||
for (const agent of erroredAgents) {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { notifyExternalObjectRefresh } from "./external-object-refresh-notifications.js";
|
||||
import { notifyStatusCardIssueChange } from "./status-card-notifications.js";
|
||||
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
|
||||
import { notifyRunDispatch } from "./run-dispatch-notifications.js";
|
||||
import { notifyApprovalWork } from "./approval-work-notifications.js";
|
||||
import { notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { notifyAttentionRetentionWork, notifyCompanyRetentionWork } from "./retention-work-notifications.js";
|
||||
import { addCents, compareCents, normalizeCents } from "@paperclipai/shared";
|
||||
import { and, desc, eq, inArray, sql, or, isNull } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -163,6 +167,19 @@ async function resolveScopeRecord(db: Db, scopeType: BudgetScopeType, scopeId: s
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveSummaryScopeRecord(db: Db, companyId: string, scopeType: BudgetScopeType, scopeId: string): Promise<ScopeRecord> {
|
||||
let scope: ScopeRecord;
|
||||
try {
|
||||
scope = await resolveScopeRecord(db, scopeType, scopeId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof HttpError) || error.status !== 404 || (scopeType !== "agent" && scopeType !== "project")) throw error;
|
||||
return { companyId, name: scopeType === "agent" ? "Deleted agent" : "Deleted project",
|
||||
paused: false, pauseReason: null, budgetHeld: false };
|
||||
}
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
return scope;
|
||||
}
|
||||
|
||||
// Keep a large dashboard from filling the database pool with queued reads.
|
||||
async function mapBudgetReads<T, R>(rows: T[], read: (row: T) => Promise<R>): Promise<R[]> {
|
||||
const result: R[] = [];
|
||||
@@ -287,6 +304,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -300,7 +319,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.paused) return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -346,6 +369,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (policy.scopeType === "project") {
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await db
|
||||
.update(projects)
|
||||
.set({
|
||||
@@ -360,7 +385,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.skillSourceSync);
|
||||
const scope = await resolveScopeRecord(db, "company", policy.scopeId);
|
||||
if (scope.pauseReason !== "budget") return;
|
||||
await notifyRunDispatch(db, { companyId: policy.companyId });
|
||||
await notifyDeferredExecutionEligibility(db, { companyId: policy.companyId });
|
||||
await notifyCompanyRetentionWork(db, policy.companyId);
|
||||
await notifyExternalObjectRefresh(db, policy.companyId);
|
||||
await notifyStatusCardIssueChange(db, policy.companyId);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.agentTimers);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.routineSchedules);
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.issueMonitors);
|
||||
@@ -395,7 +424,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
async function buildPolicySummary(policy: PolicyRow): Promise<BudgetPolicySummary> {
|
||||
const scope = await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, policy.companyId, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
const { total: observedAmount, totalExact: observedAmountExact, unpricedEventCount, pendingRunCount } = observed;
|
||||
const { windowStart: start, windowEnd: end } = observed;
|
||||
@@ -466,6 +495,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
windowEnd: end,
|
||||
});
|
||||
|
||||
await notifyAttentionRetentionWork(db, { companyId: policy.companyId, sourceKind: "budget_alert" });
|
||||
if (thresholdType === "hard") await notifyApprovalWork(db, policy.companyId);
|
||||
const approval = thresholdType === "hard"
|
||||
? await db
|
||||
@@ -504,7 +534,8 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
return incident ? { incident, created: true } : null;
|
||||
}
|
||||
|
||||
async function resolveOpenSoftIncidents(policyId: string) {
|
||||
async function resolveOpenSoftIncidents(policyId: string, companyId: string) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await db
|
||||
.update(budgetIncidents)
|
||||
.set({
|
||||
@@ -523,10 +554,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
async function resolveOpenIncidentsForPolicy(
|
||||
policyId: string,
|
||||
companyId: string,
|
||||
approvalStatus: "approved" | "rejected" | null,
|
||||
decidedByUserId: string | null,
|
||||
thresholdType?: BudgetThresholdType,
|
||||
) {
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
const condition = and(eq(budgetIncidents.policyId, policyId), eq(budgetIncidents.status, "open"),
|
||||
thresholdType ? eq(budgetIncidents.thresholdType, thresholdType) : undefined);
|
||||
const openRows = await db
|
||||
@@ -560,7 +593,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const approvalStatusById = new Map(approvalRows.map((row) => [row.id, row.status]));
|
||||
|
||||
return mapBudgetReads(rows, async (row) => {
|
||||
const scope = await resolveScopeRecord(db, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
const scope = await resolveSummaryScopeRecord(db, row.companyId, row.scopeType as BudgetScopeType, row.scopeId);
|
||||
return {
|
||||
id: row.id,
|
||||
companyId: row.companyId,
|
||||
@@ -615,9 +648,9 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observed.recoveringRunCount > 0) recoveringPolicy ??= policy;
|
||||
// A closed UTC window, inactive policy, or budget raise releases only
|
||||
// budget-owned pauses. Manual pauses remain operator decisions.
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
|
||||
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
|
||||
await resolveOpenIncidentsForPolicy(policy.id, "approved", "budget_service", "soft");
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
|
||||
}
|
||||
}
|
||||
// Agent status has its own admission gate and error code. Preserve a
|
||||
@@ -674,7 +707,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
if (observedBlocks(policy, observed)) {
|
||||
await resolveOpenSoftIncidents(policy.id);
|
||||
await resolveOpenSoftIncidents(policy.id, policy.companyId);
|
||||
const hardIncident = await createIncidentIfNeeded(policy, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(policy);
|
||||
if (hardIncident?.created) {
|
||||
@@ -844,11 +877,11 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
const observed = await computeObservedSpend(db, row);
|
||||
if (!observedBlocks(row, observed)) {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service", "hard");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service", "hard");
|
||||
if (row.notifyEnabled && compareCents(observed.totalExact, Math.ceil(row.amount * row.warnPercent / 100)) >= 0) {
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
} else {
|
||||
await resolveOpenIncidentsForPolicy(row.id, null, null, "soft");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, null, null, "soft");
|
||||
}
|
||||
} else {
|
||||
const softThreshold = Math.ceil((row.amount * row.warnPercent) / 100);
|
||||
@@ -856,14 +889,14 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
await createIncidentIfNeeded(row, "soft", observed);
|
||||
}
|
||||
if (observedBlocks(row, observed)) {
|
||||
await resolveOpenSoftIncidents(row.id);
|
||||
await resolveOpenSoftIncidents(row.id, row.companyId);
|
||||
await createIncidentIfNeeded(row, "hard", observed);
|
||||
await pauseAndCancelScopeForBudget(row);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await resumeScopeFromBudget(row);
|
||||
await resolveOpenIncidentsForPolicy(row.id, "approved", actorUserId ?? "budget_service");
|
||||
await resolveOpenIncidentsForPolicy(row.id, row.companyId, "approved", actorUserId ?? "budget_service");
|
||||
}
|
||||
|
||||
await recordActivity({
|
||||
@@ -925,6 +958,13 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
if (incident.companyId !== companyId) throw notFound("Budget incident not found");
|
||||
|
||||
const policy = await getPolicyRow(incident.policyId);
|
||||
if (policy.companyId !== companyId || policy.scopeType !== incident.scopeType || policy.scopeId !== incident.scopeId) {
|
||||
throw notFound("Budget scope not found");
|
||||
}
|
||||
const scope = input.action === "keep_paused"
|
||||
? await resolveSummaryScopeRecord(db, companyId, policy.scopeType as BudgetScopeType, policy.scopeId)
|
||||
: await resolveScopeRecord(db, policy.scopeType as BudgetScopeType, policy.scopeId);
|
||||
if (scope.companyId !== companyId) throw notFound("Budget scope not found");
|
||||
if (input.action === "raise_budget_and_resume") {
|
||||
const nextAmount = Math.max(0, Math.floor(input.amount ?? 0));
|
||||
const observed = await computeObservedSpend(db, policy);
|
||||
@@ -937,6 +977,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
}
|
||||
|
||||
const now = new Date();
|
||||
await notifyAttentionRetentionWork(db, { companyId, sourceKind: "budget_alert" });
|
||||
await notifyDeliveryWork(db, DELIVERY_QUEUES.budgetEnforcement, companyId);
|
||||
await db
|
||||
.update(budgetPolicies)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork, notifyTaskRetrySettlement } from "../deferred-execution-notifications.js";
|
||||
import { notifyNativeResponsePresentation } from "../native-response-presentation-notifications.js";
|
||||
import { notifyChatInboundWakeNotices } from "../chat-inbound-wake-notice-notifications.js";
|
||||
import { createFailedRetryNotices } from "./failed-retry-notices.js";
|
||||
@@ -1424,6 +1425,7 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
// Source authorization above checked every external receipt and its owner.
|
||||
// Reset only this certified unstarted batch under the task's admission lock.
|
||||
const skippedIds = source.taskRetryInput?.skippedActionIds ?? [];
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.agentId, input.agentId),
|
||||
@@ -1784,6 +1786,9 @@ export function createChatFailedRunRetry(db: Db, options: ChatFailedRunRetryOpti
|
||||
const attemptCount = Number(candidate.result?.attemptCount ?? 0) + 1;
|
||||
const settle = async (status: string, result: Record<string, unknown>) => {
|
||||
const [current] = await db.transaction(async (tx) => {
|
||||
if (typeof claimed.payload.taskRetryIntentId === "string") await notifyTaskRetrySettlement(tx, {
|
||||
companyId: claimed.companyId, requestId: claimed.payload.taskRetryIntentId,
|
||||
});
|
||||
await notifyChatActionWork(tx, "failed_run_retry");
|
||||
if (status === "processed") await notifyNativeResponsePresentation(tx, {
|
||||
companyId: claimed.companyId, issueId: String(claimed.payload.issueId),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { notifyDeferredExecutionWork } from "../deferred-execution-notifications.js";
|
||||
import { z } from "zod";
|
||||
import { and, asc, eq, gt, inArray, isNull, or, sql } from "drizzle-orm";
|
||||
import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, type Db } from "@paperclipai/db";
|
||||
@@ -91,6 +92,7 @@ export async function releaseUnadmittedTaskRetryInput(db: Db, input: { companyId
|
||||
)).for("update");
|
||||
if (receipt?.runId) return;
|
||||
await notifyChatInboundWakeNotices(tx, { companyId: input.companyId, issueId: issue.id });
|
||||
await notifyDeferredExecutionWork(tx, { companyId: input.companyId });
|
||||
await tx.update(agentWakeupRequests).set({ status: "deferred_issue_execution", finishedAt: null,
|
||||
payload: sql`${agentWakeupRequests.payload} - 'coalescedIntoWakeupRequestId'`, updatedAt: new Date(),
|
||||
}).where(and(eq(agentWakeupRequests.companyId, input.companyId), eq(agentWakeupRequests.status, "coalesced"),
|
||||
|
||||
Loaded 100 of 247 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user