mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-11 05:11:49 +01:00
Merge current benchmark base and preserve generated cursor migration
This commit is contained in:
commit
e5007f5c52
161 files changed
+7687
-418
No files matched your search
@@ -98,7 +98,20 @@ const stackedBase = {
|
||||
|
||||
const routeCases = [
|
||||
["allowlisted PR", {}, awsRunner],
|
||||
["allowlisted profile label", { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: "profile-ci" }, awsRunner],
|
||||
...["profile-ci", "full-ci"].flatMap((label) => {
|
||||
const event = { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: label };
|
||||
return [
|
||||
[`allowlisted ${label} label`, event, awsRunner],
|
||||
[`${label} with unlisted author`, { ...event, EVENT_PR_AUTHOR_ID: "99999999" }],
|
||||
[`${label} with unlisted sender`, { ...event, EVENT_SENDER_ID: "99999999" }],
|
||||
[`${label} with unlisted rerun actor`, { ...event, TEST_TRIGGERING_ACTOR_ID: "99999999" }],
|
||||
[`${label} on non-owner PR with owner sender and rerun`, {
|
||||
...event, EVENT_PR_AUTHOR_ID: "99999999", TEST_PR_AUTHOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]",
|
||||
}],
|
||||
[`${label} with non-owner sender`, { ...event, EVENT_SENDER_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }],
|
||||
[`${label} with non-owner rerun actor`, { ...event, TEST_TRIGGERING_ACTOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }],
|
||||
];
|
||||
}),
|
||||
["unrelated label", { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: "documentation" }],
|
||||
["validated stacked base", stackedBase, awsRunner],
|
||||
["lagging PR API base snapshot", { TEST_PR_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, awsRunner],
|
||||
|
||||
@@ -86,8 +86,8 @@ jobs:
|
||||
[[ "$EVENT_BASE_REPOSITORY_ID" == '1170821064' ]] || fail_closed 'unexpected base repository ID'
|
||||
[[ -n "$EVENT_BASE_REF" ]] || fail_closed 'missing base branch'
|
||||
[[ "$EVENT_ACTION" =~ ^(opened|reopened|synchronize|labeled)$ ]] || fail_closed 'unsupported pull_request action'
|
||||
if [[ "$EVENT_ACTION" == 'labeled' && "${EVENT_LABEL_NAME:-}" != 'profile-ci' ]]; then
|
||||
fail_closed 'only the profile-ci label can request a profiling run'
|
||||
if [[ "$EVENT_ACTION" == 'labeled' && ! "${EVENT_LABEL_NAME:-}" =~ ^(profile-ci|full-ci)$ ]]; then
|
||||
fail_closed 'only the profile-ci or full-ci labels can request CI'
|
||||
fi
|
||||
|
||||
jq -e '
|
||||
|
||||
@@ -757,6 +757,43 @@ blocks concurrent task or run updates. It allows audit inserts to retain their
|
||||
foreign-key `KEY SHARE` locks without waiting on identity acquisition. The audit
|
||||
foreign keys and their deletion behavior remain enforced.
|
||||
|
||||
## Execution admission accounting
|
||||
|
||||
The execution-admission module (`server/src/modules/execution-admission/`) owns
|
||||
three tables. Nothing writes admissions or caps in production yet, and no claim
|
||||
path enforces a project maximum.
|
||||
|
||||
- `project_task_execution_policies` stores a project's maximum concurrent tasks
|
||||
apart from `projects`, so project readers never return an unenforced cap. No
|
||||
row means never configured. A row with a null maximum means an explicit "no
|
||||
limit", which a later legacy migration must not overwrite. Maximums are
|
||||
positive. The row is deleted with its project.
|
||||
- `execution_admissions` stores one admission attempt of a run. A run can be
|
||||
released before dispatch and admitted again, so `(run_id, attempt)` is unique
|
||||
and a partial unique index allows one unsettled attempt per run. The row keeps
|
||||
the context revision, the nullable primary project as provenance (no foreign
|
||||
key) and the settlement identity, which is set together with `settled_at`.
|
||||
- `execution_admission_project_charges` stores one row per distinct admitted
|
||||
project, including unlimited projects. An admission without projects has no
|
||||
rows. Composite `(company_id, …)` foreign keys keep admissions and projects in
|
||||
one company. Rows exist only while the admission is unsettled: settlement
|
||||
deletes the whole set in one transaction, and `(company_id, project_id)` is
|
||||
indexed for active counts, so lookups scale with active work, not history.
|
||||
|
||||
Archiving a project changes no charge. Deleting a project with a charge returns
|
||||
`409 project_execution_admission_pending`; the foreign key is the backstop.
|
||||
Deleting an agent refuses unsettled admissions on its runs and removes settled
|
||||
ones. Company deletion removes the company's admissions before its runs and
|
||||
projects.
|
||||
|
||||
The proposed global lock order for the admission transaction is: company row
|
||||
(`FOR NO KEY UPDATE`), native finalization recovery owner, the whole project set
|
||||
in ascending ID order (`FOR NO KEY UPDATE`, one statement), issues, wake
|
||||
requests, the run, the external-admission advisory lock, the admission row and
|
||||
its charges, then budget reservations. Policy updates take the same project
|
||||
lock. Settlement takes no project lock. Only the project-set step is implemented
|
||||
so far; claim paths keep their current locks until they are integrated.
|
||||
|
||||
## Switching between modes
|
||||
|
||||
The database mode is controlled by `DATABASE_URL`:
|
||||
|
||||
@@ -164,6 +164,9 @@ history, changed PR heads and unverifiable ancestry still deny AWS execution.
|
||||
Missing or malformed settings
|
||||
and unverifiable authorization use GitHub-hosted runners. A maintainer rerunning
|
||||
an outside contributor's PR does not grant it AWS access.
|
||||
Both `full-ci` and `profile-ci` label events use this same authorization path.
|
||||
The labels request CI scope; they do not grant AWS access or move an authorized
|
||||
CODEOWNER run to GitHub-hosted capacity. Other label events do not request CI.
|
||||
Authorized test jobs and the final `verify` and `e2e` aggregates use the selected
|
||||
AWS fleet. A stale admission roster can allocate only the short metadata gate;
|
||||
the fresh CODEOWNERS check still denies PR execution for a removed owner.
|
||||
@@ -1848,6 +1851,17 @@ host, sandbox, or Git remote. Files changed since the last save still depend on
|
||||
the live sandbox: losing that allocation can lose those edits. There is no
|
||||
periodic workspace backup in this release.
|
||||
|
||||
Remote workspace command timeouts report `remote_workspace_operation_timed_out`
|
||||
with the operation (`prepare`, `repository`, `archive_helper`, `capture`, `restore`,
|
||||
or `cleanup`) and its timeout in milliseconds. Command output is excluded from
|
||||
that diagnostic. A timeout does not prove the remote process stopped or that a
|
||||
save failed before publication. Reconcile the existing workspace and operation
|
||||
receipts before recovery; the transport does not automatically repeat the command.
|
||||
Export cleanup keeps this typed timeout diagnostic in the operation's
|
||||
`metadata.exportCleanup.lastError`. Other cleanup errors retain a generic message.
|
||||
The saved snapshot stays acknowledged and the cleanup obligation stays pending
|
||||
until the existing reconciliation path settles it.
|
||||
|
||||
Files reads use the current committed saved version when remote authority is
|
||||
clean and has no active writer, even while its old allocation remains recorded.
|
||||
The saved loader verifies the company, workspace, snapshot, and archive before
|
||||
@@ -1878,6 +1892,34 @@ or recovery, unsafe paths, and explicitly concurrent shared writers block
|
||||
import. An interrupted apply retains its journal and admission hold; retry the
|
||||
same command to reconcile it before starting another writer.
|
||||
|
||||
An import preserves the workspace's lifecycle state and the task's current
|
||||
workspace binding. It can import an archived or superseded workspace without
|
||||
reopening the task or selecting that older workspace for future turns. A changed
|
||||
task binding during the import blocks publication.
|
||||
|
||||
If an archived folder has already been removed, the importer can reconstruct it
|
||||
in private scratch space from its encrypted task-directory restore point or
|
||||
validated Git restoration recipe. For older Git workspaces without a recipe,
|
||||
it requires a successful branch-deletion receipt and the exact saved commit in
|
||||
the verified source repository. It never substitutes today's default branch.
|
||||
Git restore points retain unpushed history and staged and unstaged edits, even
|
||||
when the source repository has pruned the saved branch. Missing, expired,
|
||||
corrupt, or unbound sources block the import. Publication still requires an
|
||||
independent download and restore of the new encrypted snapshot. The old folder
|
||||
remains absent, and the importer does not change the original restore point.
|
||||
|
||||
Shared physical folders require Cloud's offline enrollment maintenance process.
|
||||
A task-level hold cannot fence another workspace record pointing at the same
|
||||
folder. Maintenance verifies that legacy deployments have stopped and that no
|
||||
runs are active or queued, then imports nested task directories before parent
|
||||
roots. A parent can omit a runtime-created nested task folder only after an
|
||||
independent restore and source comparison prove that its committed snapshot
|
||||
covers those bytes. The parent snapshot records that dependency, and retention
|
||||
pins the child snapshot until the parent is deleted. Changed or unavailable
|
||||
nested data blocks publication. Ordinary user folders remain in the archive.
|
||||
Interrupted maintenance resumes the same import journals; completed records
|
||||
leave the pending set. Original folders remain on the retained volume.
|
||||
|
||||
Supported sources include empty task folders, repository URLs, and managed
|
||||
mutable repositories. Authorized mentioned projects attach all their configured
|
||||
Git repositories as writable copies under `.paperclip-repositories/` in the task
|
||||
|
||||
@@ -1729,7 +1729,10 @@ Scheduler must skip invocation when:
|
||||
- an existing run is active
|
||||
- hard budget limit has been hit
|
||||
|
||||
Budget admission and accounting serialize under the company accounting lock.
|
||||
Authoritative paid-work admission and accounting mutations serialize under the
|
||||
company accounting lock. Preliminary invocation checks use a read-only snapshot
|
||||
and take that lock only when a hold, incident, or enforcement transition needs
|
||||
reconciliation; reconciliation rereads current state under the lock.
|
||||
The dispatch boundary rechecks current company, agent, and project policies and
|
||||
reserves capacity before paid work starts. Accounting commits budget blocks and
|
||||
durable stop intent without waiting for provider shutdown or reconciling unrelated
|
||||
|
||||
@@ -63,6 +63,10 @@ never persisted or published on the shared company event bus.
|
||||
Authorized events still reach cache handling and component subscribers when
|
||||
eligibility is false. Lookup failure is logged but does not discard the authorized
|
||||
event. Existing task/run privacy checks and payload redaction remain in force.
|
||||
Task activity titles, fallback text, and links use the authorized task identifier.
|
||||
If the optional identifier lookup fails, the event still arrives and uses `Task`
|
||||
unless the browser already has the identifier. Permission-check failures do not
|
||||
use this fallback.
|
||||
When an authenticated comment event omits the comment ID, the visible task thread
|
||||
refreshes through its authorized HTTP read instead of trying to hydrate the comment.
|
||||
Closed sockets cannot deliver stale notifications after account/company changes.
|
||||
|
||||
@@ -756,6 +756,8 @@ Normal issue-thread feedback can still wake the current assignee according to th
|
||||
|
||||
Restart recovery revisits accepted deferred messages through the latest issue run, including historical mention queues for another agent. Ordinary messages for a former owner retain their assignment handoff. Native sources require their matching committed finalization, settled environment leases, and released finalizer and copyback ownership before recovery clears old issue claims. A successful retained warm release can satisfy the lease check. Released local bookkeeping without a provider allocation and a matching SSH bookkeeping release need no provider cleanup receipt. Plugin releases record their successful acknowledgement. Pending or failed cleanup and durable owners still hold work. A successful fenced cleanup completion clears its in-flight claim in the same database write, so a restart before retry bookkeeping cannot strand saved input. The normal release checks preserve cleanup holds, pauses, and operator Stops. Recovery checks for a competing turn under the issue lock before it promotes saved input.
|
||||
|
||||
Repeated eligibility notifications must not move an unchecked, ready deferred receipt behind receipts already reconsidered. Paginated reconstruction preserves pending ready receipts' dispatch priority while refreshing their current state. Receipts whose wait deadline has passed join the ready queue on reconstruction. Future wait grace periods, claimed-work deadlines, infrastructure retry backoffs, and current admission checks remain effective.
|
||||
|
||||
A paused recipient does not fail an accepted deferred request. Release and periodic recovery preserve its pending receipt and may process an eligible request behind it. Resuming the agent allows normal admission to reconsider the saved input. Missing, foreign, terminated, and unapproved recipients retain their rejection rules. Explicit task holds and operator Stops still prevent unauthorized execution. This does not replay historical failed requests.
|
||||
|
||||
Accepted agent feedback must survive a child changing to `done` before its active run exits. Deferred wake promotion may reopen that completed child only for its current assignee, with explicit agent resume intent and live tracked comments from another author. Claim promotion before reopening. Cancelled tasks, deleted comments, self-authored comments, empty continuations, and agent continuations without explicit intent keep their existing suppression rules. Normal pause, ownership, authorization, and budget gates still apply.
|
||||
@@ -949,6 +951,8 @@ The task thread exposes the guarded Retry action for failed or timed-out convers
|
||||
|
||||
A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed.
|
||||
|
||||
Other legacy queued-run gate cancellations also stay out of unknown-action reconciliation when the terminal run has no retained execution authority, session, workspace recovery, resources, or execution events. The same proof removes the effect of an old false recovery hold only for its exact company, task, and run; the audit row remains. Started or conflicting work retains its hold. Review waits retain their guarded Retry path. New work still passes the current approval, dependency, budget, pause, and ownership gates.
|
||||
|
||||
A successful connected run can still have newer admitted input waiting behind it.
|
||||
Task Retry resumes that input's existing durable chat receipt through the inbound
|
||||
owner, instead of replaying the successful run as a failed attempt. The original
|
||||
@@ -1600,8 +1604,9 @@ agent, workspace, retained session, and termination of prior execution. The
|
||||
checkpoint must match the exact Runner, lease, run, session, turn, and item.
|
||||
A missing restart claim does not remove these checks or authorize a fresh session.
|
||||
|
||||
After a plugin session passes provider recovery and durable run admission, its
|
||||
local suspension receipt is removed. The running session then owns authority;
|
||||
After a remote sandbox session passes provider recovery and durable run admission,
|
||||
its local suspension receipt is removed. This applies to built-in sandbox runners
|
||||
and plugin tasks. The running session then owns authority;
|
||||
the old receipt must not conflict with a later warm run's journal identity.
|
||||
Failed admission retains the receipt. Normal suspension writes a new receipt.
|
||||
|
||||
@@ -3027,7 +3032,12 @@ resumption path. This does not consume a provider attempt or stop the retained
|
||||
sandbox. A locked run/coordinator check rejects Stop, changed ownership or any
|
||||
provider progress. Startup restores even an overdue wait to the same scheduler;
|
||||
ordinary resumption rechecks current permissions and task authority. No idle
|
||||
scan is added. Per-turn continuation reconstructs a missing suspended-state
|
||||
scan is added. The same wait applies when a compatible successful predecessor
|
||||
still has a live finalization lease, even with full local controller projections.
|
||||
An expired or undated owner, incomplete canonical commands and incompatible
|
||||
execution bindings retain their existing reconciliation holds. Once the owner
|
||||
releases its lease, the existing resumption claim admits the same run once.
|
||||
Per-turn continuation reconstructs a missing suspended-state
|
||||
receipt from the admitted sandbox or prepared-task owner, including after a
|
||||
previous read failed with only a PRP projection written.
|
||||
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
# Authenticated incoming-wake proof
|
||||
|
||||
The Cloud control signer can attach `idleCapabilities.incomingWake` to an owned
|
||||
`task-drain:read` assertion. The tenant verifies the existing stack, action,
|
||||
audience, expiry and one-use request ID, then validates the closed capability
|
||||
shape. A tenant query, browser session, API key or forwarded identity header
|
||||
cannot install this capability. Unknown versions and routes fail closed.
|
||||
|
||||
The report uses outer version 3 and includes the exact version-1 incoming proof.
|
||||
Cloud compares its requested capability and the initial/final proof before stop.
|
||||
Without this capability, versions 1/2 and all existing ingress blockers remain.
|
||||
Older Cloud/app combinations retain their conservative behavior.
|
||||
|
||||
## Supported profiles
|
||||
|
||||
`tenant-api-http-v2` allows saved agent and board API key rows to remain while
|
||||
sleeping. Cloud resolves the target from its verified host registry, coalesces a
|
||||
bounded wake, waits for tenant readiness, refreshes routing, then forwards the
|
||||
original HTTP request once. Ordinary `/api/instance` administration routes are
|
||||
included. The separately reserved Muse protocol, non-API MCP routes, upgrades,
|
||||
encoded/noncanonical paths and methods outside GET/HEAD/OPTIONS/POST/PUT/PATCH/
|
||||
DELETE do not gain this wake path.
|
||||
|
||||
The Cloud capability authenticates the **host's ability to receive work**. Opaque
|
||||
tenant keys are authenticated only by the awakened tenant, under current key,
|
||||
expiry, user, membership, company, agent and resource permissions. No raw key or original
|
||||
credential hash is copied into Cloud. A signed final-drain projection supplies
|
||||
only per-cycle derived wake fingerprints. Unknown inputs cannot consume
|
||||
provider-start quota. Tenant authentication still rejects any later revocation.
|
||||
|
||||
Successful current board/agent key authentication followed by a successful HTTP
|
||||
response retains ten minutes of local API activity, measured with a monotonic
|
||||
clock. Further successful use extends the deadline. Rejection, expiry, revoked
|
||||
membership, failed requests and drain refusals do not extend it. The existing
|
||||
local-work probe observes this timestamp; there is no timer, SQL, renewal or
|
||||
network work from the hold, and browser-entry metrics are unchanged.
|
||||
|
||||
`github-broker-events-v1` uses the durable Cloud connector queue from the existing
|
||||
connector-event wake protocol. The signed proof binds the current enrolled
|
||||
connector instance and canonical fingerprint of all current GitHub bindings.
|
||||
The tenant reconstructs those bindings through the original connection-event
|
||||
owner and exact completed GitHub bot registration ownership. An active broker
|
||||
bot can retain its endpoint and paired channel connection only when setup has
|
||||
finished and every binding matches. Missing, foreign, changed, unsupported or
|
||||
unsettled binding state prevents sleep. Original GitHub signature verification
|
||||
and current endpoint/connection authorization still run on delivery.
|
||||
|
||||
Personal GitHub MCP connections additionally owe credential continuity work.
|
||||
This change exposes their required connection IDs but keeps them blocked. The
|
||||
stacked domain-deadline change must supply its inspected future continuity proof
|
||||
before those rows can be exempted. A signed catalog name alone is insufficient.
|
||||
Legacy scheduled alarms remain conservative when combined with ingress until
|
||||
that downstream composition is present.
|
||||
|
||||
## Delivery and admission guarantees
|
||||
|
||||
Synchronous HTTP is not durable acceptance. A request arriving during stopping
|
||||
or cold start is held only before forwarding, under finite capacity/time bounds.
|
||||
An active-route race with tenant drain returns its real 503 and Retry-After;
|
||||
clients retry through their normal API/domain idempotency contract. A streamed
|
||||
mutation is never automatically replayed after an ambiguous upstream outcome.
|
||||
Wake/readiness failure, manual suspension, deployment recovery holds, wrong
|
||||
hosts and exhausted admission return non-success without acknowledging work.
|
||||
|
||||
Broker events have stronger persistence: the existing queue target survives a
|
||||
lost wake hint, failed wake, process restart and lease expiry. A 202 hint does not
|
||||
settle delivery. Only the current event lease acknowledgment does. Duplicate
|
||||
transport delivery uses the domain's existing receipts/atomic claims.
|
||||
|
||||
Cloud API admission is explicitly bounded per process: 64 waiting requests,
|
||||
8 per stack, 4 per source, 32 pending provider preparations; source request
|
||||
budget 24/minute, source wake budget 3/10 minutes, stack wake budget 8/hour,
|
||||
and aggregate wake budget 32/minute. Readiness waiting is at most 90 seconds.
|
||||
A timed-out provider preparation remains coalesced until it actually settles.
|
||||
The stack budget accommodates ordinary ten-minute quiet cycles with boundary
|
||||
headroom. These are availability limits, not an unrestricted API promise.
|
||||
Only the existing explicitly configured trusted-proxy hop contract may resolve
|
||||
a forwarded source IP; otherwise the socket address is used. A shared proxy/NAT
|
||||
therefore shares its source limits and can receive truthful retry refusals.
|
||||
|
||||
## Explicit remaining coverage
|
||||
|
||||
Direct GitHub, routine, Slack, voice and X webhooks retain their existing
|
||||
configured-ingress blockers until their exact response budget, retry/retention,
|
||||
authenticated wake and replay contracts are qualified. Cloud X forwarding by
|
||||
itself does not establish wake. Saved outbound HTTP MCP remains passive under
|
||||
its existing narrow rule; public MCP/SSE subscriptions, Dot, Muse and gateway
|
||||
resources retain their own blockers. Discord Gateway, local MCP stdio, and other
|
||||
required persistent sockets need owner-proved stop, reconnect and sequence replay.
|
||||
Telegram/Teams/Photon transports require their own actual producer/lifecycle
|
||||
classification. Hosted previews belong to their separate ingress changes.
|
||||
|
||||
The source/loopback/PostgreSQL tests establish the exercised protocol boundaries;
|
||||
they do not establish a fleet deployment, actual provider/Neon suspension,
|
||||
provider startup latency or cost savings. Deployed combined qualification remains
|
||||
separate and must include every enabled transport.
|
||||
|
||||
|
||||
## Pre-wake API admission
|
||||
|
||||
The v2 API profile requires an authenticated per-sleep-cycle key projection before compute stops. The signed challenge is derived from the newly random owned idle hold and stack. In the same read-only snapshot, the app projects up to 1000 active key rows into domain-separated salted fingerprints; original key hashes and raw keys stay private. Board-key expiry is carried and enforced before wake. Unknown, duplicate, malformed or overflow projection fails closed. The complete initial and final receipts must match; changing a credential or challenge refuses stop.
|
||||
|
||||
Cloud stores the final receipt in one internal stack column under the sleep operation lock. Unknown requests cannot consume provider-start quota. Possession of a matching snapshot key only admits a bounded wake; tenant authentication and current permissions still run on delivery. A key revoked after the snapshot can therefore receive no tenant authority, even if it once qualified for wake. Wake claims recheck the exact cycle, matching key and expiry atomically. Normal next sleep re-reads revocations. Successful wake, manual/archive/delete or a known sleep abort clears the receipt; uncertain stop and a same-cycle failed wake retain recovery. A restarted host reads the durable receipt. No global TTL strands nonexpiring saved keys during long sleep, and no periodic credential refresh worker is added.
|
||||
|
||||
|
||||
Removed GitHub endpoints are retained history only after the ordinary removal
|
||||
writer archives the endpoint with its timestamp and generation, archives and
|
||||
disables the same-company channel connection, clears its credential references,
|
||||
and releases endpoint leases. Any retained lease, incomplete registration or
|
||||
malformed cleanup state still blocks. This exemption covers only GitHub; archived
|
||||
persistent transports remain blocked. Actions, deliveries and publications keep
|
||||
their separate completion checks even when their endpoint has been removed.
|
||||
|
||||
The service also keeps its existing local-work probe active while runtime
|
||||
initialization or retirement is unsettled. Both supported runtime registries
|
||||
report failed retirement even after `get` and `list` hide the runtime. Successful
|
||||
owner retry clears that debt; a failed full service shutdown retains the probe.
|
||||
These checks read process-local state and add no polling, network or SQL.
|
||||
@@ -644,3 +644,13 @@ retry dates and package/object/native-asset retention timestamps are earliest
|
||||
maintenance eligibility, not promised delivery deadlines. Workers reconstruct persisted rows and filesystem
|
||||
access leases on startup, and use the existing coordinator's drain/standby/shutdown
|
||||
controls. No external Cloud alarm or provider suspension is established by these tests.
|
||||
|
||||
|
||||
## Negotiated incoming work
|
||||
|
||||
A signed, stack/owner-scoped Cloud incoming-wake capability can produce a version-3
|
||||
safety report for saved API keys and exactly bound brokered GitHub bot endpoints.
|
||||
Tenant authorization still runs after wake. Personal GitHub credential continuity
|
||||
and unsupported transports retain their blockers. See [authenticated incoming
|
||||
wake proof](idle-incoming-wake.md) for scope, HTTP versus durable-event guarantees,
|
||||
rate/availability bounds, compatibility and remaining transport dependencies.
|
||||
@@ -45,10 +45,15 @@ Keep credential values out of logs and diagnostic reports.
|
||||
## Budget admission, accounting, and enforcement
|
||||
|
||||
The operator-gated `paperclip.budgets` tracer records
|
||||
`budget.admission.inspect`, `budget.admission.reconcile`,
|
||||
`budget.admission.evaluate`, `budget.admission.reserve`,
|
||||
`budget.accounting.transaction`, `budget.accounting.settlement`, `budget.accounting.lock_wait`,
|
||||
`budget.exposure.evaluate`, `budget.enforcement.deliver`,
|
||||
`budget.enforcement.scope`, `budget.cancellation.deliver`, and `budget.accounting.recover`.
|
||||
Inspection spans cover the preliminary read-only snapshot. Reconciliation spans
|
||||
exist only when that inspection finds a transition requiring the company lock;
|
||||
compare their counts to measure how often preliminary checks bypass the lock.
|
||||
Authoritative reservation and transactional evaluation retain their own spans.
|
||||
Transaction spans include commit time. Lock-wait spans cover acquisition of the
|
||||
company accounting lock. Settlement spans cover the ledger and reservation
|
||||
transaction. Enforcement spans measure asynchronous delivery; cancellation
|
||||
|
||||
@@ -15,6 +15,36 @@ and dependent [product PR #18069](https://github.com/paperclipai/paperclip/pull/
|
||||
Neither PR is merged. Product PR #18069's Verification section owns the latest
|
||||
candidate, check status and final acceptance decision.
|
||||
|
||||
## Hosted placement correction — 2026-10-30
|
||||
|
||||
Native Cursor Cloud uses a provider API controller. Cursor hosts the model and
|
||||
commands; Paperclip runs only its qualified API client and bounded private artifact
|
||||
stager. This placement has no Paperclip Computer, lease, task checkout or host
|
||||
provisioning command. It is derived from the validated native Cursor profile and
|
||||
persisted with the admitted run; recovery retains that backend and placement.
|
||||
Earlier Cursor profiles retain their admitted workspace binding as identity;
|
||||
recovery does not reopen that workspace or allocate a Computer. Known backend
|
||||
and driver contradictions fail closed, while historical missing driver values
|
||||
remain recoverable.
|
||||
The generic managed-sandbox-only policy and local Computer denial stay intact.
|
||||
Kubernetes-only execution remains unsupported in save, Test, inventory and run
|
||||
admission. An unavailable Computer does not advertise local Legacy execution.
|
||||
Legacy direct agents retain their existing selection and execution behavior.
|
||||
|
||||
Ordinary native setup clears the Computer binding and explains Cursor's hosted
|
||||
environment. Save/reopen and Test use the same account/configuration contract as
|
||||
run admission. The Cloud front door must pass the exact authenticated native-run
|
||||
callback to the tenant; Core still enforces live run authority and per-turn scope.
|
||||
|
||||
The ordinary Cloud card is available only with native rollout and a qualified
|
||||
controller. A server-held new setup draft can use that managed API default before
|
||||
model selection. It keeps its setup hold and does not inherit another harness's
|
||||
model or permissions. Unsaved automatic Test uses the advertised native default;
|
||||
saved Legacy agents still require an explicit conversion.
|
||||
|
||||
This correction requires fresh hosted acceptance on its final image. Local and
|
||||
synthetic checks do not qualify that deployed journey or erase prior failures.
|
||||
|
||||
## Review correction checkpoint — 2026-10-29 20:25 UTC
|
||||
|
||||
All six original acceptance gates have passed on their recorded candidates;
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
# Accounting contention and best-effort budget enforcement
|
||||
|
||||
Date: 2026-11-03
|
||||
Status: Investigation complete; first admission optimization implemented locally, remaining stages proposed
|
||||
Source revision: `367f29478d14003cc5c0f161b7ce8f3164993634`
|
||||
|
||||
Follow-up: [Lock-free accounting design](2026-11-03-lock-free-accounting-design.md)
|
||||
defines the requested target architecture. Its append-only receipt path replaces
|
||||
the run-lock proposal below; this document retains the investigation evidence
|
||||
and the possible behavior-preserving first migration step.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Keep receipt persistence, deduplication, final settlement, and audit transactional.
|
||||
Make budget detection and automatic stopping a measured, short-lag control. Move
|
||||
frequent receipt writes to transactions scoped to their run or paid operation.
|
||||
Evaluate budget policies in coalesced batches outside those transactions.
|
||||
|
||||
First remove avoidable admission transactions and measure the remaining critical
|
||||
sections. Do not begin with distributed spending allocations or a new accounting
|
||||
service. A short lock at an infrequent boundary can be acceptable; serializing
|
||||
every company's receipt stream and admission preflight is the scaling concern.
|
||||
|
||||
The first implementation now inspects preliminary eligibility in a repeatable-read,
|
||||
read-only snapshot. Healthy checks and checks with no policies bypass the company
|
||||
writer lock. Holds, hard blocks, recovering accounting and incidents needing
|
||||
resolution fall back to the existing mutating owner, which revalidates current
|
||||
state under the lock. Transactional admission, reservations and receipt accounting
|
||||
retain their authority. Separate inspection/reconciliation spans expose the bypass
|
||||
ratio. This does not implement the later append-only accounting architecture.
|
||||
|
||||
Configured-policy tradeoffs: a healthy check adds an incident lookup, and an
|
||||
unheld blocking/recovering scope can compute exposure once during inspection
|
||||
and again during locked revalidation. Already-held scopes request reconciliation
|
||||
before computing exposure. Qualify configured-policy workloads as well as the
|
||||
predominant no-policy case; the change is not a claim of lower CPU everywhere.
|
||||
|
||||
The first milestone's budget-service suite passes 18 tests against real
|
||||
PostgreSQL. A controlled writer holds the company accounting lock while routine
|
||||
checks finish; authoritative admission on the same reader instead reaches its
|
||||
500 ms lock timeout. Coverage includes healthy/warning policies, manual and
|
||||
orphaned pauses, hold release, incident cleanup and cross-company rejection.
|
||||
This establishes the lock-bypass contract, not deployed latency or CPU savings.
|
||||
Seven additional accounting/enforcement/replay/tracing suites pass 55 tests,
|
||||
for 73 passing targeted tests across eight files with no skips.
|
||||
The full workspace typecheck passes via `node scripts/typecheck-low-memory.mjs`,
|
||||
and `node scripts/check-module-boundaries.mjs` passes.
|
||||
The full workspace build passes via `PAPERCLIP_TYPECHECK_LOW_MEMORY=1 pnpm -r
|
||||
--workspace-concurrency=1 build`, including the current-source Runner release
|
||||
binary, server, UI and CLI. The initial incomplete local Copilot build artifact
|
||||
was repaired with the pinned installer before the successful canonical retry.
|
||||
|
||||
This proposes a product-contract change for the later stages. It does not change
|
||||
the current spec or silently redefine existing configured budgets. The first
|
||||
implementation stage below preserves existing enforcement semantics.
|
||||
|
||||
## What the investigation established on the source revision above
|
||||
|
||||
### The lock protects several different responsibilities
|
||||
|
||||
[`withAccountingTransaction`](../../server/src/services/accounting-transaction.ts)
|
||||
takes `FOR NO KEY UPDATE` on the company row before invoking its callback. It
|
||||
holds that lock until the transaction ends. Different companies can proceed
|
||||
independently; writers for one company serialize. Ordinary report reads already
|
||||
use a read-only repeatable-read snapshot and do not take this writer lock.
|
||||
|
||||
| Path | Work sharing the company lock before this change |
|
||||
| --- | --- |
|
||||
| Usage receipt | Lock run, validate source/sequence and replay hash, insert immutable receipt, update running usage, evaluate affected budgets if monetary exposure changed |
|
||||
| Admission | Read company/agent/project state and policies, calculate exposure, reconcile budget holds/incidents, reserve estimated capacity before dispatch |
|
||||
| Final settlement | Recover durable receipt files, validate completion, settle reservation, insert ledger entries, update projections and runtime totals, acknowledge the run, evaluate budgets |
|
||||
| Other mutations | Direct cost reports, billing imports/corrections, accounting repairs, paid browser/decision/fast-response/classifier operations, and agent lifecycle operations |
|
||||
|
||||
The wrapper was unconditional. Having no configured budget did not bypass the
|
||||
lock. The public `getInvocationBlock` method also used this mutating wrapper.
|
||||
The implementation above separates its preliminary reads while preserving
|
||||
the reconciliation and resume effects that some callers need.
|
||||
|
||||
Primary owners are [usage receipts](../../server/src/services/usage-receipts.ts),
|
||||
[budgets](../../server/src/services/budgets.ts),
|
||||
[reservations](../../server/src/services/budget-reservations.ts), and
|
||||
[run settlement](../../server/src/services/run-cost-accounting.ts).
|
||||
Queue, dispatch, and lifecycle code should consume their contracts rather than
|
||||
implement another budgeting system.
|
||||
|
||||
### The current guarantee is already narrower than a dollar cap
|
||||
|
||||
Reservations serialize admission against known exposure and estimated holds.
|
||||
They do not restrict how much an already-dispatched provider can charge.
|
||||
`reservationCents` defaults to zero. A zero reservation still prevents duplicate
|
||||
dispatch, but does not reserve a positive allowance for future model spend.
|
||||
|
||||
The [implementation spec](../SPEC-implementation.md) explicitly acknowledges that
|
||||
mid-run enforcement depends on available provider reports or supported prices.
|
||||
Providers reporting only at completion cannot provide exact mid-call detection.
|
||||
Browser work can pass a provider cost cap; preserve such supported limits.
|
||||
|
||||
The current rules are approximately:
|
||||
|
||||
- Admission counts ledger spend plus the maximum of each unfinished operation's
|
||||
reservation and observed spend, plus separate browser exposure.
|
||||
- Threshold stopping uses ledger plus observed running spend, rather than
|
||||
treating reservation estimates as money already spent.
|
||||
- Pending terminal accounting and configured unknown-price restrictions can
|
||||
independently block work.
|
||||
- Final settlement replaces live exposure with the ledger charge atomically.
|
||||
|
||||
Accurate recording of received evidence and instantaneous control of external
|
||||
spend are separate contracts. Some prices remain estimates or unknown even when
|
||||
their receipt is recorded correctly.
|
||||
|
||||
### The critical section can do substantial work
|
||||
|
||||
When applicable policies exist, [exposure evaluation](../../server/src/services/budget-exposure.ts)
|
||||
executes four sequential aggregate statements over ledger entries, held
|
||||
reservations joined to runs, pending/live runs, and browser work. Policies are
|
||||
batched into those statements; it is not four statements per policy. With no
|
||||
billable policies, these four queries are skipped.
|
||||
|
||||
A changed monetary receipt can invoke this evaluation synchronously. Duplicate,
|
||||
stale, unchanged, and subscription receipts take different branches; not every
|
||||
receipt performs the aggregate scans. Native delivery already suppresses
|
||||
accepted source-event replays in its receiver, so duplicate suppression is a
|
||||
measurement question rather than an assumed large win.
|
||||
|
||||
The hot inputs are usage and turn-lifecycle events, not every streamed text
|
||||
token. Native capture handles usage reports and turn start/terminal events.
|
||||
Legacy capture also durably spools before its database attempt. Recovery can
|
||||
still read receipt files while the settlement transaction holds the company
|
||||
lock, although it builds an initial file-identity index outside the lock.
|
||||
|
||||
[Monthly spend projections](../../server/src/services/costs.ts) update
|
||||
`companies.spentMonthlyCents` for each ledger insert, and the applicable agent
|
||||
counter. Merely deleting the explicit company lock would leave an implicit
|
||||
shared-row write bottleneck and break assumptions around settlement and repair.
|
||||
PostgreSQL updates acquire conflicting row locks too. See the
|
||||
[PostgreSQL locking contract](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS).
|
||||
|
||||
### Live traces justify a focused optimization, not a CPU diagnosis
|
||||
|
||||
Read-only observability queries examined one live host. The matched one-hour
|
||||
sample is **2026-11-03 16:45–17:45 UTC**, service revision
|
||||
`3910352c923afda320b2a5134e523932ee1d0633`.
|
||||
The aggregate measurements are reproduced below. Private trace links and host
|
||||
identifiers remain outside the repository.
|
||||
|
||||
| Observed span | Count | p50 | p95 | p99 | Maximum |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| Company lock acquisition | 9,421 | 13.00 ms | 206.85 ms | 725.14 ms | 4,301.53 ms |
|
||||
| Accounting transaction | 9,421 | 87.16 ms | 486.95 ms | 1,147.10 ms | 5,594.19 ms |
|
||||
| Admission evaluation | 8,082 | 43.45 ms | 166.84 ms | 342.13 ms | 1,212.92 ms |
|
||||
| Final settlement | 216 | 355.90 ms | 1,894.05 ms | 3,149.67 ms | 5,594.33 ms |
|
||||
|
||||
These are span counts; nested spans must not be added as independent operations.
|
||||
There were 23 `budget.admission.reserve` spans in the same window. The much larger
|
||||
admission-evaluation count warrants caller attribution, not an assumption that
|
||||
all evaluations represent new paid dispatches.
|
||||
|
||||
Of 8,292 exposure evaluations on that host in that hour, **8,280 had zero policies**
|
||||
and 12 had one. That is 99.86% with zero policies at this call boundary; it does
|
||||
not prove that every company has no budgets. These empty evaluations are cheap,
|
||||
but their surrounding admission/accounting work still takes the company lock.
|
||||
|
||||
One inspected admission transaction took 4.183 seconds, with 4.090 seconds in
|
||||
lock acquisition. Its exposure child had zero policies and took about 0.026 ms.
|
||||
A separate settlement transaction took 5.594 seconds, including 4.302 seconds
|
||||
in acquisition and a zero-policy exposure evaluation.
|
||||
Both inspected traces report mean sample rate 1.
|
||||
|
||||
Limits of this evidence:
|
||||
|
||||
- The acquisition span measures the complete awaited lock query. It includes
|
||||
database/network round trips and Node scheduling; it is not a PostgreSQL
|
||||
lock-wait measurement. Transaction duration also includes transaction setup
|
||||
and commit. Do not subtract unrelated percentiles to estimate hold time.
|
||||
- The spans do not identify the locking company, blocker, receipt frequency,
|
||||
or operation caller. They do not establish one company's throughput ceiling
|
||||
or the cause of the previously observed CPU utilization.
|
||||
- The sample identifies one host. It does not establish fleet-wide behavior or
|
||||
independently qualify the public instance serving each request.
|
||||
- The local accounting wrapper, receipt, exposure, budget service, and run-cost
|
||||
files match the traced revision in Git. Other deployed components and database
|
||||
workload have not been qualified against this checkout.
|
||||
- The existing [budget benchmark](../../scripts/benchmark-budget-enforcement.ts)
|
||||
varies unrelated idle-agent count. It does not test concurrent receipt writers,
|
||||
10,000 active runs, or growing ledger history. It was inspected, not run here.
|
||||
|
||||
## Proposed behavior
|
||||
|
||||
| Responsibility | Proposed contract |
|
||||
| --- | --- |
|
||||
| Receipt acceptance | Durable commit before acknowledgement; source ownership, sequence, hash conflicts and company scope remain enforced |
|
||||
| Ledger and settlement | Idempotent transaction scoped to the run/invocation; no lost charges, duplicate application, or transient live-plus-final double counting |
|
||||
| Budget detection | Coalesced evaluation from committed evidence; a measurable delay is allowed |
|
||||
| New work | Check current authoritative pause/Stop and known budget-block state at dispatch; concurrent admission against a not-yet-reflected spend increase can overshoot |
|
||||
| Automatic cancellation | Commit durable stop intent, then use the existing scoped cancellation worker; provider shutdown remains a separate obligation |
|
||||
| Manual Stop and authorization | Preserve current execution-authority checks; budget approximation does not weaken them |
|
||||
| Unknown accounting | Retain unresolved obligations and existing configured unknown-usage restrictions; missing evidence is never a zero-cost receipt |
|
||||
|
||||
Suggested initial service targets, to validate before adopting as promises:
|
||||
|
||||
- Coalesce dirty budget work over approximately one second while work exists.
|
||||
- p99 from committed priced receipt to durable threshold-stop intent at most
|
||||
five seconds under the qualified workload. Measure provider reporting delay
|
||||
and cancellation latency separately.
|
||||
- For applicable spend limits, stop admitting additional paid work when the
|
||||
oldest unevaluated accounting change exceeds fifteen seconds. This is a
|
||||
proposed protection against a failed evaluator, not a hard spending cap.
|
||||
An idle company's old evaluation timestamp alone must never trigger it.
|
||||
- With no dirty changes, unresolved retries, or actual budget deadlines, the
|
||||
evaluator performs no recurring database scans.
|
||||
|
||||
Best effort needs a stated overspend model. As an illustration, 10,000 runs each
|
||||
spending $0.001/second consume $10/second; five seconds of extra detection delay
|
||||
adds about $50, before unreported and uncancellable in-flight work. Those are
|
||||
hypothetical rates, not a measured cost estimate or maximum.
|
||||
|
||||
In general, additional spend depends on fleet burn rate multiplied by reporting,
|
||||
evaluation, and stop delay, plus already-committed work. Without bounds on those
|
||||
quantities, do not advertise a bounded dollar overshoot. A strict end-to-end cap
|
||||
would require provider-enforced limits or preauthorized bounded work; distributed
|
||||
allocations alone cannot cap a provider that can exceed its allocation.
|
||||
|
||||
## Implementation stages
|
||||
|
||||
### 1. Attribute the cost and eliminate unnecessary admission transactions
|
||||
|
||||
Owners: accounting transaction/tracing services, budget admission API, and its
|
||||
queue/dispatch/lifecycle callers.
|
||||
|
||||
Add a closed operation-kind label to accounting spans so receipt, finalization,
|
||||
preflight, reservation, policy edit, correction, and lifecycle work can be
|
||||
distinguished. Capture acquisition-query time, time from acquisition return to
|
||||
transaction completion, query count, and replay/outcome categories. Keep the
|
||||
operator-gated Observability contract and content-free attributes; this is not
|
||||
first-party Telemetry. Label the client-observed held interval honestly because
|
||||
it includes commit/return latency. Use sampled PostgreSQL wait/blocker evidence
|
||||
to establish actual contention, with SQL values and credentials excluded.
|
||||
|
||||
Separate read-only preliminary eligibility from mutating reconciliation.
|
||||
Preflights should not take a company writer lock just to discover no applicable
|
||||
policy. Keep the existing authoritative reservation and policy check at final
|
||||
dispatch in this stage. Budget changes, rollover, settlement and explicit
|
||||
reconciliation continue to own their necessary effects. Do not introduce a
|
||||
stale cached "no budgets" result that can bypass a newly configured limit.
|
||||
|
||||
Measure identical receipt replay and unchanged snapshots before prioritizing
|
||||
their optimization. Any early replay return must prove the immutable receipt
|
||||
and relevant projection state agree. Native event identity and lifecycle events
|
||||
cannot be discarded just because their monetary amount is unchanged.
|
||||
|
||||
Deliver a real-PostgreSQL benchmark using the service boundaries below, and the
|
||||
small behavior-preserving optimization justified by the caller measurements.
|
||||
If this removes today's significant delay, ship it independently; later stages
|
||||
remain a deliberate scale/product decision.
|
||||
|
||||
### 2. Persist receipts per run and evaluate budgets asynchronously
|
||||
|
||||
Owners: usage receipts, native usage delivery, budget exposure, budgets, and the
|
||||
existing accounting/enforcement delivery lifecycle.
|
||||
|
||||
In one run-scoped transaction:
|
||||
|
||||
1. Validate current source/ownership and lock the run.
|
||||
2. Insert or verify the receipt with its unique identity and fingerprint.
|
||||
3. Replace the run's accepted cumulative usage/exposure projection.
|
||||
4. Persist a versioned pending-evaluation marker and register the normal
|
||||
transaction-aware wake.
|
||||
5. Commit before acknowledging native delivery or removing legacy spool evidence.
|
||||
|
||||
Do not take the company accounting writer lock or evaluate all affected policies
|
||||
inside this receipt transaction. Keep final and partial receipts durable; batch
|
||||
derived budget evaluation rather than dropping intermediate accounting evidence.
|
||||
|
||||
Use the shared delivery coordinator for coalescing, retry and deadlines. Pending
|
||||
state must be scoped to the changed run/operation, or an append-only change;
|
||||
updating one company dirty counter for every receipt would recreate the lock.
|
||||
Consumption must compare revisions or claim durable work so a concurrent change
|
||||
cannot be cleared by an older evaluator. A timestamp/sequence high-water mark
|
||||
alone cannot safely assume database transactions commit in allocation order.
|
||||
|
||||
Read ledger and live contributions in one consistent database snapshot, or use
|
||||
transactionally consistent replacement contributions. The present four separate
|
||||
read-committed aggregates are not safe to reuse unchanged once other operations
|
||||
in the same company can settle concurrently: live-to-ledger
|
||||
replacement must not be missed or counted twice across queries.
|
||||
|
||||
Evaluate each affected policy once per batch, revalidate its revision before
|
||||
committing block/resume state, and retain durable cancellation generations.
|
||||
Updates arriving during evaluation remain pending. Recovery reconstructs pending
|
||||
work after restart; rollback must not publish a false wake or acknowledgement.
|
||||
Use short policy-state transactions where necessary. Preserve manual holds and
|
||||
committed stop obligations during policy changes and price corrections.
|
||||
|
||||
This is the first stage that permits an additional detection/admission race
|
||||
window. Update the product contract together with implementation; removing only
|
||||
the wrapper is not a valid intermediate change.
|
||||
|
||||
### 3. Remove remaining company writes from frequent settlement
|
||||
|
||||
Owners: run-cost accounting, costs, billing reconciliation/import, accounting
|
||||
integrity, paid-service invocation owners, and DB/shared contracts.
|
||||
|
||||
Settle ledger entries, reservation state, final receipt acknowledgement, and
|
||||
run-specific accounting together under the operation's lock. Use database
|
||||
uniqueness plus conflict/hash verification for concurrent idempotency; a
|
||||
read-before-insert check previously protected by the company lock is insufficient.
|
||||
Retain narrow agent coordination for shared runtime totals where needed.
|
||||
|
||||
Move company monthly display-counter updates to versioned, coalesced projection
|
||||
work. Update integrity inspection/repair and all counter consumers to understand
|
||||
projection freshness and checkpoint identity. Reporting from ledger snapshots
|
||||
remains available; intentional projection lag must not look like lost money.
|
||||
Handle backdated events, credits/corrections, UTC rollover and lifetime budgets.
|
||||
|
||||
For receipt-file recovery, stage immutable candidate envelopes outside the broad
|
||||
transaction, then validate source/version and closure under the operation lock.
|
||||
Retain disk evidence until the final owning transaction commits. A concurrent
|
||||
late receipt or replaced source must still prevent premature acknowledgement.
|
||||
|
||||
Apply the same boundary to browser runs, decision-model invocations, fast
|
||||
responses, prompt-injection classification, and externally reported charges.
|
||||
Inventory administrative users of `withAccountingTransaction` separately;
|
||||
agent deletion, repair, and lifecycle safety must not depend on a lock removed
|
||||
from receipt writers. Specify and test the new lock order, including issue/run
|
||||
ownership and foreign-key interactions. Keep supported provider spending caps.
|
||||
|
||||
Start with bounded batches and indexed aggregates. If history growth or batch
|
||||
time misses the target, maintain incremental scope/window contributions. If
|
||||
those counters become the next measured hot row, partition their writes and
|
||||
sum a bounded set of partitions. Do not synchronously increment a single new
|
||||
company total on every receipt and call the bottleneck solved.
|
||||
|
||||
### 4. Qualify workload and roll out
|
||||
|
||||
Extend the existing benchmark rather than using idle-agent count as scale proof.
|
||||
Use real PostgreSQL with at least two independent controller processes and
|
||||
explicitly recorded connection-pool limits. Simulated provider events should
|
||||
exercise real receipt ingestion and settlement, followed by a smaller real
|
||||
provider qualification. Keep load tests off the active staging instance.
|
||||
|
||||
| Dimension | Required cases |
|
||||
| --- | --- |
|
||||
| Distribution | 100, 1,000, then 10,000 active runs; one company and many companies |
|
||||
| Receipt cadence | One update per run per 60 seconds, 10 seconds and 1 second; record actual target-provider cadence before choosing the release load |
|
||||
| Policies | None; company; agent/project only; overlapping scopes and windows |
|
||||
| History | Increasing ledger history independently of active-run count |
|
||||
| Event shape | Cost-changing, unchanged, subscription, duplicate, stale, conflicting and out-of-order reports; reconnect bursts |
|
||||
| Races/failures | Simultaneous settlement and receipt, worker crash, failed/ambiguous commit, source replacement, policy change, manual Stop, late final charge, unknown price and UTC rollover |
|
||||
|
||||
At 10,000 runs the illustrative receipt cadences imply about 167, 1,000, and
|
||||
10,000 receipts/second before admission or settlement. This defines a workload,
|
||||
not a claim that current or proposed code supports it. A shared critical section
|
||||
held for an average of H seconds has an ideal ceiling near 1/H operations/second
|
||||
for that company; do not estimate H from the unrelated percentile summaries.
|
||||
|
||||
Acceptance requires durable receipt/ledger equality after quiescence, no duplicate
|
||||
effect or acknowledgement loss, stable pending-work depth, correct policy/manual
|
||||
hold behavior, and measured detection/cancellation/freshness targets. Record
|
||||
p50/p95/p99 ingestion, admission and settlement latency, actual PostgreSQL wait
|
||||
events, query count/rows/bytes, CPU/event-loop delay, and reconnect recovery time.
|
||||
Prove unrelated companies remain responsive and writers for unrelated runs no
|
||||
longer contend on a shared company update in the frequent path.
|
||||
|
||||
Initially compute the proposed evaluation in shadow mode without stop/resume
|
||||
effects and compare it against the authoritative ledger and existing decisions.
|
||||
Then canary the new enforcement contract on an explicitly selected environment,
|
||||
with its versioned pending work rebuilt and validated before switching. Do not
|
||||
run old and new enforcement authorities simultaneously for the same company.
|
||||
Rollback must drain/reconcile accepted receipts and rebuild authoritative state
|
||||
before restoring the older path; flipping a flag alone is insufficient.
|
||||
|
||||
Relevant existing coverage includes scoped budget enforcement, live attempts,
|
||||
native usage delivery, run-cost accounting, budget cancellation, worker recovery,
|
||||
and idle-accounting work. Add focused concurrency/failure tests for the changed
|
||||
boundaries; run the repository's full checks before a PR-ready implementation
|
||||
handoff. Browser qualification is needed if dispatch/Stop or budget UI semantics
|
||||
change, not to validate this plan file.
|
||||
|
||||
## Contract and delivery boundaries
|
||||
|
||||
Before enabling the new behavior, update `doc/SPEC.md`,
|
||||
`doc/SPEC-implementation.md` (especially scheduler and budget enforcement), the
|
||||
applicable contributor invariants, shared budget contracts, and user-facing
|
||||
budget explanations. The UI should say what auto-stop does and disclose possible
|
||||
in-flight overspend; it should not expose internal partition mechanics. Decide
|
||||
how configured existing reservations retain their estimate/visibility role
|
||||
without promising atomic aggregate admission in the new mode.
|
||||
|
||||
Distributed bounded spending allocations are deferred. Revisit them only if
|
||||
customers need stricter aggregate admission or qualified batch evaluation cannot
|
||||
meet latency/overspend needs. They require provider cost bounds as well as
|
||||
replenishment, revocation and policy-version coordination to become a monetary
|
||||
guarantee. They are not required for the recommended first implementation.
|
||||
|
||||
Investigation validation: inspected the source and existing test/benchmark
|
||||
contracts, queried live spans and two individual traces, and checked the relevant
|
||||
deployed-revision source diff. No service, configuration, database, or deployed
|
||||
code was changed. Runtime tests and load tests were not run for this planning
|
||||
change; dependency installation and implementation belong to the next stage.
|
||||
@@ -0,0 +1,47 @@
|
||||
# Preserve existing workspaces during ephemeral enrollment
|
||||
|
||||
The deployment endpoint remains a volume-free Cloud tenant, followed by a real
|
||||
rolling deployment while an agent works and new work is admitted. Enrollment must
|
||||
preserve existing task files and historical recovery sources. It must not reset
|
||||
task workspaces, manufacture successful copyback, or weaken migration guards.
|
||||
|
||||
## Evidence and remaining work
|
||||
|
||||
| Required outcome | Current evidence | Remaining gap | Owner / next action |
|
||||
| --- | --- | --- | --- |
|
||||
| Existing active task retains files | The current import encrypts, uploads, independently restores, compares the source again and publishes under the lifecycle lock. A live completed task imported successfully with its original directory retained. | The importer only accepts an active workspace with the current task binding. | Workspace migration owner: support historical records without changing the task binding or lifecycle state. |
|
||||
| Archived task retains its saved files | Retention records include encrypted task-directory restore points, Git restore points and verified Git reconstruction recipes. | The importer rejects missing local folders even when an existing restore point can reconstruct them. | Restore-point owners: reconstruct into migration scratch space, reject expired/missing/corrupt sources, and feed the verified result into the normal import publication. |
|
||||
| Shared project roots preserve all task files | Legacy roots contain nested task worktrees and multiple records can reference one physical folder. | A naive root capture recursively imports those other workspaces; the current per-record writer check does not fence aliases. | Workspace migration owner: inventory physical overlaps, fence all writers, and preserve nested workspace coverage before excluding any directory. |
|
||||
| Failed runs retain unsaved files | Provider inspection distinguishes surviving archived sandboxes from missing allocations. Existing recovery validation accepts some sources and rejects others for missing provenance or ambiguous allocation ownership. | Recovery must settle or durably preserve each exact source. A successful old preservation action does not necessarily update the legacy sync descriptor. | Workspace recovery owner: verify existing preservation receipts, recover eligible sources, and retain explicit unresolved obligations for unavailable/ambiguous sources. Do not replay agent work. |
|
||||
| Tenant is eligible without its volume | External DB and S3 are configured; the original volume remains attached. | Workspaces, local run logs and remaining owner-maintained data must pass the packaged maintenance receipt. | Cloud enrollment owner, after workspace migration is complete. |
|
||||
| Ordinary upgrades remain available | The no-drain Cloud deployment path is merged and qualified on another enrolled tenant. | Exact final app image qualification and a live deployment on this tenant. | Cloud deployment owner: qualify, enroll, then prove reads, submissions and active-run adoption through an actual replacement. |
|
||||
|
||||
## Acceptance boundaries
|
||||
|
||||
The first implementation slice supports archived and historical records,
|
||||
encrypted task-directory restoration, Git restoration recipes, and legacy
|
||||
branch-deletion receipts. PostgreSQL and real-file tests prove cold restoration
|
||||
of saved bytes, index state, and unpushed commits, including after pruning the
|
||||
old Git object store. Invalid or unavailable retained data still blocks import.
|
||||
The next slice adds leaf-first offline migration of overlapping folders, verified
|
||||
nested snapshot dependencies, interruption recovery, and dependency retention.
|
||||
Recovery-obligation reconciliation remains a separate gap; these slices do not
|
||||
yet make the whole tenant eligible for enrollment.
|
||||
|
||||
- Import existing files or a verified retained source. A missing folder alone is
|
||||
not permission to publish an empty workspace or choose a newer Git branch.
|
||||
- Preserve archived status, task bindings, task history, unpushed Git history,
|
||||
staged and unstaged edits, and saved recovery copies.
|
||||
- Preserve original files and the detached original volume until verified
|
||||
migration receipts and the retention interval permit removal.
|
||||
- Parent roots may omit a nested workspace only with durable, inspectable
|
||||
coverage of that exact nested directory. Arbitrary exclusion globs are not an
|
||||
acceptable data migration.
|
||||
- Resume interrupted imports with their original identity; cancellation may
|
||||
release an empty reservation only after the producer is stopped and no upload
|
||||
was journaled. Preserve an audit receipt.
|
||||
- Verify restored file bytes independently, including after making the old
|
||||
source unavailable in an isolated test. Exercise current and historical
|
||||
bindings, archived sources, overlap/writer races, and unavailable sources.
|
||||
- Unrelated tenants, production services, and agent task execution remain outside
|
||||
the migration's mutation scope.
|
||||
@@ -0,0 +1,432 @@
|
||||
# Lock-free accounting and best-effort budgets
|
||||
|
||||
Date: 2026-11-03
|
||||
Status: Target architecture proposed; preliminary admission optimization implemented locally
|
||||
Scope: Accounting ingestion, settlement, budget projections and budget admission
|
||||
Evidence: [Accounting contention investigation](2026-11-03-accounting-contention-and-budget-enforcement.md)
|
||||
|
||||
## Decision
|
||||
|
||||
Make the frequent accounting write path append-only. A receipt writes its own
|
||||
immutable fact and commits. It does not lock or update the company, agent, run,
|
||||
budget, aggregate counter, or worker checkpoint. Independent runs use independent
|
||||
keys, including when they belong to the same company or agent.
|
||||
|
||||
Asynchronous workers derive operation state, ledger entries and partitioned spend
|
||||
totals. Budget admission reads a recent view without making a synchronized debit
|
||||
against a company balance. Threshold detection and cancellation may lag; durable
|
||||
evidence, deterministic accounting and execution authorization remain required.
|
||||
|
||||
"Lock-free" here means no application mutex, advisory lock, or `SELECT FOR
|
||||
UPDATE` in the receipt or budget-check path, and no shared accounting row written
|
||||
for each receipt. It is not a formal nonblocking progress guarantee: PostgreSQL
|
||||
still uses internal locks, unique-index arbitration, foreign-key checks and
|
||||
transaction commit. In particular, two inserts with the same unique key can
|
||||
wait for each other. See [PostgreSQL INSERT](https://www.postgresql.org/docs/current/sql-insert.html).
|
||||
|
||||
Background batches and infrequent policy changes use optimistic conditional
|
||||
writes. Those also take database row locks internally. Moving a compare-and-swap
|
||||
onto the same company counter for every receipt would fail this design.
|
||||
|
||||
Existing issue ownership, provider-dispatch fencing, manual Stop and resource
|
||||
ownership are outside this lock-removal scope. Accounting cannot replace those
|
||||
contracts with an eventually consistent spend view.
|
||||
|
||||
## Problem fit and first milestone
|
||||
|
||||
This design removes the identified company-wide accounting serialization point
|
||||
if receipt ingestion, admission, settlement and shared counters all migrate.
|
||||
It does not establish that accounting is the current main-loop CPU culprit or
|
||||
that the resulting system supports 10,000 active agents. Those remain separate
|
||||
claims requiring measurements.
|
||||
|
||||
The matched investigation sample contained 8,280 zero-policy exposure evaluations
|
||||
out of 8,292. Those exposure evaluations returned almost immediately; the
|
||||
surrounding admission path still read state and policies inside the accounting
|
||||
transaction. This makes separating eligibility reads from reconciliation the
|
||||
most directly supported first change. It does not show that large exposure
|
||||
scans or a high receipt rate caused the sampled delays.
|
||||
|
||||
The 9,421 accounting transactions in that hour average about 2.6 per second
|
||||
across the observed host. Burst rate, company distribution and actual lock hold
|
||||
time are missing. The acquisition spans include database/network time and Node
|
||||
scheduling, so their long tails alone do not prove PostgreSQL row-lock saturation.
|
||||
|
||||
| Problem | What this design changes | Remaining condition |
|
||||
| --- | --- | --- |
|
||||
| Eligibility checks enter a mutating company transaction | Read-only admission, separate reconciliation | Classify callers and preserve manual holds, resume behavior and execution fencing |
|
||||
| Independent receipts serialize by company | Independent immutable inserts | Qualify actual receipt rate, index/FK waits and durable ingestion capacity |
|
||||
| Repeated exposure work and shared counters limit growth | Coalesced evaluation and partitioned projections | Show less work per receipt and sufficient projection throughput |
|
||||
| Main-loop CPU/event-loop contention | Some synchronous work leaves the request path | Same-process reducers still consume CPU; batching must reduce work or processing must move to another runtime |
|
||||
|
||||
The immutable identity model, closure changes and partition publication protocol
|
||||
are target-architecture work, not prerequisites for the first latency improvement.
|
||||
An asynchronous pipeline also adds backlog and recovery obligations: its stale
|
||||
decision gate can become an admission outage if reducers cannot keep up. Merely
|
||||
moving identical work behind an ACK would conceal latency without solving load.
|
||||
|
||||
Start with an independently shippable milestone: attribute admission calls and
|
||||
transaction work, provide a pure eligibility read (including a no-policy fast
|
||||
path), and move required reconciliation to its explicit owner. Retain the current
|
||||
final admission/reservation authority during this initial behavior-preserving
|
||||
step. Having no configured policy must not bypass manual company/project pauses,
|
||||
existing holds, unknown-accounting restrictions or duplicate-dispatch protection.
|
||||
|
||||
Compare end-to-end dispatch latency, accounting transactions and queries per
|
||||
dispatch, database waits, CPU and event-loop delay under the same workload.
|
||||
Then qualify receipt coalescing and the remaining single-company bottleneck.
|
||||
Use the full migration below when that evidence or the agreed capacity target
|
||||
justifies it. The accounting rewrite does not cover the other independently
|
||||
identified work in run-event persistence, recovery, snapshots or fanout.
|
||||
|
||||
## Architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
P["Runner / paid service"] --> I["Validate and append receipt"]
|
||||
I --> J[("Immutable receipt journal")]
|
||||
J --> A["ACK after durable commit"]
|
||||
J --> R["Independent partition reducers"]
|
||||
R --> L[("Ledger + operation views")]
|
||||
R --> T[("Spend totals by scope / window / partition")]
|
||||
T --> B["Coalesced budget evaluator"]
|
||||
B --> V[("Versioned budget decisions")]
|
||||
V --> D["Dispatch: read decision, record own admission"]
|
||||
B --> S[("Durable stop intent")]
|
||||
S --> C["Existing scoped cancellation path"]
|
||||
```
|
||||
|
||||
The journal, reducers and evaluator can start in the existing server and
|
||||
PostgreSQL deployment. Independent scaling is a module/runtime boundary, not a
|
||||
requirement for Kafka, Redis, another service, or a separate database today.
|
||||
|
||||
## 1. Give every paid operation a stable identity
|
||||
|
||||
Use one accounting operation for each heartbeat run, browser invocation,
|
||||
decision-model invocation, fast response, classifier invocation, or separately
|
||||
reported charge. Record its company and attribution before paid dispatch.
|
||||
Retries of the same external operation reuse its identity; a genuinely new paid
|
||||
attempt has a distinct attempt identity beneath that operation.
|
||||
|
||||
An accounting operation's immutable identity is separate from mutable run state.
|
||||
Receipts reference this identity, rather than acquiring foreign-key protection
|
||||
on a busy mutable `heartbeat_runs` row. The existing journal directly references
|
||||
that row, so removing the explicit lock alone does not eliminate this possible
|
||||
wait. Use a company-scoped composite reference to the immutable identity; keep
|
||||
company-boundary validation and the existing deletion/retention obligations.
|
||||
Separate any nullable live-run pointer from retained original attribution.
|
||||
|
||||
Admitted accounting sources have durable identities and provenance. Controller
|
||||
reconnection does not manufacture a new paid attempt. Source replacement must
|
||||
retain the existing verified handoff/baseline semantics. A higher timestamp,
|
||||
random UUID, or newly generated controller source ID is not proof of authority
|
||||
over earlier billing evidence.
|
||||
|
||||
Source handoff remains an infrequent execution-authority operation. Ordinary
|
||||
checkpoints do not mutate its ownership record. Existing authentication and
|
||||
company access checks still apply. Authority to report a closing receipt after
|
||||
Stop is accounting-only; it never grants authority to continue execution.
|
||||
|
||||
## 2. Receipt ingestion only appends evidence
|
||||
|
||||
Extend/generalize the existing `run_usage_receipts` contract. The conceptual
|
||||
receipt contains:
|
||||
|
||||
| Field | Purpose |
|
||||
| --- | --- |
|
||||
| Company and operation IDs | Tenant boundary and accounting unit |
|
||||
| Attempt and admitted source identity | Separate paid attempts from reconnect/replay |
|
||||
| Stable source event ID and source sequence | Replay identity and ordering within that source |
|
||||
| Receipt kind and normalization version | Partial usage, final usage, source handoff, closure, correction |
|
||||
| Cumulative usage, price and provenance | Evidence; unknown price stays unknown |
|
||||
| Payload fingerprint | Detect conflicting reuse of an identity |
|
||||
| Server-assigned ingestion time and deterministic partition | Recovery, pending-work age and routing |
|
||||
|
||||
Provider event time remains separate. Freshness protection never trusts a caller's
|
||||
timestamp. External charge callers need a stable request identity to obtain
|
||||
idempotent retries; matching dollar amounts or payload similarity cannot prove
|
||||
that two otherwise unidentified charges are the same purchase.
|
||||
|
||||
Within a small transaction, validate the admitted identity and input, register
|
||||
the existing transaction-aware work signal, and insert the immutable receipt.
|
||||
Enforce the logical identity with a unique constraint. On conflict, fetch the
|
||||
committed existing receipt and compare its fingerprint; equal means replay,
|
||||
different means a conflict requiring reconciliation. Do not use `DO UPDATE`
|
||||
to replace receipt evidence. At read committed isolation, a conflict check may
|
||||
need a subsequent statement to see the winning committed row.
|
||||
|
||||
Commit before ACK. An ACK means the evidence is durable, not that the spend view
|
||||
has already incorporated it. Preserve legacy durable spooling and native
|
||||
redelivery when the database write or its acknowledgement is uncertain.
|
||||
|
||||
There is no read-modify-write of `usageJson`, `spentMonthlyCents`, reservation
|
||||
capacity, or a company-wide dirty flag here. The new receipt is itself durable
|
||||
pending work. Mutable delivery metadata may be attached to it by the consumer;
|
||||
its evidence fields are immutable.
|
||||
|
||||
Batch inserts with count/byte bounds when useful. Keep lifecycle, final, unknown
|
||||
and conflicting evidence intact. Do not drop a receipt solely because its dollar
|
||||
amount matches the previous one: it may change completeness or provenance.
|
||||
|
||||
## 3. Reduce independent partitions, then publish batches atomically
|
||||
|
||||
Route by a stable hash of **operation ID**, not company ID. For example, a
|
||||
starting benchmark can use 64 virtual partitions per company. That number is a
|
||||
test parameter, not a capacity claim. One company with 10,000 active runs spreads
|
||||
over all partitions; one agent with many runs spreads too.
|
||||
|
||||
Persist the layout version and partition with the operation's admission. All its
|
||||
receipts, closure and corrections follow that assignment. Changing partition
|
||||
count requires an explicit migration; do not silently rehash active operations.
|
||||
|
||||
The minimal conceptual storage is:
|
||||
|
||||
| Record | Write frequency and ownership |
|
||||
| --- | --- |
|
||||
| Operation identity | Once per admitted operation; immutable attribution |
|
||||
| Receipt journal | Append per receipt; unique identity; durable pending metadata |
|
||||
| Operation view | Replaced by its partition reducer, at most once per operation per batch |
|
||||
| Scope/window/partition totals | Updated in a bounded batch, never by receipt ingestion |
|
||||
| Partition publication head | One conditional generation advance per batch |
|
||||
| Budget policy and decision | Policy edits and coalesced evaluations; independent of receipt acceptance |
|
||||
| Ledger, adjustments, incidents, stop intent | Existing domain records, adapted to operation identities and idempotent writes |
|
||||
|
||||
Some records can extend existing tables. Separate immutable identities from
|
||||
mutable projections; do not add a table for each timer or delivery stage.
|
||||
|
||||
For a reducer batch:
|
||||
|
||||
1. In one repeatable-read snapshot, capture partition generation `g`, a bounded
|
||||
set of unprocessed receipt IDs, and their prior operation views.
|
||||
2. Reduce those operations and compute replacement contributions outside the
|
||||
publishing transaction. No provider calls or filesystem work occurs during
|
||||
publication.
|
||||
3. In one short transaction, conditionally advance the partition head from `g`
|
||||
to `g + 1`, bulk-write operation views and aggregate deltas, insert any
|
||||
qualified ledger settlement, and mark exactly the consumed receipt IDs.
|
||||
4. If the conditional advance loses, roll back and retry from a fresh snapshot.
|
||||
Use bounded retries with scheduler backoff under contention.
|
||||
|
||||
All writers of those views, including repairs, follow this publication protocol.
|
||||
A normally assigned reducer avoids duplicate work; overlapping workers or an
|
||||
old worker after failover remain safe because publication requires the captured
|
||||
generation. No durable exclusive lease is needed for correctness of this batch.
|
||||
|
||||
An input arriving during computation remains pending. Never advance a global
|
||||
`last_seen_id` or timestamp past all smaller values: sequence allocation and
|
||||
transaction commit order differ. An earlier allocated receipt that commits
|
||||
later must still be discoverable. Retain an indexed pending predicate.
|
||||
|
||||
This has bounded coordination per partition **per batch**. It cannot stall
|
||||
receipt ACKs behind the calculation, and different partitions do not write the
|
||||
same aggregate rows. Shared company/agent/project totals are sums of these
|
||||
partition contributions, not counters incremented by every producer.
|
||||
|
||||
## 4. Define canonical money without depending on arrival order
|
||||
|
||||
The reducer preserves current normalization and source-ownership rules as a
|
||||
versioned pure function over admitted evidence. Within a source, order by the
|
||||
producer sequence, not database arrival time. Across sources, follow verified
|
||||
handoff/continuation provenance rather than arbitrarily choosing a winner.
|
||||
|
||||
Cumulative reports replace earlier reports for the same accounting component.
|
||||
Reports of $2, $3 and $5 represent $5, not $10. Distinct paid attempts contribute
|
||||
separately; session-cumulative counters require their established baseline.
|
||||
An incomplete/unknown report cannot erase known partial exposure. Final prices
|
||||
and audited corrections may legitimately reduce earlier estimates.
|
||||
|
||||
Each operation view owns mutually exclusive contribution categories:
|
||||
|
||||
```text
|
||||
unsettled: ledger = 0; live = known observed amount; hold = max(estimate, live)
|
||||
settled: ledger = final; live = 0; hold = 0
|
||||
```
|
||||
|
||||
This is a contribution model, not a declaration that unknown exposure is zero.
|
||||
Carry unknown-price, pending-terminal and unresolved-dispatch counts separately
|
||||
and apply their configured restrictions. Hosted browser work remains a separate
|
||||
paid operation from its parent run.
|
||||
|
||||
Settlement and that operation's replacement contribution publish atomically.
|
||||
Company/agent/project views sum one contribution per operation. A reader sees
|
||||
the old or new contribution, never both the live estimate and its final charge.
|
||||
UTC-month and lifetime attribution, unsettled carryover, credits, backdated
|
||||
charges and corrections need explicit reducer rules matching the selected
|
||||
product contract; do not assign costs to the month in which a worker ran.
|
||||
|
||||
### Finalization requires closure, not merely a "complete" checkpoint
|
||||
|
||||
The governing execution coordinator supplies immutable closure evidence only
|
||||
after the operation cannot start another ordinary paid attempt. It references
|
||||
the admitted attempts/sources and the final usage evidence or unresolved debt.
|
||||
An adapter's `complete` usage flag alone does not establish that boundary.
|
||||
|
||||
A closure record must prove the final cumulative coverage it names. The current
|
||||
native `sourceSeq` includes non-accounting events, so gaps between usage receipts
|
||||
are not automatically lost usage. Use explicit final references/coverage, or a
|
||||
dedicated accounting sequence if the producer contract is extended. Do not
|
||||
invent a contiguous usage sequence from today's sparse source sequence.
|
||||
|
||||
Publish a settlement certificate and its ledger components in one transaction,
|
||||
with one base settlement per `(company_id, operation_id)` and a verified content
|
||||
fingerprint. A different closure for an already-settled operation is a conflict
|
||||
or an explicit correction, not a second base settlement.
|
||||
Only the transaction that inserts that certificate inserts its components; a
|
||||
replay verifies the existing certificate. Ledger IDs and component keys are
|
||||
stable across retries. Unknown external outcomes remain unresolved.
|
||||
|
||||
Late evidence within the declared closure coverage is replay/recovery. Evidence
|
||||
beyond it remains durable and creates a reconciliation obligation or an explicit
|
||||
correction; it cannot silently reopen the same final charge or disappear because
|
||||
the run was marked terminal. Corrections use immutable adjustment identities.
|
||||
Existing mutable correction/reporting APIs need an adapter to this ledger model.
|
||||
|
||||
## 5. Budget checks read; they do not reserve a shared balance
|
||||
|
||||
For dispatch, read current company/agent/project controls and applicable budget
|
||||
decisions in one bounded read from the authoritative database, rather than an
|
||||
unbounded-lag replica or cache. A policy without a current decision must
|
||||
be evaluated or treated as pending; it is not implicitly unlimited. The decision
|
||||
records policy revision, budget window, projection generations, status and
|
||||
freshness evidence.
|
||||
|
||||
If applicable decisions allow work, record this operation's own admission with
|
||||
the decision versions and estimated cost. Preserve duplicate-dispatch fencing
|
||||
in the execution subsystem. An admission receipt is not proof that an external
|
||||
provider call occurred exactly once; existing unknown-outcome recovery remains.
|
||||
|
||||
There is no atomic decrement of company remaining budget. Estimate-fit checks
|
||||
can use the same snapshot, and admitted estimates can enter the asynchronous
|
||||
exposure view. Two dispatches can see the same remaining money and both start.
|
||||
That race is accepted by this best-effort contract. Reservations become recorded
|
||||
estimates/obligations, not exclusive claims on a shared spending allowance.
|
||||
|
||||
New snapshots after a committed budget block deny additional work. Admissions
|
||||
already in flight can still race that block. Manual Stop, pause, permissions,
|
||||
issue ownership and dispatch authority retain their existing final gates.
|
||||
Keep provider-enforced cost caps where supported.
|
||||
|
||||
## 6. Coalesce budget decisions and preserve stop authority
|
||||
|
||||
The evaluator reads the applicable scope/window partition totals in one
|
||||
consistent snapshot, rather than scanning ledger and receipt history on each
|
||||
checkpoint. Capture the policy's configuration revision and prior decision
|
||||
revision in that same snapshot.
|
||||
|
||||
Publish with a conditional update against both captured revisions. A policy
|
||||
edit advances its configuration revision and invalidates the previous decision.
|
||||
An older evaluator cannot overwrite a newer decision or a changed policy. A
|
||||
losing evaluator recomputes; it never retries publication using a fresh version
|
||||
number attached to stale calculated totals.
|
||||
|
||||
Publish the decision, any incident/approval transition, and durable stop intent
|
||||
in the same transaction. Generate idempotency identities from the winning
|
||||
transition; losing or repeated evaluations produce no duplicate side effects.
|
||||
Only actual state transitions need notification/audit effects. Cancellation
|
||||
delivery remains at least once, scoped and recoverable. Already-committed
|
||||
per-run Stop obligations survive later policy changes. Resuming a budget hold
|
||||
cannot clear a manual hold.
|
||||
|
||||
These occasional policy-row updates can contend with one another. Producers
|
||||
and admission readers do not wait on that row to record receipts or debit money.
|
||||
Do not replace them with per-receipt policy updates.
|
||||
|
||||
## 7. Bound lag and recover work without an idle polling loop
|
||||
|
||||
Retain the proposed healthy-load targets from the investigation: roughly
|
||||
one-second coalescing and a five-second p99 target from durable priced receipt
|
||||
to stop intent. Provider reporting and shutdown delay are additional. Targets
|
||||
require a workload qualification; none is a guarantee on final billed dollars.
|
||||
|
||||
For scopes with spending limits, reject additional paid admissions if an
|
||||
unrepresented accounting change has been waiting over fifteen seconds. This
|
||||
check considers pending journal inputs **and** reduced contributions not yet
|
||||
covered by the decision's projection generations. Carry their oldest pending
|
||||
time through publication. Consuming the journal must not hide evaluator lag.
|
||||
An old decision with no subsequent relevant changes remains fresh while idle.
|
||||
Policy edits and UTC-window changes are explicit new evaluation obligations.
|
||||
|
||||
Durable pending rows are truth; notifications are hints. Reuse the existing
|
||||
delivery coordinator for local coalescing, finite retries, actual deadlines,
|
||||
drain and restart. No unconditional per-company timer scans empty queues.
|
||||
|
||||
The current DB work signals are process-local by default, with optional
|
||||
commit-bound PostgreSQL notifications. Separate reducers therefore require
|
||||
qualified cross-process wake delivery, subscribed before the startup scan and
|
||||
reconciled after reconnect. `NOTIFY` alone is not durable storage or a way to
|
||||
wake stopped compute. See the [existing signal implementation](../../packages/db/src/work-signals.ts)
|
||||
and [PostgreSQL notification behavior](https://www.postgresql.org/docs/current/sql-notify.html).
|
||||
Do not claim distributed recovery or host sleep until the actual notification,
|
||||
startup, disconnect and external-wake paths are qualified.
|
||||
|
||||
Receipt acceptance continues while budgets are blocked. Otherwise stopping spend
|
||||
would also stop the evidence needed to settle it.
|
||||
|
||||
## Failure cases the design must pass
|
||||
|
||||
| Event/race | Required outcome |
|
||||
| --- | --- |
|
||||
| Same receipt delivered twice | One logical receipt; equal fingerprint accepted as replay |
|
||||
| Same identity, different content | Conflict retained/reported; no silent overwrite or second charge |
|
||||
| Sequence 8 arrives before 7 | Correct cumulative state; 7 cannot rewind it |
|
||||
| Earlier allocated receipt commits after a batch | It remains discoverable as pending work |
|
||||
| Producer loses ACK after commit | Safe redelivery with the same identity |
|
||||
| Reducer dies before or during publication | All batch effects commit together or remain replayable |
|
||||
| Two reducers compute from generation 12 | One publishes generation 13; the other recomputes |
|
||||
| Final receipt races partial receipt | Closure and canonical reduction determine one contribution, independent of arrival order |
|
||||
| Final bill is lower than live estimate | Correct signed adjustment/replacement; no double counting |
|
||||
| Source changes during recovery | Verified handoff retains prior debt; no guessed new baseline |
|
||||
| Policy is lowered during evaluation | Old policy revision cannot publish a decision |
|
||||
| Manual Stop races budget resume | Manual execution hold remains authoritative |
|
||||
| Evaluator stops while receipts continue | Evidence accumulates durably; stale-view admission protection activates |
|
||||
| No new accounting work | Reducers and evaluator become quiet |
|
||||
|
||||
## Implementation and migration
|
||||
|
||||
These steps implement the full target after the first milestone above; they are
|
||||
not required as one change to improve today's eligibility path.
|
||||
|
||||
1. Add immutable operation identities and the journal acceptance boundary,
|
||||
initially alongside the existing authoritative path. Preserve existing native
|
||||
and legacy receipts; adapt source/closure evidence before removing their
|
||||
current completion fences.
|
||||
2. Implement deterministic operation reduction, idempotent settlement and
|
||||
partition publication. Run it in shadow mode with isolated projections and
|
||||
no production ledger, stop, or incident side effects. Compare at a common
|
||||
receipt/closure boundary after draining, not at arbitrary wall times.
|
||||
3. Implement versioned budget decisions, freshness, read-only admission and
|
||||
transition-based stop delivery. Replace shared monthly/runtime counters with
|
||||
partitioned projections and update all report, repair and integrity consumers.
|
||||
4. At a controlled cutover, fence the old settlement/evaluation authority, drain
|
||||
committed inputs, seed verified ledger/adjustment baselines, and select one
|
||||
authoritative writer path. Resume admission only with current decisions.
|
||||
Continue accepting/replaying durable receipts through this transition.
|
||||
5. Rollback similarly requires reconciliation and an authority handoff. Do not
|
||||
let old and new settlement writers charge the same operation independently.
|
||||
|
||||
Owning modules are usage receipts/native usage delivery, run-cost accounting,
|
||||
costs/reconciliation/integrity, budgets and the existing enforcement worker.
|
||||
Dispatch consumes a narrow decision contract; provider normalization remains
|
||||
with adapters. Update DB/shared/API/UI contracts and the budget guarantees in
|
||||
`doc/SPEC.md`, `doc/SPEC-implementation.md` and contributor guidance before enabling
|
||||
the change. The earlier per-run-lock design is not this target's hot path.
|
||||
|
||||
Qualification uses real PostgreSQL and multiple independent processes, both one
|
||||
large company and many companies, overlapping scope budgets, ledger-history
|
||||
growth, duplicate/reconnect bursts and the races above. Verify producer SQL has
|
||||
no shared counter/head/policy writes and no company/run `FOR UPDATE`. Audit
|
||||
foreign-key, trigger and uniqueness behavior too; string searches cannot prove
|
||||
absence of contention.
|
||||
|
||||
Measure sustained receipts and bytes per second, ACK/admission/settlement latency,
|
||||
projection backlog age, stop latency, database waits and event-loop load. Ten
|
||||
thousand agents at one receipt per second would create 864 million receipts per
|
||||
day before batching/retention effects. Eliminating serialization does not remove
|
||||
database, WAL, index, storage or provider-capacity limits. Start with bounded
|
||||
batches and measured retention; introduce a different log substrate only when
|
||||
its need is demonstrated.
|
||||
|
||||
Design validation in this chat: inspected the existing journal, budget incident,
|
||||
enforcement and work-signal contracts and checked PostgreSQL concurrency and
|
||||
notification semantics. This is an architecture document, not a prototype or
|
||||
load-test result. Implementation details and capacity remain to be qualified.
|
||||
@@ -19,6 +19,14 @@ same Runner and provider history through another healthy follow-up. This is a
|
||||
controlled cache fault, not a reconstruction of a historical incident.
|
||||
Each recovery case includes another healthy turn after the recovered turn and
|
||||
asserts that it does not start another runner or provider process.
|
||||
The `warm-suspended` case suspends the first turn, resumes the second turn,
|
||||
then keeps that controller warm for the third turn. It must retire the first
|
||||
turn's local suspension receipt so the third turn can reuse the resumed Runner
|
||||
and provider history without a second process or repeated turn.
|
||||
The `finalizer-wait` case gives the completed predecessor a live finalization
|
||||
lease. The next run must wait without starting a provider turn or consuming an
|
||||
attempt. After lease release, one durable resumption claim continues that same
|
||||
run and preserves provider history through the third turn.
|
||||
The same-server case also starts preparation lease renewal during recovery and
|
||||
holds instruction collection open past a renewal tick after coordinator release.
|
||||
The executor must acquire renewal before preparation releases its reference, then
|
||||
|
||||
@@ -19,6 +19,25 @@ approval activation, built-in provisioning and import preservation described
|
||||
below. The original baseline commit above still governs unrelated entries; this
|
||||
update does not claim operation-complete review or live agent verification.
|
||||
|
||||
Native Cursor Cloud uses a server-derived `provider_api` placement after its
|
||||
complete native profile is validated. Create/update with `runner: paperclip`
|
||||
stores `defaultEnvironmentId: null`; the caller cannot supply placement, commands,
|
||||
working directories or arbitrary environment values. Only the bound
|
||||
`CURSOR_API_KEY` is passed to the qualified API proxy. Test does not acquire a
|
||||
Computer lease. Finish setup uses the same validated placement and account check;
|
||||
it does not inherit a default Computer or its warm-task lifecycle. Incomplete
|
||||
held drafts stay inactive until the complete profile passes activation validation.
|
||||
Task/run authority, company boundaries, budgets and per-turn callback scope remain enforced. Local Computer execution stays denied under
|
||||
managed-sandbox-only policy. Verify the actual saved agent and live run; a board
|
||||
Test result does not prove agent task-tool access.
|
||||
|
||||
On managed-only instances with native Runner enabled, a server-held Cursor setup
|
||||
draft can resolve an automatic runner to the API controller before it has a
|
||||
model. Unsaved automatic Test requests use the same advertised default and still
|
||||
validate the full profile. Existing active Legacy agents and explicit Legacy
|
||||
choices do not convert automatically. A client-supplied setup flag does not grant
|
||||
draft status or bypass active model validation.
|
||||
|
||||
## `hire`
|
||||
|
||||
Capability ID: `agents/hire`.
|
||||
|
||||
@@ -168,6 +168,14 @@ API: `present` — representative operation IDs:
|
||||
status remains an in-memory read. [Route coverage](../../server/src/__tests__/instance-settings-routes.test.ts)
|
||||
and [PostgreSQL coverage](../../server/src/services/heartbeat/upgrade-readiness.test.ts)
|
||||
establish this local boundary; hosted active-run continuity requires separate proof.
|
||||
- `GET /api/instance/task-drain?idleSleepSafety=1` — the negotiated incoming-wake
|
||||
extension requires a verified stack/action/owner-scoped Cloud control capability;
|
||||
query parameters and tenant credentials cannot grant it. Its version-3 proof
|
||||
permits only saved API keys and exactly bound brokered GitHub bot configuration.
|
||||
Agent and board API requests authenticate under current tenant authority after
|
||||
bounded host wake. Personal GitHub continuity remains blocked until the stacked
|
||||
deadline proof is present. See [protocol scope](../../doc/idle-incoming-wake.md)
|
||||
and [local auth/PostgreSQL evidence](../../server/src/__tests__/idle-incoming-wake.test.ts).
|
||||
- `GET /api/ready` and `HEAD /api/ready` — public deployment readiness only;
|
||||
200 after recovery, 503 while starting, recovering, or draining. No SQL,
|
||||
session resolution, company identity, or control authority.
|
||||
|
||||
@@ -37,6 +37,22 @@ Save & exit keeps its configuration; Finish setup on the agent returns to the
|
||||
same record. Final submission requests approval when required, then follows the
|
||||
existing preparation and verification lifecycle.
|
||||
|
||||
For **Cursor Cloud**, choose its card in Cloud when native Runner is enabled and
|
||||
the qualified controller is available. Its held setup draft uses the API
|
||||
controller without a Computer, before model or credential selection. Local
|
||||
operators can choose **Paperclip Runner** in Advanced. Bind a Cursor Cloud
|
||||
API key, enter the exact model and select a repository/ref or a named Cursor
|
||||
cloud environment. Cursor hosts the agent's environment; this native setup has
|
||||
no Paperclip Computer selection. Save, reopen, and run Test. Test checks the
|
||||
account catalog and callback authentication gate without a prompt; the effective-model
|
||||
warning remains visible for this experimental integration. Finish setup repeats
|
||||
that account check and reaches Ready without preparing or leasing a Computer,
|
||||
even when the instance has a default plugin Computer. The Cursor lifecycle stays
|
||||
per-turn. Assign a real task
|
||||
and verify its output, question continuation, Plan approval and Stop separately.
|
||||
Managed-sandbox-only policy stays enabled. It continues to deny local agent code.
|
||||
Kubernetes-only execution is not supported by this API controller.
|
||||
|
||||
When the native Runner adapter is available, **Google Antigravity** appears in the ordinary
|
||||
adapter picker. Before runtime acceptance, its card says **Not available yet**.
|
||||
Selecting it opens Google account setup and model configuration without saving
|
||||
|
||||
@@ -154,3 +154,15 @@ Scope: cli
|
||||
Storybook: No authored story required for doctor/health command results.
|
||||
Production: Run doctor against an owned data directory; check readiness and nonzero failures. A listening port alone does not prove readiness; repair requires observing the repaired result.
|
||||
Sources: Source: [doctor.ts](../../cli/src/commands/doctor.ts). Tests: [doctor.test.ts](../../cli/src/__tests__/doctor.test.ts)
|
||||
|
||||
|
||||
Incoming API wake is an optional Cloud deployment capability. On an owned qualified
|
||||
stack, close the browser, retain valid saved credentials, let the stack sleep,
|
||||
and send an ordinary API request. Check the persisted result and current
|
||||
revocation/company authorization after wake, truthful retry failures, and return
|
||||
to idle after successful API activity ends. Brokered GitHub bot wake uses the
|
||||
existing durable connector queue and exact registration ownership. Direct
|
||||
webhooks, persistent transports and personal GitHub credential continuity retain
|
||||
their explicit dependencies. [Source protocol](../../doc/idle-incoming-wake.md)
|
||||
and [local fixtures](../../server/src/__tests__/idle-incoming-wake.test.ts) cover
|
||||
these boundaries; they do not establish deployed host/database suspension.
|
||||
@@ -325,6 +325,46 @@ describeEmbeddedPostgres("activity service", () => {
|
||||
expect(runs[0]).not.toHaveProperty("contextSnapshot");
|
||||
});
|
||||
|
||||
it.each(["requested", "acknowledged", "pending_native", "missing_native", "wrong_run", "wrong_company",
|
||||
"malformed_execution", "null_execution", "malformed_native"] as const)(
|
||||
"projects only bounded, owner-matched Stop evidence in issue history: %s", async proof => {
|
||||
const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID();
|
||||
await db.insert(companies).values({ id: companyId, name: "Stop history", issuePrefix: `S${companyId.slice(0, 6)}` });
|
||||
await db.insert(agents).values({ id: agentId, companyId, name: "Stopped runner", adapterType: "process" });
|
||||
await db.insert(issues).values({ id: issueId, companyId, title: "Stop the current run", assigneeAgentId: agentId });
|
||||
const native = { schema: "paperclip.native-cancellation.v1", runId, companyId,
|
||||
dispatchState: proof === "pending_native" ? "pending" : "acknowledged",
|
||||
intentAuditId: "private-stop-proof", effects: ["private-stop-proof"], secret: "private-stop-proof" };
|
||||
const result: Record<string, unknown> = {
|
||||
executionCancellation: { state: proof === "requested" ? "requested" : "acknowledged",
|
||||
proof: "private-stop-proof", secret: "private-stop-proof" },
|
||||
nativeCancellation: native, unrelated: "private-stop-proof",
|
||||
};
|
||||
if (proof === "missing_native") delete result.nativeCancellation;
|
||||
if (proof === "wrong_run") native.runId = randomUUID();
|
||||
if (proof === "wrong_company") native.companyId = randomUUID();
|
||||
if (proof === "malformed_native") result.nativeCancellation = { ...native, dispatchState: { secret: "private-stop-proof" } };
|
||||
if (proof === "malformed_execution") result.executionCancellation = { state: { secret: "private-stop-proof" } };
|
||||
if (proof === "null_execution") result.executionCancellation = null;
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, scopeKind: "issue",
|
||||
nativeIssueId: issueId, runtimeMode: "native", status: "cancelled", finishedAt: new Date(),
|
||||
contextSnapshot: { issueId }, livenessState: "advanced", resultJson: result });
|
||||
|
||||
const rows = await activityService(db).runsForIssue(companyId, issueId);
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({ runId, runtimeMode: "native", status: "cancelled" });
|
||||
expect(rows[0]!.resultJson).toEqual({
|
||||
executionCancellation: { state: proof === "requested" ? "requested"
|
||||
: proof === "malformed_execution" || proof === "null_execution" ? "unknown" : "acknowledged" },
|
||||
...(["missing_native", "wrong_run", "wrong_company", "malformed_native"].includes(proof) ? {} : {
|
||||
nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId, companyId,
|
||||
dispatchState: proof === "pending_native" ? "pending" : "acknowledged" },
|
||||
}),
|
||||
});
|
||||
expect(JSON.stringify(rows)).not.toContain("private-stop-proof");
|
||||
},
|
||||
);
|
||||
|
||||
it("backfills missing liveness for completed issue runs before returning the ledger", async () => {
|
||||
const companyId = randomUUID();
|
||||
const agentId = randomUUID();
|
||||
|
||||
@@ -101,13 +101,15 @@ function registerModuleMocks() {
|
||||
}));
|
||||
}
|
||||
|
||||
registerModuleMocks();
|
||||
const [{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
|
||||
const refreshableAdapterType = "refreshable_adapter_route_test";
|
||||
|
||||
async function createApp() {
|
||||
const [{ agentRoutes }, { errorHandler }] = await Promise.all([
|
||||
vi.importActual<typeof import("../routes/agents.js")>("../routes/agents.js"),
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
]);
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
@@ -159,11 +161,6 @@ async function unregisterTestAdapter(type: string) {
|
||||
|
||||
describe("adapter model refresh route", () => {
|
||||
beforeEach(async () => {
|
||||
vi.resetModules();
|
||||
vi.doUnmock("../routes/agents.js");
|
||||
vi.doUnmock("../routes/authz.js");
|
||||
vi.doUnmock("../middleware/index.js");
|
||||
registerModuleMocks();
|
||||
vi.clearAllMocks();
|
||||
mockCompanySkillService.listRuntimeSkillEntries.mockResolvedValue([]);
|
||||
mockCompanySkillService.resolveRequestedSkillKeys.mockResolvedValue([]);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import express from "express";
|
||||
import { unprocessable, forbidden } from "../errors.js";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { vi } from "vitest";
|
||||
@@ -259,6 +260,30 @@ describe("adapter routes", () => {
|
||||
expect(resolveRunnerTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps Cursor API discovery available without granting local support when the implicit managed Computer is unavailable", async () => {
|
||||
const unavailable = unprocessable("Managed Computer unavailable", { code: "managed_sandbox_unavailable" });
|
||||
const app = createApp({}, { resolveRunnerTarget: async () => { throw unavailable; } });
|
||||
const response = await request(app).get("/api/adapters?companyId=company-1");
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.find((adapter: any) => adapter.type === "cursor_cloud")).toMatchObject({ supportedRunners: ["paperclip"], defaultRunner: "paperclip" });
|
||||
expect(response.body.find((adapter: any) => adapter.type === "codex_local").supportedRunners).toEqual([]);
|
||||
const explicit = await request(app).get("/api/adapters?companyId=company-1&environmentId=selected");
|
||||
expect(explicit.status).toBe(422);
|
||||
const foreign = await request(createApp({}, { resolveRunnerTarget: async () => { throw forbidden("Foreign Computer", { code: "environment_company_mismatch" }); } })).get("/api/adapters?companyId=company-1");
|
||||
expect(foreign.status).toBe(403);
|
||||
});
|
||||
|
||||
it("does not offer Cursor execution under Kubernetes-only policy, including the unavailable managed Computer fallback", async () => {
|
||||
for (const resolveRunnerTarget of [
|
||||
async () => ({ driver: "sandbox", executionMode: "kubernetes" as const }),
|
||||
async () => { throw unprocessable("Managed Computer unavailable", { code: "managed_sandbox_unavailable", executionMode: "kubernetes" }); },
|
||||
]) {
|
||||
const response = await request(createApp({}, { resolveRunnerTarget })).get("/api/adapters?companyId=company-1");
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body.find((adapter: any) => adapter.type === "cursor_cloud").supportedRunners).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["environmentId=selected", "companyId=", "companyId=one&companyId=two", "companyId=one&environmentId="])("rejects malformed discovery scope before resolution (%s)", async query => {
|
||||
const resolveRunnerTarget = vi.fn();
|
||||
const response = await request(createApp({}, { resolveRunnerTarget })).get(`/api/adapters?${query}`);
|
||||
|
||||
@@ -68,7 +68,7 @@ const mockApprovalService = vi.hoisted(() => ({
|
||||
const mockInstanceSettingsService = vi.hoisted(() => ({
|
||||
get: vi.fn(async () => ({ defaultEnvironmentId: null })),
|
||||
getGeneral: vi.fn(async () => ({ censorUsernameInLogs: false })),
|
||||
getExperimental: vi.fn(async (): Promise<{ enableNativeRunner: boolean; enableOpenAiDot?: boolean }> => ({ enableNativeRunner: false })),
|
||||
getExperimental: vi.fn(async (): Promise<{ enableNativeRunner: boolean; enableOpenAiDot?: boolean; enableManagedSandboxOnly?: boolean }> => ({ enableNativeRunner: false })),
|
||||
}));
|
||||
|
||||
const mockManagedAgentProfileService = vi.hoisted(() => ({
|
||||
@@ -268,6 +268,7 @@ describe("agent routes adapter validation", () => {
|
||||
mockAccessService.setPrincipalPermission.mockResolvedValue(undefined);
|
||||
mockLogActivity.mockResolvedValue(undefined);
|
||||
mockSecretService.syncEnvBindingsForTarget.mockResolvedValue(undefined);
|
||||
mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: null });
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: false });
|
||||
mockManagedAgentProfileService.requireQualified.mockResolvedValue({
|
||||
id: "managed-primary",
|
||||
@@ -959,6 +960,46 @@ describe("agent routes adapter validation", () => {
|
||||
expect(mockAgentService.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("selects the native Cursor API controller from the ordinary held Cloud setup card without a Computer", async () => {
|
||||
const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
mockInstanceSettingsService.get.mockResolvedValue(settings);
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental);
|
||||
const environments = await import("../services/environments.js");
|
||||
const findManaged = vi.fn(async () => null);
|
||||
const getEnvironment = vi.fn(async () => null);
|
||||
vi.spyOn(environments, "environmentService").mockReturnValue({ findManagedSandboxEnvironment: findManaged,
|
||||
getById: getEnvironment, listBoundCompanyIds: vi.fn(async () => []) } as never);
|
||||
const existing = { ...(await mockAgentService.getById()), adapterType: "process", adapterConfig: {},
|
||||
status: "paused", lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] };
|
||||
mockAgentService.getById.mockResolvedValue(existing);
|
||||
const app = await createApp();
|
||||
const response = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({
|
||||
name: "Cursor setup", adapterType: "cursor_cloud", runner: "auto", adapterConfig: {}, metadata: { setupAdapterPending: false },
|
||||
}));
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body).toMatchObject({ lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] });
|
||||
expect(mockAgentService.update).toHaveBeenCalledWith(existing.id, expect.objectContaining({
|
||||
adapterType: "paperclip_runner", defaultEnvironmentId: null,
|
||||
adapterConfig: expect.objectContaining({ provider: "cursor_cloud", lifecycleMode: "per_turn" }),
|
||||
}), expect.anything());
|
||||
expect(findManaged).not.toHaveBeenCalled();
|
||||
expect(getEnvironment).not.toHaveBeenCalled();
|
||||
mockAgentService.update.mockClear();
|
||||
mockAgentService.getById.mockResolvedValue({ ...existing, status: "idle", lifecycleState: "ready", pauseReason: null, lifecycleHolds: [] });
|
||||
const active = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({
|
||||
adapterType: "cursor_cloud", runner: "auto", adapterConfig: {}, setupOnly: true,
|
||||
}));
|
||||
expect(active.status).toBe(422);
|
||||
expect(active.body.details?.code).toBe("managed_sandbox_unavailable");
|
||||
expect(mockAgentService.update).not.toHaveBeenCalled();
|
||||
const incompleteActive = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({
|
||||
adapterType: "paperclip_runner", runner: "paperclip", adapterConfig: { provider: "cursor_cloud" }, setupOnly: true,
|
||||
}));
|
||||
expect(incompleteActive.status).toBe(422);
|
||||
expect(incompleteActive.body.details?.code).toBe("paperclip_runner_cursor_cloud_model_required");
|
||||
expect(mockAgentService.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["auto", "legacy"] as const)("creates Pi through the public route with %s selection and the experimental flag off", async runner => {
|
||||
const selection = await import("../services/agent-runner-selection.js");
|
||||
vi.spyOn(selection, "resolveNewAgentRunnerForCompany").mockImplementation(async (_db, _companyId, input) =>
|
||||
@@ -1107,9 +1148,9 @@ describe("agent routes adapter validation", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("saves and reopens an explicit native Cursor Cloud setup without Codex defaults", async () => {
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true });
|
||||
let existing = { ...(await mockAgentService.getById()), adapterType: "cursor_cloud", adapterConfig: {},
|
||||
it("saves and reopens native Cursor API setup under managed-only policy without a Computer", async () => {
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true, enableManagedSandboxOnly: true });
|
||||
let existing = { ...(await mockAgentService.getById()), adapterType: "cursor_cloud", adapterConfig: {}, defaultEnvironmentId: "33333333-3333-4333-8333-333333333333",
|
||||
status: "paused", lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] };
|
||||
mockAgentService.getById.mockImplementation(async () => existing);
|
||||
mockAgentService.update.mockImplementation(async (_id, patch) => { existing = { ...existing, ...patch }; return existing; });
|
||||
@@ -1123,6 +1164,7 @@ describe("agent routes adapter validation", () => {
|
||||
provider: "cursor_cloud", model: config.model, repoUrl: config.repoUrl, repoRef: "feature", env: config.env, lifecycleMode: "per_turn",
|
||||
} });
|
||||
expect(saved.body.adapterConfig).not.toHaveProperty("codexPermissionMode");
|
||||
expect(saved.body.defaultEnvironmentId).toBeNull();
|
||||
const reopened = await requestApp(app, baseUrl => request(baseUrl).get(`/api/agents/${existing.id}/configuration`));
|
||||
expect(reopened.status, JSON.stringify(reopened.body)).toBe(200);
|
||||
expect(reopened.body.adapterConfig).toEqual(saved.body.adapterConfig);
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { agents, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
|
||||
environments, environmentLeases, executionWorkspaces, heartbeatRuns, plugins, type Db } from "@paperclipai/db";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { requireServerAdapter } from "../adapters/index.js";
|
||||
import { agentHarnessVerificationService } from "../services/agent-harness-verification.js";
|
||||
import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js";
|
||||
import { createLifecycleDriver } from "../services/agent-lifecycle-driver.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_PROVIDER_API_LIFECYCLE_TEST_DATABASE_URL");
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
|
||||
describe.skipIf(!support.supported)("Cursor Cloud API controller activation", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
let home: string;
|
||||
let companyId: string;
|
||||
const workers: ReturnType<typeof configureAgentLifecycle>[] = [];
|
||||
const config = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification",
|
||||
lifecycleMode: "per_turn", env: { CURSOR_API_KEY: "synthetic-cursor-key" } };
|
||||
const fetch = vi.fn(async (input: unknown, init?: RequestInit) => {
|
||||
const url = String(input);
|
||||
if (url.startsWith("https://callback.example.test/mcp/native-runs/") && init?.method === "POST") return new Response(null, { status: 401 });
|
||||
if (url === "https://api.cursor.com/v1/models" && init?.method === "GET") return new Response(JSON.stringify({ items: [{ id: config.model, displayName: "Caller model" }] }), { headers: { "Content-Type": "application/json" } });
|
||||
throw new Error(`Unexpected provider request: ${init?.method} ${url}`);
|
||||
});
|
||||
|
||||
beforeAll(async () => {
|
||||
home = await mkdtemp(join(tmpdir(), "paperclip-provider-api-lifecycle-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined);
|
||||
vi.stubEnv("PAPERCLIP_SETTING_DEFAULTS", undefined);
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://callback.example.test");
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = externalDatabaseUrl ? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
|
||||
: await startEmbeddedPostgresTestDatabase("paperclip-provider-api-lifecycle-");
|
||||
db = createDb(database.connectionString);
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
beforeEach(async () => {
|
||||
for (const worker of workers) await worker.stop();
|
||||
await instanceSettingsService(db).update({ defaultEnvironmentId: null });
|
||||
await db.execute(sql`truncate table companies, plugins, environments restart identity cascade`);
|
||||
await instanceSettingsService(db).updateGeneral({ executionMode: "any" });
|
||||
await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true, enableNativeRunner: true });
|
||||
[companyId] = await db.insert(companies).values({ name: "API controller activation", issuePrefix: "API" }).returning({ id: companies.id }).then(rows => rows.map(row => row.id));
|
||||
fetch.mockClear();
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
});
|
||||
afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); });
|
||||
afterAll(async () => {
|
||||
for (const worker of workers) await worker.stop();
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
vi.unstubAllEnvs();
|
||||
if (home) await rm(home, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const current = (id: string) => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0]!);
|
||||
async function saved(adapterType = "paperclip_runner", adapterConfig = config) {
|
||||
return (await db.insert(agents).values({ companyId, name: "Saved controller", adapterType, adapterConfig,
|
||||
status: "paused", lifecycleState: "verifying" }).returning())[0]!;
|
||||
}
|
||||
async function noComputerWork() {
|
||||
expect(await db.select().from(environmentLeases)).toHaveLength(0);
|
||||
expect(await db.select().from(executionWorkspaces)).toHaveLength(0);
|
||||
expect(await db.select().from(heartbeatRuns)).toHaveLength(0);
|
||||
}
|
||||
|
||||
it.each([false, true])("finishes the same held draft and reaches Ready with an experimental account WARN, inherited plugin Computer=%s", async defaultComputer => {
|
||||
if (defaultComputer) {
|
||||
const pluginId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "unneeded-computer", version: "1.0.0", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", requiresAgent: true, supportsTasks: true, warmTaskVersion: 1 }] } as never });
|
||||
const [computer] = await db.insert(environments).values({ name: "Inherited Computer", driver: "plugin", status: "active",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }).returning();
|
||||
await instanceSettingsService(db).update({ defaultEnvironmentId: computer.id });
|
||||
}
|
||||
expect(await environmentService(db).findManagedSandboxEnvironment(companyId)).toBeNull();
|
||||
const lifecycle = createAgentLifecycle(db);
|
||||
const draft = await lifecycle.requestHire(companyId, { name: "Cursor setup", adapterType: "paperclip_runner",
|
||||
adapterConfig: { provider: "cursor_cloud" } }, { setupOnly: true, runnerResolved: true });
|
||||
expect(draft).toMatchObject({ status: "paused", lifecycleState: "paused", lifecycleHolds: ["setup"], defaultEnvironmentId: null });
|
||||
await lifecycle.updateConfiguration(draft.id, { adapterConfig: config, runtimeConfig: { heartbeat: { enabled: false } } });
|
||||
expect(await current(draft.id)).toMatchObject({ lifecycleState: "paused", lifecycleHolds: ["setup"] });
|
||||
const finished = await lifecycle.requestHire(companyId, { id: draft.id, name: draft.name, adapterType: "paperclip_runner",
|
||||
adapterConfig: config, runtimeConfig: { heartbeat: { enabled: false } }, defaultEnvironmentId: null }, { setupAgentId: draft.id, runnerResolved: true });
|
||||
expect(finished.id).toBe(draft.id);
|
||||
expect(finished.adapterConfig.lifecycleMode).toBe("per_turn");
|
||||
const probe = vi.spyOn(requireServerAdapter("paperclip_runner"), "testEnvironment");
|
||||
const driver = createLifecycleDriver(db, {} as never);
|
||||
const worker = configureAgentLifecycle(db, driver);
|
||||
workers.push(worker);
|
||||
await worker.process(draft.id);
|
||||
expect(await current(draft.id)).toMatchObject({ status: "idle", lifecycleState: "ready", lifecycleHolds: [], defaultEnvironmentId: null });
|
||||
expect(probe).toHaveBeenCalledOnce();
|
||||
expect(await probe.mock.results[0]!.value).toMatchObject({ status: "warn", checks: [
|
||||
{ code: "cursor_cloud_account_ready", level: "info" },
|
||||
{ code: "cursor_cloud_model_listed", level: "info" },
|
||||
{ code: "cursor_cloud_model_identity_unverified", level: "warn" },
|
||||
] });
|
||||
expect(fetch.mock.calls.map(([url, input]) => [String(url), input?.method])).toEqual([
|
||||
["https://callback.example.test/mcp/native-runs/00000000-0000-4000-8000-000000000000", "POST"],
|
||||
["https://api.cursor.com/v1/models", "GET"],
|
||||
]);
|
||||
await noComputerWork();
|
||||
});
|
||||
|
||||
it("keeps native Computer-backed setup preparation and warm-task defaults", async () => {
|
||||
const pluginId = randomUUID();
|
||||
await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "required-computer", version: "1.0.0", status: "ready",
|
||||
manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", requiresAgent: true, supportsTasks: true, warmTaskVersion: 1 }] } as never });
|
||||
const [computer] = await db.insert(environments).values({ name: "Required Computer", driver: "plugin", status: "active",
|
||||
config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }).returning();
|
||||
await instanceSettingsService(db).update({ defaultEnvironmentId: computer.id });
|
||||
const draft = await createAgentLifecycle(db).requestHire(companyId, { name: "Codex setup", adapterType: "paperclip_runner",
|
||||
adapterConfig: { provider: "codex", model: "gpt-5.6-luna", lifecycleMode: "per_turn" } }, { setupOnly: true, runnerResolved: true });
|
||||
expect(draft).toMatchObject({ status: "paused", lifecycleState: "preparing", lifecycleHolds: ["setup"], adapterConfig: { lifecycleMode: "warm" } });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
await noComputerWork();
|
||||
});
|
||||
|
||||
it("verifies a saved native controller without resolving an unavailable managed Computer", async () => {
|
||||
const agent = await saved();
|
||||
await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).resolves.toBe("complete");
|
||||
expect(fetch).toHaveBeenCalledTimes(2);
|
||||
await noComputerWork();
|
||||
});
|
||||
|
||||
it("rejects Kubernetes-only activation before any provider probe", async () => {
|
||||
const agent = await saved();
|
||||
await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" });
|
||||
await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({
|
||||
status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" },
|
||||
});
|
||||
await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Forbidden hire", adapterType: "paperclip_runner", adapterConfig: config }, { runnerResolved: true }))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" } });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
await noComputerWork();
|
||||
});
|
||||
|
||||
it("does not grant API placement to an incomplete active profile", async () => {
|
||||
const agent = await saved("paperclip_runner", { ...config, model: "" });
|
||||
await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({ code: "paperclip_runner_cursor_cloud_model_required" });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
await noComputerWork();
|
||||
});
|
||||
|
||||
it.each(["opencode_local", "cursor_cloud"])("preserves the managed Computer requirement for %s", async adapterType => {
|
||||
const agent = await saved(adapterType);
|
||||
await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({ details: { code: "managed_sandbox_unavailable" } });
|
||||
expect(fetch).not.toHaveBeenCalled();
|
||||
await noComputerWork();
|
||||
});
|
||||
});
|
||||
@@ -293,6 +293,177 @@ describe("agent test-environment route", () => {
|
||||
await unregisterTestAdapter("external_test");
|
||||
});
|
||||
|
||||
it("tests the saved Cursor API controller without allocating a Computer under managed-only policy", async () => {
|
||||
const { registerServerAdapter, unregisterServerAdapter, getServerAdapter } = await import("../adapters/index.js");
|
||||
const previous = getServerAdapter("paperclip_runner");
|
||||
const probe = vi.fn(async () => ({ adapterType: "paperclip_runner", status: "warn" as const, testedAt: new Date().toISOString(),
|
||||
checks: [{ code: "cursor_cloud_account_ready", level: "info" as const, message: "Synthetic account probe; no prompt." }] }));
|
||||
registerServerAdapter({ ...externalAdapter, type: "paperclip_runner", testEnvironment: probe });
|
||||
try {
|
||||
const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
const adapterConfig = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", env: { CURSOR_API_KEY: "synthetic" } };
|
||||
mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "paperclip_runner", adapterConfig, defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" });
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableManagedSandboxOnly: true });
|
||||
mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" });
|
||||
const app = await createApp();
|
||||
const response = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({ agentId: id, adapterConfig });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body).toMatchObject({ status: "warn", checks: [{ code: "cursor_cloud_account_ready" }] });
|
||||
expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null, config: adapterConfig }));
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.realizeWorkspace).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled();
|
||||
mockInstanceSettingsService.getGeneral.mockResolvedValue({ censorUsernameInLogs: false, executionMode: "kubernetes" });
|
||||
const denied = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({ agentId: id, adapterConfig });
|
||||
expect(denied.status).toBe(422);
|
||||
expect(denied.body.details?.code ?? denied.body.code).toBe("provider_api_kubernetes_execution_unsupported");
|
||||
expect(probe).toHaveBeenCalledOnce();
|
||||
} finally { unregisterServerAdapter("paperclip_runner"); if (previous) registerServerAdapter(previous); }
|
||||
});
|
||||
|
||||
async function mockCursorSwitchProbes() {
|
||||
const { getServerAdapter } = await import("../adapters/index.js");
|
||||
const legacy = vi.spyOn(getServerAdapter("cursor_cloud"), "testEnvironment").mockImplementation(async () => ({
|
||||
adapterType: "cursor_cloud", status: "pass", checks: [], testedAt: new Date().toISOString(),
|
||||
}));
|
||||
const native = vi.spyOn(getServerAdapter("paperclip_runner"), "testEnvironment").mockImplementation(async () => ({
|
||||
adapterType: "paperclip_runner", status: "warn", testedAt: new Date().toISOString(),
|
||||
checks: [{ code: "cursor_cloud_account_ready", level: "info", message: "Synthetic no-prompt probe" }],
|
||||
}));
|
||||
return { legacy, native, restore: () => { legacy.mockRestore(); native.mockRestore(); } };
|
||||
}
|
||||
|
||||
function savedCursorSwitchFixture() {
|
||||
const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
const environmentId = "22222222-2222-4222-8222-222222222222";
|
||||
const adapterConfig = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", env: {} };
|
||||
mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "paperclip_runner",
|
||||
adapterConfig, runtimeConfig: {}, defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" });
|
||||
const envVars = { CURSOR_API_KEY: { type: "secret_ref", secretId: "computer-key" } };
|
||||
mockEnvironmentService.getById.mockResolvedValue({ id: environmentId, name: "Chosen Computer",
|
||||
driver: "local", config: {}, envVars });
|
||||
mockSecretService.resolveEnvBindings.mockResolvedValue({ env: { CURSOR_API_KEY: "chosen-computer-key" },
|
||||
secretKeys: new Set(["CURSOR_API_KEY"]), manifest: [] });
|
||||
return { id, environmentId, adapterConfig, envVars };
|
||||
}
|
||||
|
||||
it.each([false, true])("tests a saved native Cursor's explicit Legacy switch on the selected Computer (legacyPrompt=%s)", async legacyPrompt => {
|
||||
const probes = await mockCursorSwitchProbes();
|
||||
const probe = probes.legacy;
|
||||
try {
|
||||
const f = savedCursorSwitchFixture();
|
||||
const config = { ...f.adapterConfig, ...(legacyPrompt ? { promptTemplate: "Legacy prompt" } : {}) };
|
||||
const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
|
||||
.send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId, adapterConfig: config });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(probe).toHaveBeenCalledWith(expect.objectContaining({ environmentName: "Chosen Computer", executionTarget: null,
|
||||
config: expect.objectContaining({ model: "caller-model", env: { CURSOR_API_KEY: "chosen-computer-key" },
|
||||
...(legacyPrompt ? { promptTemplate: "Legacy prompt" } : {}) }) }));
|
||||
expect(probe.mock.calls[0][0].config).not.toHaveProperty("provider");
|
||||
expect(mockSecretService.resolveEnvBindings).toHaveBeenCalledWith("company-1", f.envVars,
|
||||
expect.objectContaining({ consumerType: "environment", consumerId: f.environmentId }));
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(probes.native).not.toHaveBeenCalled();
|
||||
} finally { probes.restore(); }
|
||||
});
|
||||
|
||||
it("fails closed on a foreign Computer before credentials for an explicit Legacy switch", async () => {
|
||||
const probes = await mockCursorSwitchProbes();
|
||||
try {
|
||||
const f = savedCursorSwitchFixture();
|
||||
mockEnvironmentService.listBoundCompanyIds.mockResolvedValue(["company-2"]);
|
||||
const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
|
||||
.send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId,
|
||||
adapterConfig: { ...f.adapterConfig, promptTemplate: "Legacy prompt" } });
|
||||
expect(response.status).toBe(403);
|
||||
expect(mockSecretService.resolveAdapterConfigForRuntime).not.toHaveBeenCalled();
|
||||
expect(mockSecretService.resolveEnvBindings).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(probes.legacy).not.toHaveBeenCalled();
|
||||
expect(probes.native).not.toHaveBeenCalled();
|
||||
} finally { probes.restore(); }
|
||||
});
|
||||
|
||||
it("keeps managed-only policy for a Legacy switch rather than testing its API controller", async () => {
|
||||
const probes = await mockCursorSwitchProbes();
|
||||
const probe = probes.legacy;
|
||||
try {
|
||||
const f = savedCursorSwitchFixture();
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableManagedSandboxOnly: true });
|
||||
const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
|
||||
.send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId, adapterConfig: f.adapterConfig });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body).toMatchObject({ status: "fail", checks: expect.arrayContaining([
|
||||
expect.objectContaining({ code: "managed_sandbox_unavailable", level: "error" }),
|
||||
]) });
|
||||
expect(probe).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(probes.native).not.toHaveBeenCalled();
|
||||
} finally { probes.restore(); }
|
||||
});
|
||||
|
||||
it("keeps an explicit Paperclip Cursor test API-only despite a selected Computer", async () => {
|
||||
const probes = await mockCursorSwitchProbes();
|
||||
const probe = probes.native;
|
||||
try {
|
||||
const f = savedCursorSwitchFixture();
|
||||
const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
mockInstanceSettingsService.get.mockResolvedValue(settings);
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental);
|
||||
const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment")
|
||||
.send({ agentId: f.id, runner: "paperclip", environmentId: f.environmentId, adapterConfig: f.adapterConfig });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null,
|
||||
config: expect.objectContaining({ provider: "cursor_cloud" }) }));
|
||||
expect(mockEnvironmentService.getById).not.toHaveBeenCalled();
|
||||
expect(mockSecretService.resolveEnvBindings).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(probes.legacy).not.toHaveBeenCalled();
|
||||
} finally { probes.restore(); }
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ patch: { model: "" }, code: "paperclip_runner_cursor_cloud_model_required" },
|
||||
{ patch: { command: "untrusted" }, code: "paperclip_runner_cursor_cloud_config_invalid" },
|
||||
])("returns a useful validation error before Computer or credential work for invalid native Cursor input ($code)", async ({ patch, code }) => {
|
||||
const app = await createApp();
|
||||
const response = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({
|
||||
adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", ...patch },
|
||||
});
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(422);
|
||||
expect(response.body.details?.code).toBe(code);
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.realizeWorkspace).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([undefined, "auto"] as const)("tests the prospective managed Cursor API default without converting saved Legacy agents (%s)", async runner => {
|
||||
const { getServerAdapter } = await import("../adapters/index.js");
|
||||
const native = getServerAdapter("paperclip_runner");
|
||||
const probe = vi.spyOn(native, "testEnvironment").mockImplementation(async () => ({ adapterType: "paperclip_runner", status: "warn" as const, testedAt: new Date().toISOString(),
|
||||
checks: [{ code: "cursor_cloud_account_ready", level: "info" as const, message: "Synthetic account probe; no prompt." }] }));
|
||||
try {
|
||||
const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
mockInstanceSettingsService.get.mockResolvedValue(settings);
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental);
|
||||
const adapterConfig = { model: "caller-model", repoUrl: "https://github.com/example/qualification", env: { CURSOR_API_KEY: "synthetic" } };
|
||||
const app = await createApp();
|
||||
const response = await request(app).post("/api/companies/company-1/adapters/cursor_cloud/test-environment").send({ runner, adapterConfig });
|
||||
expect(response.status, JSON.stringify(response.body)).toBe(200);
|
||||
expect(response.body).toMatchObject({ adapterType: "paperclip_runner", checks: [{ code: "cursor_cloud_account_ready" }] });
|
||||
expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null,
|
||||
config: expect.objectContaining({ provider: "cursor_cloud", model: "caller-model" }) }));
|
||||
expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled();
|
||||
expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled();
|
||||
const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa";
|
||||
mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "cursor_cloud", adapterConfig, defaultEnvironmentId: null });
|
||||
const legacy = await request(app).post("/api/companies/company-1/adapters/cursor_cloud/test-environment").send({ agentId: id, adapterConfig });
|
||||
expect(legacy.status).toBe(422);
|
||||
expect(legacy.body.details?.code).toBe("managed_sandbox_unavailable");
|
||||
expect(probe).toHaveBeenCalledOnce();
|
||||
} finally { probe.mockRestore(); }
|
||||
});
|
||||
|
||||
it("accepts a managed Dot environment without allocating an idle sandbox", async () => {
|
||||
vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://paperclip.example");
|
||||
mockInstanceSettingsService.getExperimental.mockResolvedValue({
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
agents,
|
||||
@@ -11,10 +11,14 @@ import {
|
||||
companies,
|
||||
costEvents,
|
||||
createDb,
|
||||
heartbeatRuns,
|
||||
issues,
|
||||
nativeRunFinalizations,
|
||||
projects,
|
||||
} from "@paperclipai/db";
|
||||
import { budgetService, deliverBudgetEnforcement, hasPendingBudgetEnforcement } from "../services/budgets.ts";
|
||||
import {
|
||||
EMBEDDED_POSTGRES_TEST_TIMEOUT_MS,
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
@@ -40,13 +44,16 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => {
|
||||
await import("../services/agent-lifecycle.js");
|
||||
if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-");
|
||||
db = createDb(externalUrl ?? tempDb!.connectionString);
|
||||
}, 20_000);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
|
||||
afterEach(async () => {
|
||||
await db.delete(budgetIncidents);
|
||||
await db.delete(approvals);
|
||||
await db.delete(budgetPolicies);
|
||||
await db.delete(costEvents);
|
||||
await db.delete(nativeRunFinalizations);
|
||||
await db.delete(heartbeatRuns);
|
||||
await db.delete(issues);
|
||||
await db.delete(projects);
|
||||
await db.delete(agents);
|
||||
await db.delete(companies);
|
||||
@@ -118,6 +125,131 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => {
|
||||
return event!;
|
||||
}
|
||||
|
||||
it.each(["no policy", "healthy policy", "warning policy", "manual company pause", "manual project pause", "orphan budget pause", "recovering run"])(
|
||||
"checks %s while another accounting writer holds the company lock",
|
||||
async state => {
|
||||
const { companyId, agentId, projectId } = await createBudgetFixture();
|
||||
if (state === "healthy policy" || state === "warning policy" || state === "recovering run") {
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId,
|
||||
windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
if (state !== "recovering run") await insertCostEvent({ companyId, agentId, costCents: state === "warning policy" ? 80 : 20 });
|
||||
if (state === "warning policy") {
|
||||
const now = new Date();
|
||||
await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc",
|
||||
windowStart: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)),
|
||||
windowEnd: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1)),
|
||||
thresholdType: "soft", amountLimit: 100, amountObserved: 80 });
|
||||
}
|
||||
}
|
||||
if (state === "recovering run") {
|
||||
const [issue] = await db.insert(issues).values({ companyId, projectId, title: "Recover accounting" }).returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, scopeKind: "issue", issueId: issue.id,
|
||||
nativeIssueId: issue.id, runtimeMode: "native", status: "failed", finishedAt: new Date(), costAccountingPending: true }).returning();
|
||||
await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id, phase: "observed" });
|
||||
}
|
||||
if (state === "manual company pause" || state === "orphan budget pause") {
|
||||
await db.update(companies).set({ status: "paused", pauseReason: state === "orphan budget pause" ? "budget" : "manual" })
|
||||
.where(eq(companies.id, companyId));
|
||||
}
|
||||
if (state === "manual project pause") {
|
||||
await db.update(projects).set({ pausedAt: new Date(), pauseReason: "manual" }).where(eq(projects.id, projectId));
|
||||
}
|
||||
|
||||
const reader = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 1 });
|
||||
let acquired!: () => void, release!: () => void;
|
||||
const lockAcquired = new Promise<void>(resolve => { acquired = resolve; });
|
||||
const holdLock = new Promise<void>(resolve => { release = resolve; });
|
||||
const writer = db.transaction(async tx => {
|
||||
await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)).for("no key update");
|
||||
acquired();
|
||||
await holdLock;
|
||||
});
|
||||
try {
|
||||
await Promise.race([lockAcquired, writer]);
|
||||
await reader.execute(sql`set lock_timeout = '500ms'`);
|
||||
if (state === "recovering run") {
|
||||
await expect(budgetService(reader).getInvocationBlock(companyId, agentId, { projectId }))
|
||||
.rejects.toMatchObject({ cause: { code: "55P03" } });
|
||||
} else {
|
||||
const block = await budgetService(reader).getInvocationBlock(companyId, agentId, { projectId });
|
||||
if (state === "manual company pause") {
|
||||
expect(block).toMatchObject({ scopeType: "company", scopeId: companyId,
|
||||
reason: "Company is paused and cannot start new work." });
|
||||
} else if (state === "manual project pause") {
|
||||
expect(block).toMatchObject({ scopeType: "project", scopeId: projectId,
|
||||
reason: "Project is paused and cannot start new work." });
|
||||
} else if (state === "orphan budget pause") {
|
||||
expect(block).toMatchObject({ scopeType: "company", scopeId: companyId,
|
||||
reason: "Budget pause requires a policy or an explicit operator resume." });
|
||||
} else expect(block).toBeNull();
|
||||
if (state === "warning policy") {
|
||||
expect(await reader.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId)))
|
||||
.toMatchObject([{ thresholdType: "soft", status: "open", resolvedAt: null }]);
|
||||
}
|
||||
if (state === "no policy") {
|
||||
await expect(budgetService(reader).evaluateAdmission(companyId, agentId, { projectId }))
|
||||
.rejects.toMatchObject({ cause: { code: "55P03" } });
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
release();
|
||||
try { await writer; }
|
||||
finally { await reader.$client.end({ timeout: 1 }); }
|
||||
}
|
||||
if (state === "recovering run") {
|
||||
const cancelWorkForScope = vi.fn();
|
||||
expect(await budgetService(db, { cancelWorkForScope }).getInvocationBlock(companyId, agentId, { projectId }))
|
||||
.toMatchObject({ scopeType: "company", scopeId: companyId,
|
||||
reason: "New work must wait while a native run recovers its unfinished accounting." });
|
||||
expect(cancelWorkForScope).not.toHaveBeenCalled();
|
||||
expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0])
|
||||
.toMatchObject({ status: "active", pauseReason: null });
|
||||
expect((await db.select().from(agents).where(eq(agents.id, agentId)))[0])
|
||||
.toMatchObject({ status: "active", pauseReason: null });
|
||||
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)))
|
||||
.toMatchObject([{ status: "failed", costAccountingPending: true }]);
|
||||
expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, companyId)))
|
||||
.toMatchObject([{ phase: "observed", resultId: null }]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([true, false])("releases an agent budget hold when its policy is healthy (active: %s)", async isActive => {
|
||||
const { companyId, agentId } = await createBudgetFixture();
|
||||
await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
|
||||
windowKind: "calendar_month_utc", amount: 100, isActive });
|
||||
await db.update(agents).set({ status: "paused", lifecycleState: "paused", pauseReason: "budget", pausedAt: new Date(), lifecycleHolds: ["budget"] })
|
||||
.where(eq(agents.id, agentId));
|
||||
|
||||
expect(await budgetService(db).getInvocationBlock(companyId, agentId)).toBeNull();
|
||||
expect((await db.select().from(agents).where(eq(agents.id, agentId)))[0]).toMatchObject({
|
||||
lifecycleState: "resuming", pauseReason: null, lifecycleHolds: [],
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["hard", "soft"])("resolves an obsolete %s incident even when the scope no longer has a budget hold", async thresholdType => {
|
||||
const { companyId, agentId } = await createBudgetFixture();
|
||||
const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId,
|
||||
windowKind: "calendar_month_utc", amount: 100 }).returning();
|
||||
const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "agent", scopeId: agentId,
|
||||
metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date(0), windowEnd: new Date(1),
|
||||
thresholdType, amountLimit: 100, amountObserved: 100 }).returning();
|
||||
|
||||
expect(await budgetService(db).getInvocationBlock(companyId, agentId)).toBeNull();
|
||||
expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0])
|
||||
.toMatchObject({ status: "resolved", resolvedAt: expect.any(Date) });
|
||||
});
|
||||
|
||||
it("rejects another company's agent or project before allowing a no-policy invocation", async () => {
|
||||
const own = await createBudgetFixture();
|
||||
const other = await createBudgetFixture();
|
||||
const service = budgetService(db);
|
||||
await expect(service.getInvocationBlock(own.companyId, other.agentId)).rejects.toThrow("Budget scope not found");
|
||||
await expect(service.getInvocationBlock(own.companyId, own.agentId, { projectId: other.projectId }))
|
||||
.rejects.toThrow("Budget scope not found");
|
||||
});
|
||||
|
||||
it("raises one soft incident per window before hard-stopping and preserving safe local audit details", async () => {
|
||||
const { companyId, agentId } = await createBudgetFixture();
|
||||
const cancelWorkForScope = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
@@ -11,6 +11,7 @@ import { retainGitHubReviewReceipt } from "../services/chat-channels/github/task
|
||||
import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js";
|
||||
import { recordRunPresentationDecision } from "../services/heartbeat/run-completion.js";
|
||||
import { environmentService } from "../services/environments.js";
|
||||
import { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } from "../services/workspace-runtime.js";
|
||||
import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js";
|
||||
import { chatFailedRetryNoticeWork } from "../services/chat-failed-retry-notice-work.js";
|
||||
import { issueTreeControlService } from "../services/issue-tree-control.js";
|
||||
@@ -33,6 +34,10 @@ import * as githubTaskPreparation from "../services/chat-channels/github/task-pr
|
||||
import { budgetPolicies } from "@paperclipai/db";
|
||||
import { budgetService } from "../services/budgets.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { readIdleLocalWork } from "../services/idle-local-work.js";
|
||||
import { incomingWakeWorkQuery } from "../services/idle-incoming-wake.js";
|
||||
import { createChatRuntime } from "../services/chat-runtime.js";
|
||||
import { createChatSdkRuntime } from "../services/chat-sdk-runtime.js";
|
||||
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
|
||||
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
|
||||
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
|
||||
@@ -809,6 +814,8 @@ class FakeChatSdkRuntime {
|
||||
return this.endpoints.get(endpointId) ?? null;
|
||||
}
|
||||
|
||||
hasUnsettledLifecycle() { return false; }
|
||||
|
||||
async replaceEndpoint(options: CreateChatSdkEndpointRuntimeOptions) {
|
||||
this.replaceCount += 1;
|
||||
this.configurations.set(options.endpointId, options);
|
||||
@@ -1128,8 +1135,20 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
// Do not expire or time out its connections on those virtual clock jumps.
|
||||
db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, {
|
||||
idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0,
|
||||
...(historyTestDatabaseUrl ? { maxConnections: 4 } : {}),
|
||||
...(externalTestDatabaseUrl ? { maxConnections: 4 } : {}),
|
||||
});
|
||||
// The composed local-probe inventory includes the workspace owner. Match
|
||||
// server startup on this fresh fixture; never adopt or start a leftover.
|
||||
const [runtimeState] = await db.execute<{ configured: boolean }>(sql`SELECT
|
||||
EXISTS (SELECT 1 FROM workspace_runtime_services)
|
||||
OR EXISTS (SELECT 1 FROM project_workspaces)
|
||||
OR EXISTS (SELECT 1 FROM execution_workspaces) AS configured`);
|
||||
expect(runtimeState.configured).toBe(false);
|
||||
expect(await reconcilePersistedRuntimeServicesOnStartup(db)).toEqual({
|
||||
reconciled: 0, adopted: 0, stopped: 0, backfilled: 0, restarted: 0,
|
||||
restartFailed: 0, exposureReservationDrift: [],
|
||||
});
|
||||
expect(readWorkspaceRuntimeIdleWork()).toBe("none");
|
||||
}, 90_000);
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -4185,10 +4204,14 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
.toEqual({ next: null, active: 0 });
|
||||
expect(f.responseWrites).toHaveLength(baseline);
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction");
|
||||
const readStacks: string[] = [], select = db.select;
|
||||
const reads = vi.spyOn(db, "select").mockImplementation((...args) => {
|
||||
if (readStacks.length < 5) readStacks.push(new Error("Unexpected idle database read").stack ?? "Stack unavailable");
|
||||
return Reflect.apply(select, db, args);
|
||||
}), writes = vi.spyOn(db, "transaction");
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(120_000);
|
||||
expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled();
|
||||
expect(reads, readStacks.join("\n\n")).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled();
|
||||
expect(f.responseWrites).toHaveLength(baseline);
|
||||
} finally { reads.mockRestore(); writes.mockRestore(); vi.useRealTimers(); }
|
||||
const settlePresentation = async (tx: Parameters<Parameters<Db["transaction"]>[0]>[0]) => {
|
||||
@@ -29957,13 +29980,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
}),
|
||||
);
|
||||
await initializationEntered;
|
||||
expect(runtime.hasUnsettledLifecycle()).toBe(false); // Fake registry tail is already settled.
|
||||
expect(await registeredIdleProbeState()).toBe("present");
|
||||
await service.configure(endpoint.id, { action: "pause" }, "owner-user");
|
||||
let shutdownSettled = false;
|
||||
const shutdown = service.shutdown().then(() => { shutdownSettled = true; });
|
||||
await new Promise<void>(resolve => setImmediate(resolve));
|
||||
expect(shutdownSettled).toBe(false);
|
||||
expect(await registeredIdleProbeState()).toBe("present");
|
||||
releaseInitialization();
|
||||
|
||||
await expect(initializingWebhook).rejects.toMatchObject({
|
||||
status: 409,
|
||||
details: { code: "chat_endpoint_runtime_superseded" },
|
||||
});
|
||||
await shutdown;
|
||||
expect(shutdownSettled).toBe(true);
|
||||
expect(await registeredIdleProbeState()).toBe("none");
|
||||
expect(runtime.endpoints.has(endpoint.id)).toBe(false);
|
||||
await expect(service.get(endpoint.id)).resolves.toMatchObject({
|
||||
status: "paused",
|
||||
@@ -49515,7 +49548,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
.then((rows) => rows.length);
|
||||
expect(deliveryCount).toBeGreaterThan(0);
|
||||
|
||||
const endpointBlocksSleep = async () => (await db.execute(sql`
|
||||
SELECT EXISTS (${incomingWakeWorkQuery("chat_endpoints")} AND endpoint.id = ${endpoint.id}::uuid) AS blocked`))[0].blocked;
|
||||
if (provider === "github") expect(await endpointBlocksSleep()).toBe(true);
|
||||
await service.configure(endpoint.id, { action: "remove" }, "owner-user");
|
||||
if (provider === "github") expect(await endpointBlocksSleep()).toBe(false);
|
||||
else expect(await endpointBlocksSleep()).toBe(true); // No persistent/provider coverage expansion.
|
||||
|
||||
await expect(service.get(endpoint.id)).resolves.toMatchObject({
|
||||
status: "archived",
|
||||
@@ -49557,6 +49595,44 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
},
|
||||
);
|
||||
|
||||
async function registeredIdleProbeState() {
|
||||
let state: string | undefined;
|
||||
await readIdleLocalWork({ diagnostic: (reason, value) => { if (reason === "local.registered_probes") state = value; } });
|
||||
return state;
|
||||
}
|
||||
|
||||
it.each([
|
||||
{ name: "SDK", create: createChatSdkRuntime },
|
||||
{ name: "shared", create: createChatRuntime },
|
||||
])("retains the service idle probe after failed $name retirement and shutdown until owner retry", async ({ create }) => {
|
||||
const fixture = await seedCompany();
|
||||
const f = await configuredGitHubEndpoint(fixture);
|
||||
const registry = create();
|
||||
const first = await registry.replaceEndpoint(f.runtime.configurations.get(f.endpoint.id)!);
|
||||
const shutdown = vi.spyOn(first, "shutdown")
|
||||
.mockRejectedValueOnce(new Error("remove retirement uncertain"))
|
||||
.mockRejectedValueOnce(new Error("shutdown retirement uncertain"))
|
||||
.mockResolvedValue(undefined);
|
||||
const service = chatChannelService(db, { runtime: registry, fetch: f.providerFetch, publicBaseUrl: "https://paperclip.example" });
|
||||
fixtureServices.add(service);
|
||||
expect(await registeredIdleProbeState()).toBe("none");
|
||||
await service.configure(f.endpoint.id, { action: "remove" }, "owner-user");
|
||||
expect(registry.get(f.endpoint.id)).toBeNull();
|
||||
expect(registry.list()).toEqual([]);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
const select = vi.spyOn(db, "select"), transaction = vi.spyOn(db, "transaction");
|
||||
try {
|
||||
expect(await registeredIdleProbeState()).toBe("present");
|
||||
expect(select).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled();
|
||||
} finally { select.mockRestore(); transaction.mockRestore(); }
|
||||
await expect(service.shutdown()).rejects.toThrow("shutdown retirement uncertain");
|
||||
expect(await registeredIdleProbeState()).toBe("present");
|
||||
await service.shutdown();
|
||||
expect(shutdown).toHaveBeenCalledTimes(3);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(false);
|
||||
expect(await registeredIdleProbeState()).toBe("none");
|
||||
});
|
||||
|
||||
it("retries credential cleanup after archival made a remove partially complete", async () => {
|
||||
const fixture = await seedCompany();
|
||||
const { endpoint, service } = await configuredSlackEndpoint(fixture);
|
||||
@@ -60587,7 +60663,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
const commentsBefore = await db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId));
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(asc(issueComments.id));
|
||||
const wakesBefore = wakeup.mock.calls.length;
|
||||
const reactionsBefore = providerRuntime.reactions.length;
|
||||
const postsBefore = providerRuntime.posts.length;
|
||||
@@ -60617,7 +60694,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
await db
|
||||
.select({ id: issueComments.id })
|
||||
.from(issueComments)
|
||||
.where(eq(issueComments.issueId, conversation.issueId)),
|
||||
.where(eq(issueComments.issueId, conversation.issueId))
|
||||
.orderBy(asc(issueComments.id)),
|
||||
).toEqual(commentsBefore);
|
||||
expect(wakeup).toHaveBeenCalledTimes(wakesBefore);
|
||||
expect(cancelRun).not.toHaveBeenCalled();
|
||||
@@ -64441,6 +64519,45 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
|
||||
} finally { await f.service.shutdown(); }
|
||||
});
|
||||
|
||||
it("retries saved GitHub input after an endpoint admission lock collision", async () => {
|
||||
const f = await successfulGitHubQueuedInputFixture();
|
||||
const heartbeat = heartbeatService(db);
|
||||
const failures: unknown[] = [];
|
||||
const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true,
|
||||
onError: error => failures.push(error) });
|
||||
let unlock!: () => void, locked!: () => void;
|
||||
const acquired = new Promise<void>(resolve => { locked = resolve; });
|
||||
const released = new Promise<void>(resolve => { unlock = resolve; });
|
||||
const competingAdmission = db.transaction(async tx => {
|
||||
await tx.select({ id: chatEndpoints.id }).from(chatEndpoints)
|
||||
.where(eq(chatEndpoints.id, f.endpoint.id)).for("no key update");
|
||||
locked();
|
||||
await released;
|
||||
});
|
||||
try {
|
||||
await acquired;
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => true).ready;
|
||||
const priorFailures = failures.length;
|
||||
const retry = await requestTaskRetry(f.input);
|
||||
await expect.poll(() => failures.length, { timeout: 5_000 }).toBeGreaterThan(priorFailures);
|
||||
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
|
||||
unlock(); await competingAdmission;
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId,
|
||||
{ timeout: 40_000 }).toBeTruthy();
|
||||
await expect.poll(async () => (await readAgentWakeupRequestsById(retry.requestId!))[0].status,
|
||||
{ timeout: 40_000 }).toBe("coalesced");
|
||||
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
|
||||
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId))
|
||||
.filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
|
||||
expect((await readAgentWakeupRequestsById(retry.requestId!))[0].runId).toBe(saved.runId);
|
||||
expect((await readHeartbeatRunsById(f.runId))[0]).toMatchObject({ status: "succeeded" });
|
||||
} finally {
|
||||
unlock(); await competingAdmission;
|
||||
await coordinator.stop();
|
||||
await f.service.shutdown();
|
||||
}
|
||||
}, 90_000);
|
||||
|
||||
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
|
||||
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
|
||||
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
|
||||
|
||||
@@ -14,7 +14,7 @@ function createService(db: Db) {
|
||||
const runtimeShutdown = vi.fn().mockResolvedValue(undefined);
|
||||
const service = chatChannelService(db, {
|
||||
heartbeat: { wakeup },
|
||||
runtime: { shutdown: runtimeShutdown } as unknown as ChatChannelServiceOptions["runtime"],
|
||||
runtime: { shutdown: runtimeShutdown, hasUnsettledLifecycle: () => false } as unknown as ChatChannelServiceOptions["runtime"],
|
||||
});
|
||||
services.add(service);
|
||||
return { service, wakeup, runtimeShutdown };
|
||||
|
||||
@@ -72,14 +72,18 @@ describe("deferred receipt durable dispatch", () => {
|
||||
// during actual process execution; accelerated steps only settle idle work.
|
||||
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true });
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
coordinator = createCoordinator();
|
||||
});
|
||||
function createCoordinator() {
|
||||
const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true,
|
||||
onError: error => { errors.push(error); errorTimes.push(Date.now()); } });
|
||||
const register = coordinator.register.bind(coordinator);
|
||||
coordinator.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const register = worker.register.bind(worker);
|
||||
worker.register = (queue, task) => register(queue, { ...task, run: signal => {
|
||||
const pass = Promise.resolve(task.run(signal)); passes.add(pass);
|
||||
void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass;
|
||||
} });
|
||||
});
|
||||
return worker;
|
||||
}
|
||||
afterEach(async () => {
|
||||
await coordinator.stop(); await heartbeat.drainActiveRunExecutions();
|
||||
for (const stop of extraStops.splice(0)) await stop();
|
||||
@@ -153,6 +157,78 @@ describe("deferred receipt durable dispatch", () => {
|
||||
source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } });
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet();
|
||||
});
|
||||
it.each([false, true])("reconsiders every saved receipt during repeated company eligibility changes (expired wait: %s)", async expiredWait => {
|
||||
const f = await fixture();
|
||||
const ids = Array.from({ length: 178 }, (_, index) => `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`);
|
||||
await db.insert(agentWakeupRequests).values(ids.map(id => ({ id, companyId: f.companyId, agentId: f.agent.id,
|
||||
source: "on_demand", status: "deferred_issue_execution",
|
||||
updatedAt: new Date(Date.now() - 60_000),
|
||||
payload: { issueId: f.issue.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] },
|
||||
...(expiredWait ? { executionWait: { reason: "recovery" } } : {}) },
|
||||
})));
|
||||
const checked = new Set<string>();
|
||||
await registerDeferredReceiptWork(db, coordinator, {
|
||||
resume: async scope => { checked.add(scope.requestId!); },
|
||||
settleTaskChat: async () => {}, createNativeBatch: () => async () => {}, schedulingSuppressed: async () => false,
|
||||
}, () => !drain).ready;
|
||||
for (let pass = 0; pass < 12; pass++) {
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.advanceTimersByTimeAsync(31_000);
|
||||
await Promise.allSettled([...passes]);
|
||||
await immediate();
|
||||
}
|
||||
expect(checked.size).toBe(ids.length);
|
||||
await quiet();
|
||||
});
|
||||
it("dispatches task Retry past a repeatedly invalidated backlog into real process output", async () => {
|
||||
const f = await fixture();
|
||||
// Hold executor capacity while queue time advances; reconstruct the saved
|
||||
// run with fresh workers at PostgreSQL time before real process execution.
|
||||
const [occupant] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id,
|
||||
status: "running", startedAt: new Date() }).returning();
|
||||
await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0,
|
||||
wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agent.id));
|
||||
const [closed] = await db.insert(issues).values({ companyId: f.companyId, title: "Closed task history",
|
||||
status: "done", assigneeAgentId: f.agent.id }).returning();
|
||||
await db.insert(agentWakeupRequests).values(Array.from({ length: 178 }, (_, index) => ({
|
||||
id: `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`,
|
||||
companyId: f.companyId, agentId: f.agent.id, source: "on_demand", status: "deferred_issue_execution",
|
||||
payload: { issueId: closed.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] } },
|
||||
})));
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId,
|
||||
observedRunId: f.source.id, idempotencyKey: randomUUID() });
|
||||
const read = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0];
|
||||
for (let pass = 0; pass < 12 && !(await read()).runId; pass++) {
|
||||
await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId }));
|
||||
await vi.advanceTimersByTimeAsync(31_000);
|
||||
await Promise.allSettled([...passes]);
|
||||
await immediate();
|
||||
}
|
||||
const receipt = await read();
|
||||
expect(receipt.runId).toBeTruthy();
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!)))[0].status).toBe("queued");
|
||||
await coordinator.stop();
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
vi.setSystemTime(vi.getRealSystemTime());
|
||||
heartbeat = heartbeatService(db);
|
||||
coordinator = createCoordinator();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready;
|
||||
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupant.id));
|
||||
await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready;
|
||||
await waitForReal(async () => {
|
||||
await heartbeat.drainActiveRunExecutions();
|
||||
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!));
|
||||
expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded");
|
||||
expect(run.stdoutExcerpt).toContain("receipt-proof");
|
||||
}, { timeout: 15_000 });
|
||||
expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||
.where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${retry.requestId}`)).toHaveLength(1);
|
||||
await finish(f.issue.id);
|
||||
await quiet();
|
||||
});
|
||||
it("dispatches an ordinary committed task Retry into saved process output", async () => {
|
||||
const f = await fixture();
|
||||
await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready;
|
||||
|
||||
@@ -10,8 +10,11 @@ import { randomBytes, randomUUID } from "node:crypto";
|
||||
import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { readRemoteWorkspaceAuthority } from "@paperclipai/shared";
|
||||
import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js";
|
||||
import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js";
|
||||
import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js";
|
||||
import { agentFileStore } from "../services/agent-file-store.js";
|
||||
import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js";
|
||||
@@ -35,6 +38,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
|
||||
home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-"));
|
||||
await s3.send(new CreateBucketCommand({ Bucket: bucket }));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true");
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true");
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3");
|
||||
vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1");
|
||||
vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!);
|
||||
@@ -43,6 +47,10 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
|
||||
});
|
||||
afterEach(async () => {
|
||||
await db.delete(activityLog).where(eq(activityLog.companyId, companyId));
|
||||
await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId));
|
||||
await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId));
|
||||
await db.delete(issues).where(eq(issues.companyId, companyId));
|
||||
await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId));
|
||||
await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId));
|
||||
await db.delete(agents).where(eq(agents.companyId, companyId));
|
||||
await db.delete(companies).where(eq(companies.id, companyId));
|
||||
@@ -90,6 +98,33 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll
|
||||
await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" }));
|
||||
await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/);
|
||||
}, 120_000);
|
||||
it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => {
|
||||
vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32));
|
||||
const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task");
|
||||
await fs.mkdir(child, { recursive: true });
|
||||
const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }];
|
||||
const ids: string[] = [];
|
||||
for (const fixture of fixtures) {
|
||||
await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes);
|
||||
const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning();
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id,
|
||||
name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning();
|
||||
ids.push(workspace!.id);
|
||||
await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id));
|
||||
}
|
||||
expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } });
|
||||
await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-"));
|
||||
vi.stubEnv("PAPERCLIP_HOME", home);
|
||||
for (const [index, workspaceId] of ids.entries()) {
|
||||
const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId));
|
||||
const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!;
|
||||
const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! });
|
||||
const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`);
|
||||
await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath });
|
||||
await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest });
|
||||
expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes);
|
||||
}
|
||||
}, 120_000);
|
||||
const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url));
|
||||
it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => {
|
||||
await agent();
|
||||
|
||||
@@ -16,6 +16,7 @@ import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
|
||||
@@ -92,7 +93,7 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: {
|
||||
}
|
||||
throw new Error("refresh worker did not settle");
|
||||
}
|
||||
async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); }
|
||||
async function quiet() { await settle(() => coordinator!.nextWakeAt() === null && idleWorkSnapshot().active === 0); }
|
||||
function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); }
|
||||
async function change(companyId: string, patch: Partial<typeof externalObjects.$inferInsert>) {
|
||||
await db.transaction(async tx => {
|
||||
@@ -601,7 +602,10 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: {
|
||||
await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId,
|
||||
sourceIssueId: f.issueId, sourceKind: "description" as const, objectId })));
|
||||
}
|
||||
await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime());
|
||||
await start(svc);
|
||||
await vi.waitFor(() => {
|
||||
expect(coordinator!.nextWakeAt()).toBe(future.getTime());
|
||||
}, { timeout: 10_000, interval: 5 });
|
||||
const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming",
|
||||
description: "https://example.com/new-object" }).returning();
|
||||
const reads: Array<{ query: string; params: unknown[] }> = [];
|
||||
|
||||
@@ -151,6 +151,16 @@ const support = await getEmbeddedPostgresTestSupport();
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace", "nativeWorkspaceFinalizationOwner", "sessionCheckpoint"])(
|
||||
"retains an admission-wait hold with a retained %s profile", async key => {
|
||||
const f = await seed(true);
|
||||
const [run] = await db.update(heartbeatRuns).set({ runnerProfileJson: { [key]: { retained: true } } })
|
||||
.where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId });
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["started", "runtime_selected", "process", "session", "controller", "stage", "receipt", "receipt_issue", "receipt_action", "lease", "coordinator", "launch", "provider", "workspace"])(
|
||||
"retains a hold with conflicting or incomplete %s evidence", async kind => {
|
||||
const f = await seed(true);
|
||||
|
||||
@@ -16,6 +16,7 @@ import { prepareNativeHeartbeatRun } from "../../services/native-runtime/prepare
|
||||
import { createIsolatedTestHome } from "./isolated-test-home.js";
|
||||
import { type NativeControllerIdentity } from "../../services/native-runtime/native-restart-recovery.js";
|
||||
import { waitForPendingRunFailureReports } from "../../services/run-failure-report.js";
|
||||
import { registerLocalRuntimeApiServer } from "../../runtime-api.js";
|
||||
|
||||
// An explicitly supplied disposable database also permits qualification on hosts
|
||||
// where native Postgres cannot allocate semaphores. Never fall back silently.
|
||||
@@ -106,6 +107,7 @@ export function nativeRestartFixture() {
|
||||
: new Date(),
|
||||
};
|
||||
server = createServer();
|
||||
registerLocalRuntimeApiServer(server);
|
||||
await new Promise<void>((resolveListen) =>
|
||||
server!.listen(0, "127.0.0.1", resolveListen),
|
||||
);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { createDb } from "@paperclipai/db";
|
||||
import { closeIdleWarmNativeSessionsForRestart, executePaperclipNativeSession } from "../../services/native-runtime/native-session-executor.js";
|
||||
import { dockerCommandRunner } from "./docker-command-runner.js";
|
||||
import { createServer } from "node:http";
|
||||
import { registerLocalRuntimeApiServer } from "../../runtime-api.js";
|
||||
|
||||
// Separate TypeScript fixture import time from actual controller admission.
|
||||
process.send?.({ ready: true });
|
||||
@@ -15,7 +17,10 @@ process.once("message", async (input: {
|
||||
}) => {
|
||||
const target = input.options.runnerExecutionTarget;
|
||||
if (target?.kind !== "remote") throw new Error("Expected remote worker target");
|
||||
const api = createServer((_request, response) => { response.writeHead(404).end(); });
|
||||
registerLocalRuntimeApiServer(api);
|
||||
try {
|
||||
await new Promise<void>(resolve => api.listen(0, "127.0.0.1", resolve));
|
||||
const runner = dockerCommandRunner(input.container);
|
||||
let inspectionOverridden = false;
|
||||
const execute = runner.execute.bind(runner);
|
||||
@@ -55,6 +60,9 @@ process.once("message", async (input: {
|
||||
process.send?.({ result: { ...result, inspectionOverridden } });
|
||||
} catch (error) {
|
||||
process.send?.({ error: error instanceof Error ? error.stack : String(error) });
|
||||
} finally {
|
||||
api.closeAllConnections();
|
||||
await new Promise<void>(resolve => api.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
setInterval(() => undefined, 60_000);
|
||||
@@ -0,0 +1,700 @@
|
||||
import { matchesApiWakeAdmission } from "../services/idle-api-wake-admission.js";
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
beforeEach,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from "vitest";
|
||||
import {
|
||||
agents,
|
||||
agentApiKeys,
|
||||
authUsers,
|
||||
boardApiKeys,
|
||||
companies,
|
||||
companyMemberships,
|
||||
createDb,
|
||||
toolApplications,
|
||||
toolConnections,
|
||||
connectionGrants,
|
||||
chatEndpoints,
|
||||
chatGitHubRegistrations,
|
||||
chatSlackRegistrations,
|
||||
chatEndpointLeases,
|
||||
chatActions,
|
||||
chatDeliveries,
|
||||
chatPublications,
|
||||
chatConversations,
|
||||
issues,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { setCloudRuntimeConfigSnapshot } from "../services/cloud-runtime-config-snapshot.js";
|
||||
import { actorMiddleware } from "../middleware/auth.js";
|
||||
import {
|
||||
idleAdmissionMiddleware,
|
||||
trackIdleRequestHandlers,
|
||||
} from "../middleware/idle-admission.js";
|
||||
import { errorHandler } from "../middleware/error-handler.js";
|
||||
import { companyRoutes } from "../routes/companies.js";
|
||||
import { agentService } from "../services/agents.js";
|
||||
import { boardAuthService } from "../services/board-auth.js";
|
||||
import { readIdleSleepSafety } from "../services/idle-sleep-safety.js";
|
||||
import {
|
||||
readIdleIncomingWakeProof,
|
||||
incomingWakeFingerprint,
|
||||
incomingWakeWorkQuery,
|
||||
} from "../services/idle-incoming-wake.js";
|
||||
import { registerConnectionEventWake } from "../services/connection-event-wake.js";
|
||||
import { idleApiActivity } from "../services/idle-api-activity.js";
|
||||
import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js";
|
||||
import {
|
||||
verifyCloudControlAssertion,
|
||||
CLOUD_CONTROL_AUDIENCE,
|
||||
CLOUD_CONTROL_JWS_TYPE,
|
||||
CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
} from "../services/cloud-runtime-identity.js";
|
||||
import type {
|
||||
IdleIncomingWakeCapability,
|
||||
VerifiedIdleCapabilities,
|
||||
} from "../services/idle-incoming-wake-capability.js";
|
||||
|
||||
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
|
||||
import {
|
||||
getEmbeddedPostgresTestSupport,
|
||||
startEmbeddedPostgresTestDatabase,
|
||||
} from "./helpers/embedded-postgres.js";
|
||||
const url = resolveExternalTestDatabaseUrl(
|
||||
"PAPERCLIP_INCOMING_WAKE_TEST_DATABASE_URL",
|
||||
);
|
||||
const support = url
|
||||
? { supported: true }
|
||||
: await getEmbeddedPostgresTestSupport();
|
||||
const ownerId = "00000000-0000-4000-8000-000000000001";
|
||||
const now = Date.now();
|
||||
const hold = () => ({
|
||||
ownerId,
|
||||
draining: true,
|
||||
startedAt: new Date(now - 1000),
|
||||
expiresAt: new Date(now + 300_000),
|
||||
activeRuns: 0,
|
||||
pendingWakes: 0,
|
||||
});
|
||||
const capability: IdleIncomingWakeCapability = {
|
||||
version: 1,
|
||||
ownerId,
|
||||
generation: "a".repeat(64),
|
||||
apiKeyProof: { version: 1 as const, challenge: "c".repeat(64) },
|
||||
routes: ["tenant-api-http-v2"],
|
||||
};
|
||||
function verified(incomingWake = capability): VerifiedIdleCapabilities {
|
||||
const pair = generateKeyPairSync("ed25519"),
|
||||
kid = randomUUID(),
|
||||
issued = Math.floor(Date.now() / 1000);
|
||||
const encode = (v: unknown) =>
|
||||
Buffer.from(JSON.stringify(v)).toString("base64url");
|
||||
const header = encode({ alg: "EdDSA", typ: CLOUD_CONTROL_JWS_TYPE, kid });
|
||||
const payload = encode({
|
||||
v: 1,
|
||||
iss: CLOUD_RUNTIME_IDENTITY_ISSUER,
|
||||
aud: CLOUD_CONTROL_AUDIENCE,
|
||||
sub: "fixture-stack",
|
||||
action: "task-drain:read",
|
||||
requestId: randomUUID(),
|
||||
iat: issued,
|
||||
exp: issued + 60,
|
||||
idleCapabilities: { incomingWake },
|
||||
});
|
||||
const token = `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), pair.privateKey).toString("base64url")}`;
|
||||
return verifyCloudControlAssertion({
|
||||
compactJws: token,
|
||||
expectedAction: "task-drain:read",
|
||||
env: {
|
||||
PAPERCLIP_CLOUD_STACK_ID: "fixture-stack",
|
||||
PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({
|
||||
keys: [
|
||||
{
|
||||
...pair.publicKey.export({ format: "jwk" }),
|
||||
kid,
|
||||
use: "sig",
|
||||
alg: "EdDSA",
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
}).idleCapabilities as VerifiedIdleCapabilities;
|
||||
}
|
||||
|
||||
(support.supported ? describe : describe.skip)(
|
||||
"scoped incoming wake PostgreSQL contract",
|
||||
() => {
|
||||
let db: Db;
|
||||
let database:
|
||||
Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | undefined;
|
||||
beforeAll(async () => {
|
||||
if (!url)
|
||||
database = await startEmbeddedPostgresTestDatabase(
|
||||
"idle-incoming-wake-",
|
||||
);
|
||||
db = createDb(url ?? database!.connectionString, { maxConnections: 4 });
|
||||
await db.execute(sql`SELECT 1 FROM agent_api_keys LIMIT 1`); // Coordinator migrates the fresh fixture; no embedded fallback.
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-only");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack");
|
||||
setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: null });
|
||||
});
|
||||
afterAll(async () => {
|
||||
setCloudRuntimeConfigSnapshot(null);
|
||||
stopTaskDrain();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
await db?.$client.end({ timeout: 5 });
|
||||
await database?.cleanup();
|
||||
});
|
||||
// The externally supplied database is dedicated to this suite. Preserve all
|
||||
// rows during every eligibility scenario, then isolate the next scenario.
|
||||
const resetFixture = async () => {
|
||||
await db.execute(
|
||||
sql`TRUNCATE companies, "user", chat_github_registrations CASCADE`,
|
||||
);
|
||||
};
|
||||
beforeEach(resetFixture);
|
||||
afterEach(async () => {
|
||||
stopTaskDrain();
|
||||
vi.restoreAllMocks();
|
||||
await resetFixture();
|
||||
});
|
||||
|
||||
const read = (incomingWake?: VerifiedIdleCapabilities) =>
|
||||
readIdleSleepSafety(
|
||||
db,
|
||||
hold,
|
||||
() => now,
|
||||
ownerId,
|
||||
async () => "none",
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
{ incomingWake },
|
||||
);
|
||||
async function seed() {
|
||||
const companyId = randomUUID(),
|
||||
agentId = randomUUID(),
|
||||
userId = randomUUID(),
|
||||
foreignId = randomUUID();
|
||||
await db.insert(companies).values([
|
||||
{
|
||||
id: companyId,
|
||||
name: "Incoming fixture",
|
||||
issuePrefix: `I${companyId.slice(0, 6)}`,
|
||||
},
|
||||
{
|
||||
id: foreignId,
|
||||
name: "Foreign fixture",
|
||||
issuePrefix: `F${foreignId.slice(0, 6)}`,
|
||||
},
|
||||
]);
|
||||
await db
|
||||
.insert(authUsers)
|
||||
.values({
|
||||
id: userId,
|
||||
name: "Fixture owner",
|
||||
email: `${userId}@example.test`,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await db
|
||||
.insert(companyMemberships)
|
||||
.values({
|
||||
companyId,
|
||||
principalType: "user",
|
||||
principalId: userId,
|
||||
membershipRole: "owner",
|
||||
status: "active",
|
||||
});
|
||||
await db
|
||||
.insert(agents)
|
||||
.values({
|
||||
id: agentId,
|
||||
companyId,
|
||||
name: "Fixture agent",
|
||||
role: "engineer",
|
||||
adapterType: "process",
|
||||
status: "idle",
|
||||
});
|
||||
const board = await boardAuthService(db).createNamedBoardApiKey({
|
||||
userId,
|
||||
name: "incoming board",
|
||||
expiresAt: null,
|
||||
});
|
||||
const agent = await agentService(db).createApiKey(
|
||||
agentId,
|
||||
"incoming agent",
|
||||
{ kind: "standard" },
|
||||
{ responsibleUserId: userId },
|
||||
);
|
||||
return { companyId, foreignId, userId, agentId, board, agent };
|
||||
}
|
||||
it("retains real saved keys and exempts them only for the verified exact owner/profile", async () => {
|
||||
const f = await seed();
|
||||
expect((await read()).backgroundWork).toBe("present");
|
||||
expect(await read(verified())).toMatchObject({
|
||||
version: 3,
|
||||
backgroundWork: "none",
|
||||
incomingWake: capability,
|
||||
});
|
||||
const receipt = (await read(verified())).apiWakeAdmission!;
|
||||
expect(receipt.keys).toHaveLength(2);
|
||||
expect(matchesApiWakeAdmission(receipt, { stackId: "fixture-stack", token: f.board.token, now })).toBe(true);
|
||||
expect(matchesApiWakeAdmission(receipt, { stackId: "fixture-stack", token: f.agent.token, now })).toBe(true);
|
||||
const serialized = JSON.stringify(receipt);
|
||||
expect(serialized).not.toContain(f.board.token);
|
||||
expect(serialized).not.toContain(f.agent.token);
|
||||
await db.update(boardApiKeys).set({ revokedAt: new Date() }).where(eq(boardApiKeys.id, f.board.id));
|
||||
const changedReceipt = (await read(verified())).apiWakeAdmission!;
|
||||
expect(changedReceipt.generation).not.toBe(receipt.generation);
|
||||
expect(matchesApiWakeAdmission(changedReceipt, { stackId: "fixture-stack", token: f.board.token, now })).toBe(false);
|
||||
expect(
|
||||
(await read(verified({ ...capability, ownerId: randomUUID() })))
|
||||
.backgroundWork,
|
||||
).toBe("unknown");
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(agentApiKeys)
|
||||
.where(eq(agentApiKeys.id, f.agent.id)),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(boardApiKeys)
|
||||
.where(eq(boardApiKeys.id, f.board.id)),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
it("delivers through actual tenant auth/routes, denies current revoked/expired/foreign authority and holds only successful API use", async () => {
|
||||
const f = await seed();
|
||||
let monotonic = 1;
|
||||
vi.spyOn(performance, "now").mockImplementation(() => monotonic);
|
||||
let release: (() => void) | undefined, arrived: (() => void) | undefined;
|
||||
const app = express();
|
||||
// Named Cloud wake boundary: preserve request until the simulated compute handoff finishes.
|
||||
app.use(async (_req, _res, next) => {
|
||||
if (release) {
|
||||
await new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
arrived?.();
|
||||
});
|
||||
}
|
||||
next();
|
||||
});
|
||||
app.use(idleAdmissionMiddleware);
|
||||
app.use(express.json());
|
||||
app.use(
|
||||
actorMiddleware(db, {
|
||||
deploymentMode: "authenticated",
|
||||
resolveSession: async () => null,
|
||||
}),
|
||||
);
|
||||
app.use("/api/companies", companyRoutes(db));
|
||||
app.use(errorHandler);
|
||||
trackIdleRequestHandlers(app);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.get(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", "Bearer unknown")
|
||||
).status,
|
||||
).toBe(401);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.get(`/api/companies/${f.foreignId}`)
|
||||
.set("Authorization", `Bearer ${f.board.token}`)
|
||||
).status,
|
||||
).toBe(403);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
const own = await request(app)
|
||||
.patch(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", `Bearer ${f.board.token}`)
|
||||
.send({ name: "Persisted after wake" });
|
||||
expect(own.status).toBe(200);
|
||||
expect(idleApiActivity.active()).toBe(true);
|
||||
expect(
|
||||
(
|
||||
await db.select().from(companies).where(eq(companies.id, f.companyId))
|
||||
)[0]!.name,
|
||||
).toBe("Persisted after wake");
|
||||
monotonic += 300_000;
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.get(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", `Bearer ${f.agent.token}`)
|
||||
).status,
|
||||
).toBe(200);
|
||||
monotonic += 300_001;
|
||||
expect(idleApiActivity.active()).toBe(true);
|
||||
monotonic += 299_999;
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
const arrival = new Promise<void>((resolve) => {
|
||||
arrived = resolve;
|
||||
});
|
||||
release = () => {};
|
||||
const held = request(app)
|
||||
.patch(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", `Bearer ${f.board.token}`)
|
||||
.send({ name: "Must not commit" })
|
||||
.then((r) => r);
|
||||
await arrival;
|
||||
await db
|
||||
.update(boardApiKeys)
|
||||
.set({ revokedAt: new Date() })
|
||||
.where(eq(boardApiKeys.id, f.board.id));
|
||||
release!();
|
||||
release = undefined;
|
||||
expect((await held).status).toBe(401);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
const expired = await boardAuthService(db).createNamedBoardApiKey({
|
||||
userId: f.userId,
|
||||
name: "expired",
|
||||
expiresAt: new Date(0),
|
||||
});
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.get(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", `Bearer ${expired.token}`)
|
||||
).status,
|
||||
).toBe(401);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
startTaskDrain({ purpose: "idle", ttlMs: 60_000 });
|
||||
expect(
|
||||
(
|
||||
await request(app)
|
||||
.patch(`/api/companies/${f.companyId}`)
|
||||
.set("Authorization", `Bearer ${f.agent.token}`)
|
||||
.send({ name: "Draining" })
|
||||
).status,
|
||||
).toBe(503);
|
||||
expect(idleApiActivity.active()).toBe(false);
|
||||
stopTaskDrain();
|
||||
expect(
|
||||
(
|
||||
await db.select().from(companies).where(eq(companies.id, f.companyId))
|
||||
)[0]!.name,
|
||||
).toBe("Persisted after wake");
|
||||
});
|
||||
// Fault injection below supplies only malformed/incomplete persisted states;
|
||||
// the positive archived transition is covered by the real configure(remove)
|
||||
// producer in chat-channels.integration.test.ts, without deleting history.
|
||||
async function removedGitHubFixture() {
|
||||
const f = await seed(), applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID();
|
||||
await db.insert(toolApplications).values({ id: applicationId, companyId: f.companyId, name: "Removed GitHub", type: "chat" });
|
||||
await db.insert(toolConnections).values({ id: connectionId, companyId: f.companyId, applicationId,
|
||||
name: "Removed GitHub", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel",
|
||||
status: "archived", enabled: false, credentialSecretRefs: [] });
|
||||
await db.insert(chatEndpoints).values({ id: endpointId, companyId: f.companyId, connectionId,
|
||||
assignedAgentId: f.agentId, provider: "github", publicId: randomUUID(), status: "archived",
|
||||
archivedAt: new Date(), setup: { step: "complete", runtimeGeneration: 1 } });
|
||||
const blocked = async () => (await db.execute(sql`SELECT EXISTS (
|
||||
${incomingWakeWorkQuery("chat_endpoints")} AND endpoint.id = ${endpointId}::uuid) AS blocked`))[0].blocked;
|
||||
return { ...f, applicationId, connectionId, endpointId, blocked };
|
||||
}
|
||||
it.each(["stamp", "active", "paused", "revoked", "persistent_provider", "enabled_connection", "active_connection",
|
||||
"non_sdk_connection", "credentials", "malformed_credentials", "malformed_setup", "missing_generation", "fractional_generation",
|
||||
"lease", "expired_lease", "pending_registration", "unconsumed_registration", "wrong_registry"])(
|
||||
"keeps incomplete removed GitHub proof blocked: %s", async fault => {
|
||||
const f = await removedGitHubFixture();
|
||||
if (fault === "stamp") await db.update(chatEndpoints).set({ archivedAt: null }).where(eq(chatEndpoints.id, f.endpointId));
|
||||
if (["active", "paused", "revoked"].includes(fault)) await db.update(chatEndpoints)
|
||||
.set({ status: fault as "active" | "paused" | "revoked" }).where(eq(chatEndpoints.id, f.endpointId));
|
||||
if (fault === "persistent_provider") await db.update(chatEndpoints).set({ provider: "discord" }).where(eq(chatEndpoints.id, f.endpointId));
|
||||
if (fault === "enabled_connection") await db.update(toolConnections).set({ enabled: true }).where(eq(toolConnections.id, f.connectionId));
|
||||
if (fault === "active_connection") await db.update(toolConnections).set({ status: "active" }).where(eq(toolConnections.id, f.connectionId));
|
||||
if (fault === "non_sdk_connection") await db.update(toolConnections).set({ transport: "voice" }).where(eq(toolConnections.id, f.connectionId));
|
||||
if (fault === "credentials" || fault === "malformed_credentials") await db.execute(sql`UPDATE tool_connections
|
||||
SET credential_secret_refs = ${fault === "credentials" ? '[{"secretId":"retained"}]' : '{}'}::jsonb WHERE id = ${f.connectionId}::uuid`);
|
||||
if (fault === "malformed_setup") await db.execute(sql`UPDATE chat_endpoints SET setup = '"unknown"'::jsonb WHERE id = ${f.endpointId}::uuid`);
|
||||
if (fault === "missing_generation" || fault === "fractional_generation") await db.update(chatEndpoints)
|
||||
.set({ setup: fault === "missing_generation" ? { step: "complete" } : { step: "complete", runtimeGeneration: 1.5 } })
|
||||
.where(eq(chatEndpoints.id, f.endpointId));
|
||||
if (fault === "lease" || fault === "expired_lease") await db.insert(chatEndpointLeases).values({ companyId: f.companyId,
|
||||
endpointId: f.endpointId, leaseKey: "credentials", token: randomUUID(), expiresAt: new Date(fault === "lease" ? Date.now() + 60_000 : 0) });
|
||||
if (fault === "pending_registration" || fault === "unconsumed_registration") await db.insert(chatGitHubRegistrations).values({
|
||||
companyId: f.companyId, endpointId: f.endpointId, userId: f.userId, stateHash: randomUUID(), trustedOrigin: "https://tenant.example.test",
|
||||
status: fault === "pending_registration" ? "pending" : "completed", expiresAt: new Date(Date.now() + 60_000) });
|
||||
if (fault === "wrong_registry") await db.insert(chatSlackRegistrations).values({ endpointId: f.endpointId, companyId: f.companyId,
|
||||
requestId: randomUUID(), status: "uncertain", manifest: {}, manifestHash: "a".repeat(64), callbackUri: "https://tenant.example.test",
|
||||
createdByUserId: f.userId, secretIds: { clientSecret: randomUUID() } });
|
||||
expect(await f.blocked()).toBe(true);
|
||||
expect(await db.select({ id: chatEndpoints.id }).from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId))).toHaveLength(1);
|
||||
});
|
||||
it("retains tenant boundaries and rejects unfamiliar archived state at the schema boundary", async () => {
|
||||
const f = await removedGitHubFixture();
|
||||
await expect(db.execute(sql`UPDATE chat_endpoints SET connection_id = ${randomUUID()}::uuid WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow();
|
||||
const foreignApplication = randomUUID(), foreignConnection = randomUUID();
|
||||
await db.insert(toolApplications).values({ id: foreignApplication, companyId: f.foreignId, name: "Foreign", type: "chat" });
|
||||
await db.insert(toolConnections).values({ id: foreignConnection, companyId: f.foreignId, applicationId: foreignApplication,
|
||||
name: "Foreign", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel", status: "archived", enabled: false });
|
||||
await expect(db.execute(sql`UPDATE chat_endpoints SET connection_id = ${foreignConnection}::uuid WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow();
|
||||
await expect(db.execute(sql`UPDATE chat_endpoints SET status = 'unknown' WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow();
|
||||
expect((await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId)))[0].connectionId).toBe(f.connectionId);
|
||||
});
|
||||
it("does not let removed GitHub endpoint history discharge pending delivery, publication or action obligations", async () => {
|
||||
const f = await removedGitHubFixture();
|
||||
await db.insert(chatGitHubRegistrations).values({ companyId: f.companyId, endpointId: f.endpointId, userId: f.userId,
|
||||
stateHash: randomUUID(), trustedOrigin: "https://tenant.example.test", status: "completed", consumedAt: new Date(), expiresAt: new Date(0) });
|
||||
expect(await f.blocked()).toBe(false);
|
||||
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Retained chat debt", status: "done" }).returning();
|
||||
const [conversation] = await db.insert(chatConversations).values({ companyId: f.companyId, endpointId: f.endpointId,
|
||||
issueId: issue.id, externalConversationId: "github:retained", externalLabel: "retained", state: "endpoint_removed" }).returning();
|
||||
await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id,
|
||||
providerEventId: "retained", deduplicationKey: "retained", eventKind: "message", normalizedEvent: {}, state: "retry", nextAttemptAt: new Date() });
|
||||
await db.insert(chatPublications).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id,
|
||||
issueId: issue.id, idempotencyKey: "retained", payload: { text: "Retained" }, state: "retry", nextAttemptAt: new Date() });
|
||||
await db.insert(chatActions).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id,
|
||||
kind: "provider_effect", providerActionId: "retained", payload: {}, status: "received" });
|
||||
const report = await readIdleSleepSafety(db, hold, () => now, ownerId, async () => "none", undefined, undefined,
|
||||
false, false, { incomingWake: verified(), collectDiagnostics: true });
|
||||
expect(report.backgroundWork).toBe("present");
|
||||
expect(report.diagnostics?.checks).toEqual(expect.arrayContaining([
|
||||
{ reason: "chat_endpoints", state: "none" }, { reason: "chat_deliveries", state: "present" },
|
||||
{ reason: "chat_publications", state: "present" }, { reason: "chat_actions", state: "present" },
|
||||
]));
|
||||
expect(await db.select().from(chatDeliveries)).toHaveLength(1);
|
||||
expect(await db.select().from(chatPublications)).toHaveLength(1);
|
||||
expect(await db.select().from(chatActions)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("requires current broker instance and exact owner binding, preserving continuity and unsupported transport blockers", async () => {
|
||||
const f = await seed();
|
||||
const applicationId = randomUUID(),
|
||||
connectionId = randomUUID(),
|
||||
grantId = randomUUID();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1");
|
||||
const botConnectionId = randomUUID(),
|
||||
endpointId = randomUUID(),
|
||||
registration = "b".repeat(43);
|
||||
const botApplication = randomUUID();
|
||||
await db
|
||||
.insert(toolApplications)
|
||||
.values({
|
||||
id: botApplication,
|
||||
companyId: f.companyId,
|
||||
name: "GitHub bot",
|
||||
type: "chat",
|
||||
});
|
||||
await db
|
||||
.insert(toolConnections)
|
||||
.values({
|
||||
id: botConnectionId,
|
||||
companyId: f.companyId,
|
||||
applicationId: botApplication,
|
||||
name: "GitHub bot",
|
||||
uid: randomUUID(),
|
||||
transport: "chat_sdk",
|
||||
connectionPurpose: "channel",
|
||||
status: "active",
|
||||
enabled: true,
|
||||
config: { provider: "github" },
|
||||
});
|
||||
await db
|
||||
.insert(chatEndpoints)
|
||||
.values({
|
||||
id: endpointId,
|
||||
companyId: f.companyId,
|
||||
connectionId: botConnectionId,
|
||||
assignedAgentId: f.agentId,
|
||||
provider: "github",
|
||||
publicId: "p".repeat(43),
|
||||
status: "active",
|
||||
activatedAt: new Date(),
|
||||
providerAccountId: "fixture-app",
|
||||
botExternalId: "fixture-bot",
|
||||
setup: {
|
||||
step: "complete",
|
||||
github: {
|
||||
stage: "test",
|
||||
registrationStatus: "completed",
|
||||
cloudRegistrationId: registration,
|
||||
},
|
||||
},
|
||||
});
|
||||
await db
|
||||
.insert(chatGitHubRegistrations)
|
||||
.values({
|
||||
companyId: f.companyId,
|
||||
endpointId,
|
||||
userId: f.userId,
|
||||
stateHash: randomUUID(),
|
||||
trustedOrigin: "https://tenant.example.test",
|
||||
status: "completed",
|
||||
expiresAt: new Date(0),
|
||||
handoff: {
|
||||
cloudId: registration,
|
||||
redemptionId: randomUUID(),
|
||||
returnState: randomUUID(),
|
||||
},
|
||||
});
|
||||
const botTuple = [
|
||||
`github-app:${registration}`,
|
||||
f.companyId,
|
||||
f.userId,
|
||||
"github.bot",
|
||||
`gateway:${registration}`,
|
||||
registration,
|
||||
registration,
|
||||
];
|
||||
const stop = registerConnectionEventWake(db, () => {});
|
||||
const botCap: IdleIncomingWakeCapability = {
|
||||
...capability,
|
||||
routes: ["github-broker-events-v1", "tenant-api-http-v2"],
|
||||
githubBroker: {
|
||||
instanceId: "fixture-instance",
|
||||
bindingsFingerprint: incomingWakeFingerprint([botTuple]),
|
||||
},
|
||||
};
|
||||
expect((await read(verified())).backgroundWork).toBe("present");
|
||||
expect(await read(verified(botCap))).toMatchObject({
|
||||
version: 3,
|
||||
backgroundWork: "none",
|
||||
});
|
||||
const botProof = await readIdleIncomingWakeProof(db, {
|
||||
capabilities: verified(botCap),
|
||||
ownerId,
|
||||
receiverOwner: db,
|
||||
});
|
||||
expect(botProof?.endpointIds).toEqual([endpointId]);
|
||||
expect(botProof?.connectionIds).toEqual([botConnectionId]);
|
||||
expect(
|
||||
await db
|
||||
.select()
|
||||
.from(chatEndpoints)
|
||||
.where(eq(chatEndpoints.id, endpointId)),
|
||||
).toHaveLength(1);
|
||||
|
||||
await db
|
||||
.insert(toolApplications)
|
||||
.values({
|
||||
id: applicationId,
|
||||
companyId: f.companyId,
|
||||
name: "GitHub",
|
||||
type: "mcp_http",
|
||||
});
|
||||
await db
|
||||
.insert(toolConnections)
|
||||
.values({
|
||||
id: connectionId,
|
||||
companyId: f.companyId,
|
||||
applicationId,
|
||||
name: "GitHub",
|
||||
uid: randomUUID(),
|
||||
transport: "mcp_remote",
|
||||
connectionPurpose: "tool",
|
||||
status: "active",
|
||||
enabled: true,
|
||||
config: {
|
||||
sourceTemplateKey: "github",
|
||||
oauth: { connectorProfile: "github.code" },
|
||||
},
|
||||
});
|
||||
const github = {
|
||||
userId: "42",
|
||||
login: "fixture",
|
||||
installationCount: 1,
|
||||
repositoryCount: 1,
|
||||
repositorySelection: "all" as const,
|
||||
installationIds: ["123"],
|
||||
installationOwnerLogins: ["fixture"],
|
||||
lastAccessRefreshAt: new Date().toISOString(),
|
||||
};
|
||||
await db
|
||||
.insert(connectionGrants)
|
||||
.values({
|
||||
id: grantId,
|
||||
companyId: f.companyId,
|
||||
connectionId,
|
||||
kind: "user",
|
||||
subjectUserId: f.userId,
|
||||
providerTenant: { github },
|
||||
status: "active",
|
||||
});
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test");
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1");
|
||||
const brokerCap: IdleIncomingWakeCapability = {
|
||||
...capability,
|
||||
routes: ["github-broker-events-v1", "tenant-api-http-v2"],
|
||||
githubBroker: {
|
||||
instanceId: "fixture-instance",
|
||||
bindingsFingerprint: incomingWakeFingerprint([
|
||||
botTuple,
|
||||
[
|
||||
`${grantId}_123`,
|
||||
f.companyId,
|
||||
f.userId,
|
||||
"github.code",
|
||||
"123",
|
||||
connectionId,
|
||||
grantId,
|
||||
],
|
||||
]),
|
||||
},
|
||||
};
|
||||
try {
|
||||
const proof = await readIdleIncomingWakeProof(db, {
|
||||
capabilities: verified(brokerCap),
|
||||
ownerId,
|
||||
receiverOwner: db,
|
||||
});
|
||||
expect(proof?.requiredContinuityConnectionIds).toEqual([connectionId]);
|
||||
expect(proof?.connectionIds).toEqual([botConnectionId]);
|
||||
expect((await read(verified(brokerCap))).backgroundWork).toBe(
|
||||
"present",
|
||||
); // transport alone never discharges real credential deadline
|
||||
await db
|
||||
.update(connectionGrants)
|
||||
.set({
|
||||
providerTenant: {
|
||||
github: {
|
||||
...github,
|
||||
bindingSync: { id: randomUUID(), removeInstallationIds: [] },
|
||||
},
|
||||
},
|
||||
})
|
||||
.where(eq(connectionGrants.id, grantId));
|
||||
expect((await read(verified(brokerCap))).backgroundWork).toBe(
|
||||
"unknown",
|
||||
);
|
||||
await db
|
||||
.update(connectionGrants)
|
||||
.set({ status: "revoked" })
|
||||
.where(eq(connectionGrants.id, grantId));
|
||||
expect((await read(verified(brokerCap))).backgroundWork).toBe(
|
||||
"unknown",
|
||||
);
|
||||
await db
|
||||
.update(toolConnections)
|
||||
.set({ transport: "local_stdio" })
|
||||
.where(eq(toolConnections.id, connectionId));
|
||||
expect((await read(verified())).backgroundWork).toBe("present");
|
||||
} finally {
|
||||
stop();
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -285,11 +285,14 @@ describeEmbeddedPostgres("Mine inbox snapshot routes", () => {
|
||||
const responses = await Promise.all(requests);
|
||||
expect(responses.every(response => response.status === 200)).toBe(true);
|
||||
if (coalesced) expect((await coalesced).headers["x-paperclip-request-cache"]).toBe("coalesced");
|
||||
await request(app).get(snapshotPath).expect(200);
|
||||
const computesBeforeFreshRead = started.mock.calls.length;
|
||||
const fresh = await request(app).get(snapshotPath).expect(200);
|
||||
// The aborted snapshot can outlive the seven ordinary reads. A coalesced
|
||||
// response waits for that compute to settle before we require a fresh one.
|
||||
const settled = await request(app).get(snapshotPath).expect(200);
|
||||
const fresh = settled.headers["x-paperclip-request-cache"] === "coalesced"
|
||||
? await request(app).get(snapshotPath).expect(200)
|
||||
: settled;
|
||||
expect(fresh.headers["x-paperclip-request-cache"]).toBe("miss");
|
||||
expect(started).toHaveBeenCalledTimes(computesBeforeFreshRead + 1);
|
||||
expect(started).toHaveBeenCalledTimes(9);
|
||||
} finally {
|
||||
release();
|
||||
await aborted;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
||||
import express from "express";
|
||||
import request from "supertest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -81,21 +82,24 @@ function localAdminActor() {
|
||||
|
||||
describe("instance settings routes", () => {
|
||||
const routeModules = hoistModuleGraph(registerModuleMocks, async () => {
|
||||
const [{ errorHandler }, { instanceSettingsRoutes }] = await Promise.all([
|
||||
const [{ errorHandler }, { instanceSettingsRoutes }, { cloudControlMiddleware }, controlIdentity] = await Promise.all([
|
||||
vi.importActual<typeof import("../middleware/index.js")>("../middleware/index.js"),
|
||||
vi.importActual<typeof import("../routes/instance-settings.js")>("../routes/instance-settings.js"),
|
||||
vi.importActual<typeof import("../middleware/cloud-control.js")>("../middleware/cloud-control.js"),
|
||||
vi.importActual<typeof import("../services/cloud-runtime-identity.js")>("../services/cloud-runtime-identity.js"),
|
||||
]);
|
||||
return { errorHandler, instanceSettingsRoutes };
|
||||
return { errorHandler, instanceSettingsRoutes, cloudControlMiddleware, controlIdentity };
|
||||
});
|
||||
|
||||
function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise<boolean>) {
|
||||
const { errorHandler, instanceSettingsRoutes } = routeModules.value;
|
||||
function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise<boolean>, verifyControl = false) {
|
||||
const { errorHandler, instanceSettingsRoutes, cloudControlMiddleware } = routeModules.value;
|
||||
const app = express();
|
||||
app.use(express.json());
|
||||
app.use((req, _res, next) => {
|
||||
req.actor = actor;
|
||||
next();
|
||||
});
|
||||
if (verifyControl) app.use(cloudControlMiddleware());
|
||||
app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup));
|
||||
app.use(errorHandler);
|
||||
return app;
|
||||
@@ -1101,7 +1105,7 @@ describe("instance settings routes", () => {
|
||||
await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);
|
||||
expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });
|
||||
await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined });
|
||||
});
|
||||
|
||||
it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {
|
||||
@@ -1128,7 +1132,7 @@ describe("instance settings routes", () => {
|
||||
const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined });
|
||||
expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);
|
||||
});
|
||||
|
||||
@@ -1142,7 +1146,7 @@ describe("instance settings routes", () => {
|
||||
.get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId=fixture-owner").expect(200);
|
||||
expect(response.body.idleSleepSafety.diagnostics.checks).toEqual([{ reason: "agent_api_keys", state: "present" }]);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, false, false, { collectDiagnostics: true });
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined, collectDiagnostics: true });
|
||||
mockReadIdleSleepSafety.mockClear();
|
||||
for (const actor of [nonAdminActor, { type: "agent", agentId: "agent-1", companyId: "company-1", source: "api_key" }]) {
|
||||
await request(createApp(actor))
|
||||
@@ -1164,7 +1168,93 @@ describe("instance settings routes", () => {
|
||||
await request(createApp({ ...adminActor, source: "cloud_control" }))
|
||||
.get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, true, version === "2");
|
||||
"fixture-owner", expect.any(Function), undefined, undefined, true, version === "2", { incomingWake: undefined, deadlineCapability: undefined });
|
||||
});
|
||||
|
||||
describe("signed incoming capability handoff", () => {
|
||||
const createControlApp = (actor: any) => createApp(actor, undefined, undefined, true);
|
||||
const keyPair = generateKeyPairSync("ed25519");
|
||||
const publicJwk = { ...keyPair.publicKey.export({ format: "jwk" }), kid: "idle-route-test", alg: "EdDSA", use: "sig" };
|
||||
const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895";
|
||||
const catalog = {
|
||||
incomingWake: { version: 1, ownerId, generation: "a".repeat(64), apiKeyProof: { version: 1, challenge: "c".repeat(64) }, routes: ["tenant-api-http-v2"] },
|
||||
deadlineCatalog: "domain_deadlines_v1",
|
||||
};
|
||||
const path = `/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`;
|
||||
function assertion(claims: Record<string, unknown> = {}) {
|
||||
const identity = routeModules.value.controlIdentity;
|
||||
const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
const iat = Math.floor(Date.now() / 1000);
|
||||
const header = encode({ alg: "EdDSA", typ: identity.CLOUD_CONTROL_JWS_TYPE, kid: publicJwk.kid });
|
||||
const payload = encode({ v: 1, iss: identity.CLOUD_RUNTIME_IDENTITY_ISSUER, aud: identity.CLOUD_CONTROL_AUDIENCE,
|
||||
sub: "stack-idle-route-test", action: "task-drain:read", requestId: randomUUID(), iat, exp: iat + 60,
|
||||
idleCapabilities: catalog, ...claims });
|
||||
return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, "ascii"), keyPair.privateKey).toString("base64url")}`;
|
||||
}
|
||||
beforeEach(() => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS", JSON.stringify({ keys: [publicJwk] }));
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-idle-route-test");
|
||||
routeModules.value.controlIdentity.resetCloudControlReplayFenceForTests();
|
||||
mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
|
||||
mockReadIdleSleepSafety.mockResolvedValue({ version: 3, backgroundWork: "none" });
|
||||
});
|
||||
afterEach(() => vi.unstubAllEnvs());
|
||||
|
||||
it("passes only the immutable signed catalog through real middleware and the production route", async () => {
|
||||
const token = assertion();
|
||||
const response = await request(createControlApp({ type: "none" }))
|
||||
.get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify({ ...catalog.incomingWake, generation: "b".repeat(64) }))}`)
|
||||
.set("x-paperclip-cloud-control", token)
|
||||
.send({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, ownerId: randomUUID() } } }).expect(200);
|
||||
expect(response.body.idleSleepSafety).toEqual({ version: 3, backgroundWork: "none" });
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledExactlyOnceWith(mockDb, expect.any(Function), Date.now,
|
||||
ownerId, expect.any(Function), undefined, undefined, false, false,
|
||||
{ incomingWake: catalog, deadlineCapability: catalog });
|
||||
const options = mockReadIdleSleepSafety.mock.calls[0][9];
|
||||
expect(options.incomingWake).toBe(options.deadlineCapability);
|
||||
expect(Object.isFrozen(options.incomingWake)).toBe(true);
|
||||
expect(Object.isFrozen(options.incomingWake.incomingWake)).toBe(true);
|
||||
expect(Object.isFrozen(options.incomingWake.incomingWake.routes)).toBe(true);
|
||||
await request(createControlApp({ type: "none" })).get(path).set("x-paperclip-cloud-control", token).expect(401);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["wrong stack", () => ({ sub: "other-stack" })],
|
||||
["wrong action", () => ({ action: "task-drain:start" })],
|
||||
["expired", () => ({ iat: Math.floor(Date.now() / 1000) - 600, exp: Math.floor(Date.now() / 1000) - 300 })],
|
||||
["unsupported catalog", () => ({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, routes: ["unverified-path"] } } })],
|
||||
])("rejects %s before safety inspection", async (_name, claims) => {
|
||||
await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", assertion(claims())).expect(401);
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a tampered catalog signature before safety inspection", async () => {
|
||||
const [header, payload, signature] = assertion().split(".");
|
||||
const changed = JSON.parse(Buffer.from(payload!, "base64url").toString());
|
||||
changed.idleCapabilities.incomingWake.generation = "b".repeat(64);
|
||||
const tampered = `${header}.${Buffer.from(JSON.stringify(changed)).toString("base64url")}.${signature}`;
|
||||
await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", tampered).expect(401);
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("leaves legacy signed reads without a catalog conservative", async () => {
|
||||
await request(createControlApp({ type: "none" })).get(path)
|
||||
.set("x-paperclip-cloud-control", assertion({ idleCapabilities: undefined })).expect(200);
|
||||
expect(mockReadIdleSleepSafety.mock.calls[0][9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined });
|
||||
});
|
||||
|
||||
it("cannot install capabilities from tenant, actor, query or body values", async () => {
|
||||
for (const source of ["session", "cloud_tenant", "local_implicit", "cloud_control"]) {
|
||||
const actor = { ...adminActor, source, idleCapabilities: catalog, incomingWake: catalog };
|
||||
await request(createControlApp(actor)).get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify(catalog))}`)
|
||||
.send({ idleCapabilities: catalog }).expect(200);
|
||||
expect(mockReadIdleSleepSafety.mock.lastCall?.[9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined });
|
||||
}
|
||||
mockReadIdleSleepSafety.mockClear();
|
||||
await request(createControlApp(nonAdminActor)).get(path).send({ idleCapabilities: catalog }).expect(403);
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the live worker manager for inspection and releases only the owned hold", async () => {
|
||||
@@ -1199,7 +1289,7 @@ describe("instance settings routes", () => {
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false);
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined });
|
||||
mockReadIdleSleepSafety.mockClear();
|
||||
await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);
|
||||
expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
|
||||
@@ -1221,7 +1311,7 @@ describe("instance settings routes", () => {
|
||||
const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup);
|
||||
await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
|
||||
expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false);
|
||||
"fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined });
|
||||
} finally {
|
||||
delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN;
|
||||
}
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
import { agents, agentWakeupRequests, companies, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { recoveryService } from "../services/recovery/service.js";
|
||||
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
|
||||
import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapters/postgres.js";
|
||||
import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js";
|
||||
import { getExecutionBlocker } from "../services/execution-blocker.js";
|
||||
|
||||
describe("legacy continuation persisted authority", () => {
|
||||
let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
@@ -33,6 +36,103 @@ describe("legacy continuation persisted authority", () => {
|
||||
}
|
||||
return { companyId, agentId, issueId, runId, createRecovery, runs, actions, finish };
|
||||
}
|
||||
it("does not freeze a never-started dispatch cancellation or retain its historical hold", async () => {
|
||||
const f = await fixture();
|
||||
await db.update(heartbeatRuns).set({ status: "queued", resultJson: null }).where(eq(heartbeatRuns.id, f.runId));
|
||||
await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId));
|
||||
expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId: f.companyId,
|
||||
runId: f.runId, expectedStatus: "queued", now: new Date() })).toMatchObject({ outcome: "cancelled", errorCode: "issue_assignee_changed" });
|
||||
const run = (await f.runs())[0];
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(false);
|
||||
await terminalizeLegacyExecution({ db, run, status: "cancelled", fromStatuses: ["cancelled"], reconcileIfNeeded: true });
|
||||
expect(await f.actions()).toHaveLength(0);
|
||||
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board",
|
||||
fingerprint: `legacy-execution:${run.id}`, status: "resolved", outcome: "blocked", nextAction: "Reconcile",
|
||||
evidence: { runId: run.id, automaticRecovery: { replay: "blocked" } } });
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
for (const errorCode of ["issue_not_found", "issue_not_in_progress", "issue_waiting_for_response",
|
||||
"issue_dependencies_blocked", "issue_blocked",
|
||||
"issue_execution_lock_changed", "issue_terminal_status", "issue_review_participant_changed"]) {
|
||||
const changed = { ...run, errorCode };
|
||||
await db.update(heartbeatRuns).set(changed).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, changed)).toBe(false);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
}
|
||||
const reviewWait = { ...run, errorCode: "issue_continuation_waiting_on_review", resultJson: {
|
||||
stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate",
|
||||
} };
|
||||
await db.update(issues).set({ assigneeAgentId: f.agentId }).where(eq(issues.id, f.issueId));
|
||||
await db.update(heartbeatRuns).set(reviewWait).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, reviewWait)).toBe(false);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ canRetry: true });
|
||||
for (const patch of [{ finishedAt: null }, { startedAt: new Date() }, { runtimeModeResolvedAt: new Date() },
|
||||
{ processPid: 123456789 }, { processGroupId: 123456789 }, { processStartedAt: new Date() },
|
||||
{ sessionIdAfter: "provider-session" }, { controllerBootId: randomUUID() },
|
||||
{ controllerLeaseExpiresAt: new Date() }, { executionStage: "adapter_running" }, { errorCode: "cancelled" },
|
||||
{ resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } },
|
||||
{ resultJson: { workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1" } } },
|
||||
...["nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace", "nativeWorkspaceFinalizationOwner", "sessionCheckpoint"]
|
||||
.map(key => ({ runnerProfileJson: { [key]: { retained: true } } }))]) {
|
||||
const changed = { ...run, ...patch };
|
||||
await db.update(heartbeatRuns).set(changed).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, changed)).toBe(true);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
||||
}
|
||||
await db.update(heartbeatRuns).set(run).where(eq(heartbeatRuns.id, run.id));
|
||||
await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: run.id, agentId: f.agentId,
|
||||
seq: 2, eventType: "adapter.invoke" });
|
||||
await db.update(heartbeatRuns).set({ nextEventSeq: 3 }).where(eq(heartbeatRuns.id, run.id));
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
||||
});
|
||||
it("keeps historical holds for missing, foreign, or conflicting sources and workspace recovery", async () => {
|
||||
const f = await fixture(), foreign = await fixture();
|
||||
await db.update(issues).set({ status: "done" }).where(eq(issues.id, foreign.issueId));
|
||||
await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(),
|
||||
errorCode: "issue_dependencies_blocked", resultJson: null }).where(eq(heartbeatRuns.id, f.runId));
|
||||
const evidence = { runId: f.runId, automaticRecovery: { replay: "blocked" } };
|
||||
const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board",
|
||||
fingerprint: `legacy-execution:${f.runId}`, status: "resolved", outcome: "blocked", nextAction: "Reconcile", evidence }).returning();
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
for (const changed of [{ ...evidence, runId: "malformed" }, { ...evidence, runId: randomUUID() },
|
||||
{ ...evidence, runId: foreign.runId }, { automaticRecovery: evidence.automaticRecovery },
|
||||
{ ...evidence, workspaceRestoreFailure: "restore_unsafe_archive" },
|
||||
{ ...evidence, workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1" } }]) {
|
||||
await db.update(issueRecoveryActions).set({ evidence: changed }).where(eq(issueRecoveryActions.id, hold.id));
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
||||
}
|
||||
await db.update(issueRecoveryActions).set({ evidence: { sourceRunId: f.runId,
|
||||
automaticRecovery: evidence.automaticRecovery } }).where(eq(issueRecoveryActions.id, hold.id));
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Another task" }).returning();
|
||||
await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: other.id } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
||||
await db.update(heartbeatRuns).set({ issueId: f.issueId,
|
||||
contextSnapshot: { issueId: f.issueId } }).where(eq(heartbeatRuns.id, f.runId));
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
expect(await f.actions()).toHaveLength(1);
|
||||
});
|
||||
it.each(["released environment", "provider event"])("retains a hold with a recorded %s", async retained => {
|
||||
const f = await fixture();
|
||||
const [run] = await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(),
|
||||
errorCode: "issue_dependencies_blocked", resultJson: null }).where(eq(heartbeatRuns.id, f.runId)).returning();
|
||||
await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId,
|
||||
kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board",
|
||||
fingerprint: `legacy-execution:${f.runId}`, status: "active", nextAction: "Reconcile", evidence: { runId: f.runId } });
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(false);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull();
|
||||
if (retained === "released environment") await db.insert(environmentLeases).values({ companyId: f.companyId,
|
||||
heartbeatRunId: f.runId, issueId: f.issueId, provider: "local", status: "released", releasedAt: new Date(), cleanupStatus: "success" });
|
||||
else {
|
||||
await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.runId, agentId: f.agentId,
|
||||
seq: 1, eventType: "provider.diagnostic", sourceEventId: "provider-event-1" });
|
||||
await db.update(heartbeatRuns).set({ nextEventSeq: 2 }).where(eq(heartbeatRuns.id, f.runId));
|
||||
}
|
||||
expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true);
|
||||
expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull();
|
||||
});
|
||||
it.each([1, 2, 3, 4, 5])("deduplicates concurrent replay with stale prose-derived liveness (%s)", async () => {
|
||||
const f = await fixture();
|
||||
const recovery = f.createRecovery();
|
||||
|
||||
@@ -8,6 +8,7 @@ import { describe, expect, it, vi } from "vitest";
|
||||
import { setupLiveEventsWebSocketServer } from "../realtime/live-events-ws.js";
|
||||
import { publishLiveEvent } from "../services/live-events.js";
|
||||
import { taskToastEligible } from "../realtime/task-toast-eligibility.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
vi.mock("../realtime/task-toast-eligibility.js", () => ({ taskToastEligible: vi.fn(async () => false) }));
|
||||
|
||||
@@ -29,7 +30,7 @@ function database(onPredicate: (kind: "where" | "join", condition: SQL) => void
|
||||
innerJoin: (_table: unknown, condition: SQL) => { onPredicate("join", condition); return query; },
|
||||
where: (condition: SQL) => { onPredicate("where", condition); return query; },
|
||||
limit: async () => {
|
||||
if (table === "issues") return [{ id: "task" }];
|
||||
if (table === "issues") return [{ id: "task", identifier: "PAP-1234" }];
|
||||
if (table === "company_memberships") return fields.name ? [{ name: "Alex Example", image: "/alex.png" }] : [{ id: "membership" }];
|
||||
if (table === "agents") return [{ name: "Coder", appearance: { schemaVersion: 1, characterVersion: "cap-v1", paletteId: "bubblegum-sky" } }];
|
||||
return [];
|
||||
@@ -39,6 +40,102 @@ function database(onPredicate: (kind: "where" | "join", condition: SQL) => void
|
||||
} };
|
||||
}
|
||||
describe("authenticated activity notifications", () => {
|
||||
it("retains live delivery idle work until the identifier read settles after the last socket closes", async () => {
|
||||
const baseline = idleWorkSnapshot().active;
|
||||
let release!: () => void, entered!: () => void;
|
||||
const pending = new Promise<void>(resolve => { release = resolve; });
|
||||
const started = new Promise<void>(resolve => { entered = resolve; });
|
||||
const db = database(), select = db.select;
|
||||
db.select = fields => {
|
||||
const query = select(fields);
|
||||
if (fields.identifier) {
|
||||
const limit = query.limit;
|
||||
query.limit = async () => { entered(); await pending; return limit(); };
|
||||
}
|
||||
return query;
|
||||
};
|
||||
const server = createServer();
|
||||
const wss = setupLiveEventsWebSocketServer(server, db as never, { deploymentMode: "local_trusted" });
|
||||
const closed = new Promise<void>(resolve => wss.on("connection", socket => socket.on("close", resolve)));
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as { port: number }).port}/api/companies/company/events/ws`);
|
||||
try {
|
||||
await once(socket, "open");
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline));
|
||||
// System activity has no actor lookup: this gate is the new identifier SQL.
|
||||
publishLiveEvent({ companyId: "company", type: "activity.logged", payload: {
|
||||
action: "issue.updated", entityType: "issue", entityId: "visible", actorType: "system",
|
||||
} });
|
||||
await started;
|
||||
socket.terminate();
|
||||
await closed;
|
||||
expect(wss.clients.size).toBe(0);
|
||||
expect(idleWorkSnapshot().active).toBe(baseline + 1);
|
||||
release();
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline));
|
||||
} finally {
|
||||
release();
|
||||
socket.terminate();
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline));
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["local_trusted", "authenticated"] as const)("keeps authorized notifications when optional identifier reads fail (%s)", async deploymentMode => {
|
||||
const db = database();
|
||||
const select = db.select;
|
||||
const identifierRead = vi.fn();
|
||||
db.select = fields => {
|
||||
if (fields.identifier) {
|
||||
identifierRead();
|
||||
if (identifierRead.mock.calls.length <= 3) throw new Error("Identifier lookup unavailable");
|
||||
}
|
||||
return select(fields);
|
||||
};
|
||||
const server = createServer();
|
||||
setupLiveEventsWebSocketServer(server, db as never, { deploymentMode,
|
||||
resolveCloudActor: async () => ({ userId: "viewer", companyIds: ["company"] }) });
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as { port: number }).port}/api/companies/company/events/ws`);
|
||||
try {
|
||||
await once(socket, "open");
|
||||
const messages: Array<{ payload: Record<string, unknown> }> = [];
|
||||
const delivered = new Promise<void>(resolve => socket.on("message", data => {
|
||||
const event = JSON.parse(data.toString());
|
||||
messages.push(event);
|
||||
if (event.payload.entityId === "after-failure") resolve();
|
||||
}));
|
||||
const activity = (entityId: string, actorType: string) => publishLiveEvent({
|
||||
companyId: "company", type: "activity.logged", payload: {
|
||||
action: "issue.updated", entityType: "issue", entityId, actorType,
|
||||
actorId: actorType === "user" ? "viewer" : actorType === "agent" ? "agent" : null,
|
||||
issueIdentifier: "SPOOF-1", details: { secret: "private task details" },
|
||||
},
|
||||
});
|
||||
if (deploymentMode === "authenticated") activity("hidden", "system");
|
||||
for (const actorType of ["system", "agent", "user"]) {
|
||||
activity("visible", actorType);
|
||||
}
|
||||
activity("after-failure", "system");
|
||||
await delivered;
|
||||
expect(messages).toHaveLength(4);
|
||||
expect(identifierRead).toHaveBeenCalledTimes(4);
|
||||
for (const [index, actorType] of ["system", "agent", "user"].entries()) {
|
||||
const event = messages[index];
|
||||
expect(event.payload).toMatchObject({ entityId: "visible", actorType, issueIdentifier: null });
|
||||
if (actorType === "agent") expect(event.payload.actorName).toBe("Coder");
|
||||
if (actorType === "user") expect(event.payload.actorName).toBe("Alex Example");
|
||||
if (deploymentMode === "authenticated") expect(event.payload).not.toHaveProperty("details");
|
||||
}
|
||||
expect(messages[3].payload.issueIdentifier).toBe("PAP-1234");
|
||||
} finally {
|
||||
socket.terminate();
|
||||
await new Promise<void>(resolve => server.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves only an authorized handoff task identity for empty case-panel refresh", async () => {
|
||||
const server = createServer();
|
||||
setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode: "authenticated",
|
||||
@@ -88,6 +185,7 @@ describe("authenticated activity notifications", () => {
|
||||
} });
|
||||
const [data] = await delivery;
|
||||
expect(JSON.parse(data.toString()).toastEligible).toBe(true);
|
||||
expect(JSON.parse(data.toString()).payload.issueIdentifier).toBe("PAP-1234");
|
||||
expect(taskToastEligible).toHaveBeenLastCalledWith(expect.anything(), expect.anything(), "local-board");
|
||||
} finally {
|
||||
socket.terminate();
|
||||
@@ -123,6 +221,8 @@ describe("authenticated activity notifications", () => {
|
||||
expect(b.toastEligible).toBe(false);
|
||||
expect(a.payload.entityId).toBe(entityId);
|
||||
expect(b.payload.entityId).toBe(entityId);
|
||||
expect(a.payload.issueIdentifier).toBe("PAP-1234");
|
||||
expect(b.payload.issueIdentifier).toBe("PAP-1234");
|
||||
expect(original).not.toHaveProperty("toastEligible");
|
||||
}
|
||||
} finally {
|
||||
@@ -132,9 +232,9 @@ describe("authenticated activity notifications", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps actor identity after authorization and excludes hidden tasks and private details", async () => {
|
||||
it.each(["authenticated", "local_trusted"] as const)("keeps task and actor identity for %s without exposing hidden tasks or copying private details", async (deploymentMode) => {
|
||||
const server = createServer();
|
||||
setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode: "authenticated",
|
||||
setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode,
|
||||
resolveCloudActor: async () => ({ userId: "viewer", companyIds: ["company"] }) });
|
||||
server.listen(0, "127.0.0.1");
|
||||
await once(server, "listening");
|
||||
@@ -145,19 +245,21 @@ describe("authenticated activity notifications", () => {
|
||||
socket.on("message", (data) => messages.push(JSON.parse(data.toString())));
|
||||
const activity = (entityId: string, actorType = "user") => publishLiveEvent({ companyId: "company", type: "activity.logged", payload: {
|
||||
action: "issue.created", entityType: "issue", entityId, actorType, actorId: actorType === "user" ? "viewer" : "agent",
|
||||
issueIdentifier: "SPOOF-1",
|
||||
details: { bodySnippet: "private comment", secret: "not for the socket" },
|
||||
} });
|
||||
activity("hidden");
|
||||
if (deploymentMode === "authenticated") activity("hidden");
|
||||
const first = once(socket, "message");
|
||||
activity("visible");
|
||||
await first;
|
||||
expect(messages).toHaveLength(1);
|
||||
expect(messages[0].payload).toEqual({ action: "issue.created", entityType: "issue", entityId: "visible", actorType: "user", actorId: "viewer", actorName: "Alex Example", actorImage: "/alex.png" });
|
||||
expect(messages[0].payload).toMatchObject({ action: "issue.created", entityType: "issue", entityId: "visible", issueIdentifier: "PAP-1234", actorType: "user", actorId: "viewer", actorName: "Alex Example", actorImage: "/alex.png" });
|
||||
if (deploymentMode === "authenticated") expect(messages[0].payload).not.toHaveProperty("details");
|
||||
const second = once(socket, "message");
|
||||
activity("visible", "agent");
|
||||
await second;
|
||||
expect(messages[1].payload).toMatchObject({ actorName: "Coder", actorType: "agent", actorId: "agent", actorAppearance: { paletteId: "bubblegum-sky" } });
|
||||
expect(messages[1].payload).not.toHaveProperty("details");
|
||||
expect(messages[1].payload).toMatchObject({ issueIdentifier: "PAP-1234", actorName: "Coder", actorType: "agent", actorId: "agent", actorAppearance: { paletteId: "bubblegum-sky" } });
|
||||
if (deploymentMode === "authenticated") expect(messages[1].payload).not.toHaveProperty("details");
|
||||
} finally {
|
||||
socket.terminate();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
@@ -167,6 +269,20 @@ describe("authenticated activity notifications", () => {
|
||||
|
||||
|
||||
describe("activity profile query boundaries", () => {
|
||||
it("reads the task identifier within its company even when there is no actor", async () => {
|
||||
const predicates: { sql: string; params: unknown[] }[] = [];
|
||||
const db = database((_kind, condition) => {
|
||||
const { sql, params } = new PgDialect().sqlToQuery(condition);
|
||||
predicates.push({ sql, params });
|
||||
});
|
||||
expect(await activityToastIdentity(db as never, "company-A", {
|
||||
action: "issue.updated", entityType: "issue", entityId: "task-A", actorType: "system",
|
||||
})).toMatchObject({ issueIdentifier: "PAP-1234" });
|
||||
expect(predicates).toEqual([
|
||||
{ sql: '("issues"."company_id" = $1 and "issues"."id" = $2)', params: ["company-A", "task-A"] },
|
||||
]);
|
||||
});
|
||||
|
||||
it("requires the exact company, human actor, and active membership before reading a person profile", async () => {
|
||||
const predicates: { kind: string; sql: string; params: unknown[] }[] = [];
|
||||
const db = database((kind, condition) => {
|
||||
|
||||
@@ -32,8 +32,14 @@ import { terminalizeLegacyExecution } from "../services/legacy-execution-recover
|
||||
import { issueService } from "../services/issues.js";
|
||||
import { nativeRetryCancellationEligible, pendingFailedNativeRetryCancellationCondition } from "../services/native-runtime/native-cancellation-request.js";
|
||||
import { requestNativeQuestionRunCancellation } from "../services/native-runtime/native-question-bridge.js";
|
||||
import { readNativeLocalProcessStop } from "../services/native-local-process-stop.js";
|
||||
import { readNativeLocalProcessStop, releaseSettledNativeRunProcess, settleNativeRunProcess } from "../services/native-local-process-stop.js";
|
||||
import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js";
|
||||
import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js";
|
||||
import * as nativeStopEvidence from "../services/acknowledged-native-stop.js";
|
||||
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
|
||||
import { subscribeCompanyLiveEvents } from "../services/live-events.js";
|
||||
import type { LiveEvent } from "@paperclipai/shared";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
vi.setConfig({ testTimeout: 90_000 });
|
||||
@@ -295,6 +301,115 @@ describe("residual recovery production matrix", () => {
|
||||
}, { timeout: 45_000 });
|
||||
await competing.stop(); extraStops.pop(); await quiet();
|
||||
});
|
||||
it.each(["confirmed", "already_retired", "missing_terminal", "unproven_terminal", "ack_failure", "late_terminal"] as const)(
|
||||
"settles an API-only Stop after deferred controller retirement without inventing proof: %s", async mode => {
|
||||
const f = await fixture(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID();
|
||||
const statusEvents: LiveEvent[] = [];
|
||||
const unsubscribe = subscribeCompanyLiveEvents(f.companyId, event => {
|
||||
if (event.type === "heartbeat.run.status" && event.payload.runId === runId) statusEvents.push(event);
|
||||
});
|
||||
extraStops.push(async () => unsubscribe());
|
||||
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" });
|
||||
await once(child, "spawn");
|
||||
const stopChild = async () => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
const exited = once(child, "exit"); child.kill("SIGTERM"); await exited;
|
||||
};
|
||||
extraStops.push(stopChild);
|
||||
const recordedAt = new Date().toISOString();
|
||||
const nativeExecutionInput = buildNativeExecutionInput({ companyId: f.companyId, agentId: f.agent.id, runId,
|
||||
provider: "cursor_cloud", model: "explicit-model", cursorCloud: { schema: "paperclip.cursor-cloud.config.v1",
|
||||
repos: [{ url: "https://github.com/example/qualification" }] },
|
||||
issue: { id: f.issue.id, identifier: "API-1", title: f.issue.title, description: null, workMode: "standard" },
|
||||
taskPrompt: "Stop the foreground command", normalizedSessionId: sessionId,
|
||||
workspace: { id: randomUUID(), cwd: "/private/controller", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
|
||||
contract: { revision: "1", objective: "Stop the foreground command", criteria: [{ id: "stop", requirement: "Stop the foreground command" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture(),
|
||||
});
|
||||
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agent.id,
|
||||
runtimeMode: "native", driverKind: "cursor_cloud_api", nativeIssueId: f.issue.id, nativeSessionId: sessionId,
|
||||
runnerInstanceId: runnerId, status: "cancelled", finishedAt: new Date(),
|
||||
runnerProfileJson: { backend: "cursor_cloud_api", nativeExecutionInput, sessionCheckpoint: { providerSessionId: "cloud-agent" },
|
||||
executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } },
|
||||
resultJson: { executionCancellation: { state: "requested", requestedAt: recordedAt },
|
||||
cancelledByActorType: "user", cancelledByUserId: f.actorId, nativeCancellation: {
|
||||
schema: "paperclip.native-cancellation.v1", companyId: f.companyId, runId, issueId: f.issue.id,
|
||||
scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true,
|
||||
effects: ["release_run_resources"], recordedAt, acknowledgedAt: recordedAt,
|
||||
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() } } });
|
||||
await persistHeartbeatRunProcessMetadata(db, runId, { pid: child.pid!, processGroupId: child.pid!, startedAt: recordedAt, targetKind: "local" });
|
||||
await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId, phase: "committed" });
|
||||
const appendTerminal = async (target: typeof db = db) => {
|
||||
const terminal = { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "turn.interrupted",
|
||||
sourceKind: "runner", sourceInstanceId: runnerId, sourceEventId: `${runnerId}:${runId}:1`, sourceSeq: 1,
|
||||
runId, turnId: "stopped-turn", normalizedSessionId: sessionId, emittedAt: new Date().toISOString(), priority: 1,
|
||||
payload: { status: "interrupted", error: null, ...(mode === "unproven_terminal" ? { providerTerminalObserved: false } : {}) } };
|
||||
await appendHeartbeatRunEvent(target, { companyId: f.companyId, agentId: f.agent.id, runId,
|
||||
eventType: terminal.eventType, payload: { prpEvent: terminal }, nativeSource: {
|
||||
sourceInstanceId: runnerId, sourceEventId: terminal.sourceEventId, sourceSeq: 1,
|
||||
protocolSchemaVersion: 1, canonicalPayload: terminal } });
|
||||
};
|
||||
if (mode !== "missing_terminal" && mode !== "late_terminal") await appendTerminal();
|
||||
expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, runId))).toEqual([]);
|
||||
expect(await settleNativeRunProcess(db, runId)).toBe("pending");
|
||||
expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" });
|
||||
expect(statusEvents).toEqual([]);
|
||||
if (mode === "already_retired") { await stopChild(); expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
|
||||
await start();
|
||||
if (mode !== "already_retired") {
|
||||
await vi.waitFor(() => expect(deadlines.get(DELIVERY_QUEUES.settledNativeProcesses)?.()).toBeGreaterThan(Date.now()));
|
||||
expect((await getRun(runId)).processPid).toBe(child.pid);
|
||||
await stopChild();
|
||||
}
|
||||
const failure = new Error("synthetic Stop proof read failure after controller retirement");
|
||||
if (mode === "ack_failure") vi.spyOn(nativeStopEvidence, "acknowledgedNativeApiStopExecutionHasStopped").mockRejectedValueOnce(failure);
|
||||
await advance(31_000);
|
||||
if (mode === "ack_failure") {
|
||||
expect((await getRun(runId)).processPid).toBeNull();
|
||||
expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" });
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0]).toBeInstanceOf(AggregateError);
|
||||
expect((errors[0] as AggregateError).errors).toEqual([failure]); errors.length = 0;
|
||||
expect(statusEvents).toEqual([]);
|
||||
await advance(31_000);
|
||||
}
|
||||
const stopped = await getRun(runId);
|
||||
expect(stopped).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null });
|
||||
expect(await readNativeLocalProcessStop(db, f.companyId, runId)).toMatchObject({ processPid: child.pid, processGroupId: child.pid });
|
||||
expect(stopped.resultJson?.executionCancellation).toMatchObject(["missing_terminal", "unproven_terminal", "late_terminal"].includes(mode)
|
||||
? { state: "requested" } : { state: "acknowledged", proof: "provider_api_stop_receipt" });
|
||||
expect(statusEvents).toHaveLength(["missing_terminal", "unproven_terminal", "late_terminal"].includes(mode) ? 0 : 1);
|
||||
await quiet();
|
||||
if (mode === "late_terminal") {
|
||||
const rollback = new Error("rollback the uncommitted terminal receipt");
|
||||
await expect(db.transaction(async tx => { await appendTerminal(tx as unknown as typeof db); throw rollback; })).rejects.toBe(rollback);
|
||||
await advance(1_000);
|
||||
expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" });
|
||||
expect(statusEvents).toEqual([]);
|
||||
await quiet();
|
||||
let commit!: () => void, entered!: () => void;
|
||||
const gate = new Promise<void>(resolve => { commit = resolve; });
|
||||
const recorded = new Promise<void>(resolve => { entered = resolve; });
|
||||
const transaction = db.transaction(async tx => { await appendTerminal(tx as unknown as typeof db); entered(); await gate; });
|
||||
await recorded;
|
||||
try {
|
||||
await vi.advanceTimersByTimeAsync(500); await immediate();
|
||||
expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" });
|
||||
expect(statusEvents).toEqual([]);
|
||||
expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId),
|
||||
eq(heartbeatRunEvents.eventType, "turn.interrupted")))).toEqual([]);
|
||||
} finally { commit(); await transaction; }
|
||||
await vi.waitFor(async () => expect((await getRun(runId)).resultJson?.executionCancellation)
|
||||
.toMatchObject({ state: "acknowledged", proof: "provider_api_stop_receipt" }));
|
||||
await quiet();
|
||||
expect(statusEvents).toHaveLength(1);
|
||||
expect(statusEvents[0]).toMatchObject({ companyId: f.companyId, type: "heartbeat.run.status",
|
||||
payload: { runId, agentId: f.agent.id, status: "cancelled" } });
|
||||
expect(statusEvents[0]!.payload).not.toHaveProperty("resultJson");
|
||||
}
|
||||
},
|
||||
);
|
||||
it("releases a settled native process only after matching host and birth evidence arrives", async () => {
|
||||
const f = await fixture(), startedAt = new Date();
|
||||
const child = spawn(process.execPath, ["-e", "console.log('identity-proof')"], { stdio: "ignore" });
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import * as wakeSettlementNotifications from "../services/external-conversation-settlement-notifications.js";
|
||||
import { heartbeatService } from "../services/heartbeat.js";
|
||||
import { instanceSettingsService } from "../services/instance-settings.js";
|
||||
import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js";
|
||||
@@ -8,14 +9,14 @@ import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db";
|
||||
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
|
||||
import { nativeRunProjectionWork } from "../services/native-runtime/native-run-projection-work.js";
|
||||
import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js";
|
||||
import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js";
|
||||
import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js";
|
||||
import { finalizeNativeRun, recordNativeFinalizationFailure } from "../services/native-runtime/native-run-finalizer.js";
|
||||
import * as statusCommitter from "../services/native-runtime/status-decision-committer.js";
|
||||
import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js";
|
||||
import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js";
|
||||
@@ -120,6 +121,154 @@ describe("committed native run projection lifecycle", () => {
|
||||
async function accepted(status: "succeeded" | "failed" | "cancelled" = "succeeded") {
|
||||
const binding = await seed(); await complete(binding, "control_plane", db, status); await barrier(binding); return binding;
|
||||
}
|
||||
async function claimedWake(binding: Awaited<ReturnType<typeof seed>>) {
|
||||
const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "issue",
|
||||
status: "claimed", runId: binding.runId, claimedAt: new Date(), payload: { issueId: binding.issueId } }).returning();
|
||||
await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, binding.runId));
|
||||
return wake;
|
||||
}
|
||||
const readWake = async (id: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)))[0];
|
||||
|
||||
it.each(["succeeded", "failed", "cancelled"] as const)("settles the exact wake atomically with a fresh %s projection", async status => {
|
||||
const binding = await accepted(status), wake = await claimedWake(binding);
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const run = await read(binding.runId), settled = await readWake(wake.id);
|
||||
expect(run.status).toBe(status);
|
||||
expect(settled).toMatchObject({ status: status === "succeeded" ? "completed" : status, runId: binding.runId,
|
||||
companyId, agentId, finishedAt: run.finishedAt, error: run.error });
|
||||
expect(settled.finishedAt).not.toBeNull();
|
||||
await projectCommittedRun({ db, run, coordinator: await state(binding.runId) });
|
||||
expect(await readWake(wake.id)).toEqual(settled);
|
||||
});
|
||||
|
||||
it.each(["provider", "workspace"] as const)("settles the exact wake when %s finalization retries exhaust", async scope => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
if (scope === "workspace") await db.update(workspaceOperations).set({ status: "failed" })
|
||||
.where(eq(workspaceOperations.heartbeatRunId, binding.runId));
|
||||
if (scope === "provider") await db.update(nativeRunResults).set({
|
||||
resultJson: sql`jsonb_set(${nativeRunResults.resultJson}, '{terminal,runTerminalState}', '"unknown"'::jsonb)`,
|
||||
}).where(eq(nativeRunResults.runId, binding.runId));
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) })
|
||||
.where(eq(nativeRunFinalizations.runId, binding.runId));
|
||||
if (scope === "provider") await reconcileNativeFinalizations(db, [binding.runId]);
|
||||
else await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true,
|
||||
failureScope: "workspace", error: new Error("native_workspace_finalize_failed") });
|
||||
if (attempt < 2) {
|
||||
expect((await read(binding.runId)).status).toBe("running");
|
||||
expect(await readWake(wake.id)).toMatchObject({ status: "claimed", finishedAt: null });
|
||||
}
|
||||
}
|
||||
const run = await read(binding.runId);
|
||||
expect((await state(binding.runId)).phase).toBe("terminal_failure");
|
||||
expect(run).toMatchObject({ status: scope === "provider" ? "succeeded" : "failed", nativePhase: "terminal_failure" });
|
||||
expect(run.finishedAt).not.toBeNull();
|
||||
expect(await readWake(wake.id)).toMatchObject({ status: scope === "provider" ? "completed" : "failed",
|
||||
runId: binding.runId, finishedAt: run.finishedAt });
|
||||
});
|
||||
|
||||
it("keeps the source wake claimed when exhausted failure does not own terminal projection", async () => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId));
|
||||
for (let attempt = 0; attempt < 3; attempt++) await recordNativeFinalizationFailure({ db, runId: binding.runId,
|
||||
projectRunStatus: false, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") });
|
||||
expect((await state(binding.runId)).phase).toBe("terminal_failure");
|
||||
expect((await read(binding.runId)).status).toBe("running");
|
||||
expect(await readWake(wake.id)).toEqual(wake);
|
||||
});
|
||||
|
||||
it("rolls back exhausted projection and its claim if wake notification registration fails", async () => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId));
|
||||
const fail = () => recordNativeFinalizationFailure({ db, runId: binding.runId,
|
||||
projectRunStatus: true, failureScope: "workspace" as const, error: new Error("native_workspace_finalize_failed") });
|
||||
await fail(); await fail();
|
||||
const beforeRun = await read(binding.runId), beforeCoordinator = await state(binding.runId);
|
||||
const notify = wakeSettlementNotifications.notifyExternalConversationSettlement;
|
||||
vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement").mockImplementation(async (tx, scope) => {
|
||||
const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
expect(current.status).toBe("claimed");
|
||||
await notify(tx, scope);
|
||||
throw new Error("exhaustion wake notification failure");
|
||||
});
|
||||
await expect(fail()).rejects.toThrow("exhaustion wake notification failure");
|
||||
expect(await read(binding.runId)).toEqual(beforeRun);
|
||||
expect(await state(binding.runId)).toEqual(beforeCoordinator);
|
||||
expect(await readWake(wake.id)).toEqual(wake);
|
||||
});
|
||||
|
||||
it.each(["succeeded", "failed", "cancelled"] as const)("discovers and repairs an old %s run's claimed wake once", async status => {
|
||||
const binding = await accepted(status);
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true });
|
||||
const run = await read(binding.runId), wake = await claimedWake(binding);
|
||||
const work = nativeRunProjectionWork(db);
|
||||
const [entry] = await work.load({ companyId, runId: binding.runId });
|
||||
expect(entry).toMatchObject({ id: binding.runId });
|
||||
await work.check(entry);
|
||||
expect(await readWake(wake.id)).toMatchObject({ status: status === "succeeded" ? "completed" : status,
|
||||
finishedAt: run.finishedAt });
|
||||
expect((await read(binding.runId)).finishedAt).toEqual(run.finishedAt);
|
||||
expect(await work.load({ companyId, runId: binding.runId })).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(["unbound", "newer-run", "queued", "cancelled", "foreign-company", "foreign-agent", "live-owner"] as const)(
|
||||
"does not settle a %s wake claim", async fault => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
if (fault === "unbound" || fault === "newer-run") await db.update(agentWakeupRequests)
|
||||
.set({ runId: fault === "unbound" ? null : randomUUID() }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
if (fault === "queued" || fault === "cancelled") await db.update(agentWakeupRequests)
|
||||
.set({ status: fault }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
if (fault === "foreign-company") {
|
||||
const otherCompany = randomUUID();
|
||||
await db.insert(companies).values({ id: otherCompany, name: "Other wake owner", issuePrefix: `OW${randomUUID().slice(0, 6)}` });
|
||||
await db.update(agentWakeupRequests).set({ companyId: otherCompany }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
}
|
||||
if (fault === "foreign-agent") {
|
||||
const otherAgent = randomUUID();
|
||||
await db.insert(agents).values({ id: otherAgent, companyId, name: "Other wake owner", adapterType: "paperclip_runner" });
|
||||
await db.update(agentWakeupRequests).set({ agentId: otherAgent }).where(eq(agentWakeupRequests.id, wake.id));
|
||||
}
|
||||
if (fault === "live-owner") await db.update(heartbeatRuns).set({ status: "running", nativePhase: "terminal_failure",
|
||||
errorCode: "native_execution_ownership_unverified" }).where(eq(heartbeatRuns.id, binding.runId));
|
||||
const before = await readWake(wake.id);
|
||||
await projectCommittedRun({ db, run: await read(binding.runId), coordinator: await state(binding.runId) });
|
||||
expect(await readWake(wake.id)).toEqual(before);
|
||||
if (fault === "live-owner") expect((await read(binding.runId)).status).toBe("running");
|
||||
},
|
||||
);
|
||||
|
||||
it("registers wake settlement intent before mutation and rolls back notification failure", async () => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
const before = await read(binding.runId);
|
||||
const notify = wakeSettlementNotifications.notifyExternalConversationSettlement;
|
||||
const notification = vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement")
|
||||
.mockImplementation(async (tx, scope) => {
|
||||
const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id));
|
||||
expect(current.status).toBe("claimed");
|
||||
await notify(tx, scope);
|
||||
throw new Error("wake intent registration failure");
|
||||
});
|
||||
await expect(projectCommittedRun({ db, run: before, coordinator: await state(binding.runId) }))
|
||||
.rejects.toThrow("wake intent registration failure");
|
||||
expect(notification).toHaveBeenCalledOnce();
|
||||
expect(await read(binding.runId)).toEqual(before);
|
||||
expect(await readWake(wake.id)).toEqual(wake);
|
||||
});
|
||||
|
||||
it("rolls back terminal projection and exact wake settlement together", async () => {
|
||||
const binding = await accepted(), wake = await claimedWake(binding);
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
const before = await read(binding.runId);
|
||||
await expect(db.transaction(async tx => {
|
||||
await projectCommittedRun({ db: tx as unknown as Db, run: before, coordinator: await state(binding.runId) });
|
||||
throw new Error("projection rollback fixture");
|
||||
})).rejects.toThrow("projection rollback fixture");
|
||||
expect(await read(binding.runId)).toEqual(before);
|
||||
expect(await readWake(wake.id)).toEqual(wake);
|
||||
});
|
||||
|
||||
it("reconstructs a committed decision with an unfinished run projection and stays quiet after repair", async () => {
|
||||
const binding = await accepted();
|
||||
await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" });
|
||||
|
||||
@@ -8,6 +8,7 @@ import { skillInstallationMaintenance } from "../services/skill-installation-mai
|
||||
import { skillPackageMaintenance } from "../services/skill-package-maintenance.js";
|
||||
import { hasUnsettledLocalInstructionCopies } from "../services/agent-instruction-working-copies.js";
|
||||
import { providerTraceStore } from "../services/provider-trace-store.js";
|
||||
import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js";
|
||||
|
||||
/** Runs only in the maintenance process after Cloud has confirmed all legacy
|
||||
* deployments stopped. Each owner retains its own migration/publication rules;
|
||||
@@ -23,10 +24,15 @@ export async function migrateEphemeralServerData(db: Db, input: {
|
||||
if (await hasUnsettledLocalInstructionCopies(db)) throw new Error("local_instruction_copies_require_settlement");
|
||||
const before = await readEphemeralServerDataEligibility(db);
|
||||
const unsupported = before.blockers.filter(blocker => ![
|
||||
"agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration",
|
||||
"agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration",
|
||||
].includes(blocker));
|
||||
if (unsupported.length) throw new Error(unsupported[0]);
|
||||
|
||||
phase("workspaces");
|
||||
const storage = getStorageService();
|
||||
if (storage.provider !== "s3") throw new Error("shared_object_storage_required");
|
||||
counts.workspaces = await migrateWorkspaceTrees(db, { storage, signal, phase });
|
||||
|
||||
phase("agent_files");
|
||||
let after: string | undefined;
|
||||
do {
|
||||
@@ -77,8 +83,6 @@ export async function migrateEphemeralServerData(db: Db, input: {
|
||||
}
|
||||
|
||||
phase("verify_agent_objects");
|
||||
const storage = getStorageService();
|
||||
if (storage.provider !== "s3") throw new Error("shared_object_storage_required");
|
||||
counts.verifiedAgentObjects = await verifyAgentFileObjects(db, storage, signal);
|
||||
phase("verify_skill_versions");
|
||||
after = undefined;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { idleApiActivity } from "../services/idle-api-activity.js";
|
||||
import { createHash, timingSafeEqual } from "node:crypto";
|
||||
import type { Request, RequestHandler } from "express";
|
||||
import { and, eq, isNull, sql } from "drizzle-orm";
|
||||
@@ -43,7 +44,7 @@ import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compa
|
||||
import { forbidden, unauthorized, unprocessable } from "../errors.js";
|
||||
|
||||
export { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js";
|
||||
import { cloudTenantPrimaryCompanyId, isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
|
||||
function hashToken(token: string) {
|
||||
return createHash("sha256").update(token).digest("hex");
|
||||
@@ -329,6 +330,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa
|
||||
runId: runIdHeader || undefined,
|
||||
source: "board_key",
|
||||
};
|
||||
if (isCloudManagedInstance() && (access.isInstanceAdmin || access.companyIds.length)) {
|
||||
_res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); });
|
||||
}
|
||||
next();
|
||||
return;
|
||||
}
|
||||
@@ -521,7 +525,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa
|
||||
runId: runIdHeader || undefined,
|
||||
source: "agent_key",
|
||||
};
|
||||
|
||||
if (isCloudManagedInstance() && req.actor.onBehalfOfMemberships?.length) {
|
||||
_res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); });
|
||||
}
|
||||
next();
|
||||
};
|
||||
return async (req, res, next) => {
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { Request } from "express";
|
||||
import type { VerifiedIdleCapabilities } from "../services/idle-incoming-wake-capability.js";
|
||||
import type { RequestHandler } from "express";
|
||||
import { logger } from "./logger.js";
|
||||
import {
|
||||
@@ -6,6 +8,12 @@ import {
|
||||
type CloudControlAction,
|
||||
} from "../services/cloud-runtime-identity.js";
|
||||
|
||||
const idleCapabilities = new WeakMap<Request, VerifiedIdleCapabilities>();
|
||||
/** Only this middleware's verified assertion can install request capabilities. */
|
||||
export function verifiedIdleCapabilities(req: Request): VerifiedIdleCapabilities | undefined {
|
||||
return idleCapabilities.get(req);
|
||||
}
|
||||
|
||||
/** Endpoint → method → the one action a control assertion must name to take it. */
|
||||
const ACTIONS_BY_ENDPOINT: Record<string, Record<string, CloudControlAction>> = {
|
||||
"/api/instance/connector-events/wake": { POST: "connector-events:wake" },
|
||||
@@ -50,7 +58,8 @@ export function cloudControlMiddleware(): RequestHandler {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
verifyCloudControlAssertion({ compactJws: assertion, expectedAction });
|
||||
const claims = verifyCloudControlAssertion({ compactJws: assertion, expectedAction });
|
||||
if (claims.idleCapabilities) idleCapabilities.set(req, claims.idleCapabilities as VerifiedIdleCapabilities);
|
||||
} catch (error) {
|
||||
logger.warn({ err: error }, "Rejected Cloud control assertion");
|
||||
res.status(401).json({ error: "invalid_cloud_control_assertion" });
|
||||
|
||||
@@ -56,3 +56,9 @@ Ordinary configuration updates use the root lifecycle command.
|
||||
|
||||
The [agent lifecycle module](../../../doc/AGENT-LIFECYCLE.md) owns agent creation,
|
||||
pause, resume, and termination. Use its commands for these changes.
|
||||
|
||||
The execution admission module owns project task-execution policy and the
|
||||
durable admission records that charge project capacity. Today it exposes only
|
||||
internal policy commands and the deletion guards that project, agent, and
|
||||
company deletion call. Execution claims still belong to `services/heartbeat/queue.ts`
|
||||
and native review dispatch. See [execution admission accounting](../../../doc/DATABASE.md#execution-admission-accounting).
|
||||
@@ -28,7 +28,9 @@ export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects
|
||||
const prepareSetup = changed && current.lifecycleHolds.length === 1 && current.lifecycleHolds[0] === "setup"
|
||||
&& ["paused", "preparing", "verifying"].includes(current.lifecycleState)
|
||||
&& await effects.setupRequiresPreparation(tx, current.companyId,
|
||||
Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") ? patch.defaultEnvironmentId : current.defaultEnvironmentId);
|
||||
Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") ? patch.defaultEnvironmentId : current.defaultEnvironmentId,
|
||||
{ adapterType: patch.adapterType ?? current.adapterType,
|
||||
adapterConfig: Object.prototype.hasOwnProperty.call(patch, "adapterConfig") ? patch.adapterConfig : current.adapterConfig });
|
||||
const invalidate = changed && (["preparing", "verifying", "resuming"].includes(current.lifecycleState) || prepareSetup);
|
||||
if (invalidate) await effects.notifyWork(tx);
|
||||
const [updated] = await tx.update(agents).set({ ...patch, updatedAt: new Date(), ...(invalidate ? {
|
||||
|
||||
@@ -6,7 +6,8 @@ import type { ActivityPublication } from "../../../types/activity-publication.js
|
||||
|
||||
/** Services supply integrations; lifecycle owns the transaction and state writes. */
|
||||
export interface LifecycleEffects {
|
||||
setupRequiresPreparation(db: Db, companyId: string, environmentId?: string | null): Promise<boolean>;
|
||||
setupRequiresPreparation(db: Db, companyId: string, environmentId: string | null | undefined,
|
||||
profile: Pick<CreateAgentData, "adapterType" | "adapterConfig">): Promise<boolean>;
|
||||
notifyWork(db: Db): Promise<void>;
|
||||
beforeStatusChange(db: Db, companyId: string, agentId: string, before: string, after: string): Promise<void>;
|
||||
reportFailure(context: LifecycleFailure, error: unknown): void;
|
||||
|
||||
@@ -64,7 +64,7 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec
|
||||
if (command === "complete" && next === "ready" && holds.length) next = "pausing";
|
||||
if (command === "complete" && next === "paused" && !holds.length) next = "resuming";
|
||||
if (["complete", "reconcile"].includes(command) && next === "paused" && holds.length === 1 && holds[0] === "setup"
|
||||
&& await effects.setupRequiresPreparation(tx, agent.companyId, agent.defaultEnvironmentId)) next = "preparing";
|
||||
&& await effects.setupRequiresPreparation(tx, agent.companyId, agent.defaultEnvironmentId, agent)) next = "preparing";
|
||||
if (next === agent.lifecycleState) {
|
||||
if (JSON.stringify(holds) !== JSON.stringify(agent.lifecycleHolds) || agent.pauseReason !== (holds[0] ?? null)) {
|
||||
const [updated] = await tx.update(agents).set({ lifecycleHolds: holds, pauseReason: holds[0] ?? null }).where(eq(agents.id, agent.id)).returning();
|
||||
|
||||
@@ -36,7 +36,7 @@ export function agentRecords(db: Db, effects: LifecycleEffects) {
|
||||
const [company] = await tx.select({ status: companies.status }).from(companies).where(eq(companies.id, companyId));
|
||||
const companyHolds = company?.status === "paused" ? ["company_paused"] : company?.status === "archived" ? ["company_archived"] : [];
|
||||
const holds = [...(status === "paused" ? [pauseReason ?? "manual"] : []), ...companyHolds];
|
||||
const prepareSetup = options?.setupOnly && !companyHolds.length && await effects.setupRequiresPreparation(txDb, companyId, prepared.defaultEnvironmentId);
|
||||
const prepareSetup = options?.setupOnly && !companyHolds.length && await effects.setupRequiresPreparation(txDb, companyId, prepared.defaultEnvironmentId, prepared);
|
||||
const lifecycleState = status === "pending_approval" ? "pending_approval" : status === "terminated" ? "terminated"
|
||||
: prepareSetup ? "preparing" : status === "paused" ? "paused" : companyHolds.length ? "pausing" : "preparing";
|
||||
if (lifecycleState === "preparing" || lifecycleState === "pausing") await effects.notifyWork(tx);
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import { executionAdmissions } from "@paperclipai/db";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { CompanyDeletionParticipant } from "../../../lib/company-deletion.js";
|
||||
|
||||
/** Removes every admission, settled or not, before the company's runs and projects are deleted. */
|
||||
export const executionAdmissionCompanyDeletion: CompanyDeletionParticipant = {
|
||||
async deleteCompanyData(tx, companyId) {
|
||||
if (!("nestedIndex" in tx)) throw new Error("Company deletion requires a database transaction");
|
||||
await tx.delete(executionAdmissions).where(eq(executionAdmissions.companyId, companyId));
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,239 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
agents,
|
||||
companies,
|
||||
createDb,
|
||||
executionAdmissionProjectCharges as charges,
|
||||
executionAdmissions as admissions,
|
||||
heartbeatRuns,
|
||||
projectTaskExecutionPolicies as policies,
|
||||
projects,
|
||||
} from "@paperclipai/db";
|
||||
import { asc, eq, sql } from "drizzle-orm";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
|
||||
import { withAccountingTransaction } from "../../../services/accounting-transaction.js";
|
||||
import { deleteAgentDependencies } from "../../../services/agent-deletion.js";
|
||||
import { deleteCompany } from "../../../services/company-deletion.js";
|
||||
import { projectService } from "../../../services/projects.js";
|
||||
import { prepareAdmissionRecord, type AdmissionRecord } from "../domain/admission.js";
|
||||
import { createExecutionAdmission } from "../index.js";
|
||||
import { countActiveProjectCharges, recordAdmission, settleAdmission } from "./postgres.js";
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
type TestDb = ReturnType<typeof createDb>;
|
||||
|
||||
function record(primaryProjectId: string | null, referencedProjectIds: string[], contextRevision = "issue:1") {
|
||||
const result = prepareAdmissionRecord({ primaryProjectId, referencedProjectIds, contextRevision });
|
||||
if (result.kind !== "ok") throw new Error(result.kind);
|
||||
return result.record;
|
||||
}
|
||||
|
||||
describe.skipIf(!support.supported)("execution admission storage", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: TestDb;
|
||||
|
||||
beforeAll(async () => {
|
||||
database = await startEmbeddedPostgresTestDatabase("execution-admission-");
|
||||
db = createDb(database.connectionString);
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await db?.$client.end();
|
||||
await database?.cleanup();
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const [company] = await db.insert(companies).values({ name: "Admission fixture", issuePrefix: `A${randomUUID().slice(0, 6)}` }).returning();
|
||||
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Admission agent", adapterType: "process" }).returning();
|
||||
const projectRows = await db.insert(projects).values(["Design", "Docs", "Unlimited"].map(name => ({ companyId: company.id, name })))
|
||||
.returning();
|
||||
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id }).returning();
|
||||
return { companyId: company.id, agentId: agent.id, projectIds: projectRows.map(row => row.id).sort(), runId: run.id };
|
||||
}
|
||||
|
||||
const inTransaction = <T>(work: (tx: TestDb) => Promise<T>) => db.transaction(tx => work(tx as unknown as TestDb));
|
||||
const admit = (input: { companyId: string; runId: string; admissionId?: string; record: AdmissionRecord }) =>
|
||||
inTransaction(tx => recordAdmission(tx, { ...input, admissionId: input.admissionId ?? randomUUID() }));
|
||||
const settle = (companyId: string, admissionId: string, settlementId = randomUUID()) =>
|
||||
inTransaction(tx => settleAdmission(tx, { companyId, admissionId, settlementId }));
|
||||
const chargedProjects = async (admissionId: string) => (await db.select({ projectId: charges.projectId }).from(charges)
|
||||
.where(eq(charges.admissionId, admissionId)).orderBy(asc(charges.projectId))).map(row => row.projectId);
|
||||
|
||||
it("stores absent, explicit unlimited and positive maximums with partial updates", async () => {
|
||||
const { companyId, projectIds: [projectId] } = await fixture();
|
||||
const policy = createExecutionAdmission(db);
|
||||
const scope = { companyId, projectId };
|
||||
expect(await policy.readTaskExecutionPolicy(scope)).toEqual({ kind: "found", policy: { state: "absent", maxConcurrentTasks: null } });
|
||||
expect(await policy.updateTaskExecutionPolicy(scope, {})).toMatchObject({ kind: "unchanged", policy: { state: "absent" } });
|
||||
expect(await db.select().from(policies)).not.toContainEqual(expect.objectContaining({ projectId }));
|
||||
|
||||
expect(await policy.updateTaskExecutionPolicy(scope, { maxConcurrentTasks: 2 }))
|
||||
.toEqual({ kind: "updated", policy: { state: "configured", maxConcurrentTasks: 2 } });
|
||||
expect(await policy.updateTaskExecutionPolicy(scope, undefined)).toMatchObject({ kind: "unchanged", policy: { maxConcurrentTasks: 2 } });
|
||||
expect(await policy.updateTaskExecutionPolicy(scope, { maxConcurrentTasks: 0 })).toMatchObject({ kind: "invalid" });
|
||||
expect(await policy.updateTaskExecutionPolicy(scope, null))
|
||||
.toEqual({ kind: "updated", policy: { state: "configured", maxConcurrentTasks: null } });
|
||||
expect(await policy.readTaskExecutionPolicy(scope)).toEqual({ kind: "found", policy: { state: "configured", maxConcurrentTasks: null } });
|
||||
|
||||
const foreign = { companyId: (await fixture()).companyId, projectId };
|
||||
expect(await policy.readTaskExecutionPolicy(foreign)).toEqual({ kind: "project_not_found" });
|
||||
expect(await policy.updateTaskExecutionPolicy(foreign, { maxConcurrentTasks: 1 })).toEqual({ kind: "project_not_found" });
|
||||
expect(await policy.readTaskExecutionPolicy(scope)).toMatchObject({ policy: { maxConcurrentTasks: null } });
|
||||
});
|
||||
|
||||
it("enforces policy constraints and company scope in the database", async () => {
|
||||
const { companyId, projectIds: [projectId] } = await fixture();
|
||||
const other = await fixture();
|
||||
await expect(db.insert(policies).values({ companyId, projectId, maxConcurrentTasks: 0 }))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "project_task_execution_policies_maximum_check" } });
|
||||
await expect(db.insert(policies).values({ companyId: other.companyId, projectId, maxConcurrentTasks: 1 }))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "project_task_execution_policies_company_project_fk" } });
|
||||
});
|
||||
|
||||
it("charges every distinct project, including unlimited and primaryless sets", async () => {
|
||||
const { companyId, runId, projectIds: [a, b, c] } = await fixture();
|
||||
await createExecutionAdmission(db).updateTaskExecutionPolicy({ companyId, projectId: c }, null);
|
||||
const admissionId = randomUUID();
|
||||
const admitted = record(a, [c, b, a]);
|
||||
expect(await admit({ companyId, runId, admissionId, record: admitted })).toEqual({ kind: "recorded", attempt: 1 });
|
||||
expect(await chargedProjects(admissionId)).toEqual([a, b, c]);
|
||||
const [epoch] = await db.select().from(admissions).where(eq(admissions.id, admissionId));
|
||||
expect(epoch).toMatchObject({ companyId, runId, attempt: 1, primaryProjectId: a, contextRevision: "issue:1", settlementId: null });
|
||||
|
||||
const other = await fixture();
|
||||
const primaryless = randomUUID();
|
||||
expect(await admit({ ...other, admissionId: primaryless, record: record(null, [other.projectIds[1]]) }))
|
||||
.toMatchObject({ kind: "recorded" });
|
||||
expect(await chargedProjects(primaryless)).toEqual([other.projectIds[1]]);
|
||||
|
||||
const empty = await fixture();
|
||||
const emptyId = randomUUID();
|
||||
expect(await admit({ ...empty, admissionId: emptyId, record: record(null, []) })).toMatchObject({ kind: "recorded" });
|
||||
expect(await chargedProjects(emptyId)).toEqual([]);
|
||||
expect([...(await countActiveProjectCharges(db, empty.companyId, empty.projectIds)).values()]).toEqual([0, 0, 0]);
|
||||
});
|
||||
|
||||
it("replays one admission identity and refuses a second live epoch for the run", async () => {
|
||||
const { companyId, runId, projectIds: [a, b] } = await fixture();
|
||||
const admissionId = randomUUID();
|
||||
const admitted = record(a, [b]);
|
||||
await admit({ companyId, runId, admissionId, record: admitted });
|
||||
expect(await admit({ companyId, runId, admissionId, record: admitted })).toEqual({ kind: "replayed", attempt: 1 });
|
||||
expect(await admit({ companyId, runId, admissionId, record: record(a, []) })).toEqual({ kind: "admission_conflict" });
|
||||
expect(await admit({ companyId, runId, admissionId, record: record(a, [b], "issue:2") })).toEqual({ kind: "admission_conflict" });
|
||||
expect(await admit({ companyId, runId, record: admitted })).toEqual({ kind: "run_already_admitted", admissionId });
|
||||
expect(await db.select({ id: admissions.id }).from(admissions).where(eq(admissions.runId, runId))).toHaveLength(1);
|
||||
await expect(db.insert(admissions).values({ id: randomUUID(), companyId, runId, attempt: 2, contextRevision: "x" }))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "execution_admissions_live_run_uq" } });
|
||||
});
|
||||
|
||||
it("settles the whole set once and starts a new attempt for the same run", async () => {
|
||||
const { companyId, runId, projectIds: [a, b] } = await fixture();
|
||||
const first = randomUUID();
|
||||
await admit({ companyId, runId, admissionId: first, record: record(a, [b]) });
|
||||
const settlementId = randomUUID();
|
||||
expect(await settle(companyId, first, settlementId)).toEqual({ kind: "settled", releasedProjectIds: [a, b] });
|
||||
expect(await settle(companyId, first, settlementId)).toEqual({ kind: "already_settled" });
|
||||
expect(await settle(companyId, first)).toEqual({ kind: "settled_by_other" });
|
||||
expect(await settle(randomUUID(), first)).toEqual({ kind: "not_found" });
|
||||
expect(await chargedProjects(first)).toEqual([]);
|
||||
expect(await admit({ companyId, runId, admissionId: first, record: record(a, [b]) })).toEqual({ kind: "admission_settled" });
|
||||
|
||||
const second = randomUUID();
|
||||
expect(await admit({ companyId, runId, admissionId: second, record: record(a, [], "issue:2") })).toEqual({ kind: "recorded", attempt: 2 });
|
||||
expect(await db.select({ attempt: admissions.attempt, settled: sql<boolean>`${admissions.settledAt} is not null` })
|
||||
.from(admissions).where(eq(admissions.runId, runId)).orderBy(asc(admissions.attempt)))
|
||||
.toEqual([{ attempt: 1, settled: true }, { attempt: 2, settled: false }]);
|
||||
await expect(db.update(admissions).set({ settlementId: randomUUID() }).where(eq(admissions.id, second)))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "execution_admissions_settlement_check" } });
|
||||
});
|
||||
|
||||
it("keeps company isolation and rolls back the whole record", async () => {
|
||||
const { companyId, runId, projectIds: [a] } = await fixture();
|
||||
const other = await fixture();
|
||||
expect(await admit({ companyId, runId, record: record(a, [other.projectIds[0]]) }))
|
||||
.toEqual({ kind: "projects_unavailable", projectIds: [other.projectIds[0]] });
|
||||
expect(await admit({ companyId, runId: other.runId, record: record(a, []) })).toEqual({ kind: "run_not_found" });
|
||||
const admissionId = randomUUID();
|
||||
await admit({ companyId, runId, admissionId, record: record(a, []) });
|
||||
expect(await admit({ companyId: other.companyId, runId: other.runId, admissionId, record: record(null, []) }))
|
||||
.toEqual({ kind: "admission_conflict" });
|
||||
await expect(db.insert(charges).values({ admissionId, companyId, projectId: other.projectIds[0] }))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_company_project_fk" } });
|
||||
await expect(db.insert(charges).values({ admissionId, companyId, projectId: a }))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_admission_id_project_id_pk" } });
|
||||
|
||||
const rolledBack = randomUUID();
|
||||
await expect(inTransaction(async tx => {
|
||||
await recordAdmission(tx, { companyId: other.companyId, runId: other.runId, admissionId: rolledBack, record: record(null, other.projectIds) });
|
||||
throw new Error("claim lost");
|
||||
})).rejects.toThrow("claim lost");
|
||||
expect(await db.select().from(admissions).where(eq(admissions.id, rolledBack))).toEqual([]);
|
||||
expect(await chargedProjects(rolledBack)).toEqual([]);
|
||||
await expect(recordAdmission(db, { companyId, runId, admissionId: randomUUID(), record: record(a, []) }))
|
||||
.rejects.toThrow("requires a database transaction");
|
||||
});
|
||||
|
||||
it("counts active charges per project through the project index", async () => {
|
||||
const first = await fixture();
|
||||
const [a, b, c] = first.projectIds;
|
||||
const [secondRun] = await db.insert(heartbeatRuns).values({ companyId: first.companyId, agentId: first.agentId }).returning();
|
||||
const held = randomUUID();
|
||||
await admit({ ...first, admissionId: held, record: record(a, [b]) });
|
||||
await admit({ companyId: first.companyId, runId: secondRun.id, record: record(null, [a]) });
|
||||
expect(Object.fromEntries(await countActiveProjectCharges(db, first.companyId, [a, b, c]))).toEqual({ [a]: 2, [b]: 1, [c]: 0 });
|
||||
await settle(first.companyId, held);
|
||||
expect(Object.fromEntries(await countActiveProjectCharges(db, first.companyId, [a, b, c]))).toEqual({ [a]: 1, [b]: 0, [c]: 0 });
|
||||
expect(Object.fromEntries(await countActiveProjectCharges(db, randomUUID(), [a]))).toEqual({ [a]: 0 });
|
||||
|
||||
const plan = await inTransaction(async tx => {
|
||||
await tx.execute(sql`set local enable_seqscan = off`);
|
||||
return tx.execute(sql`explain select project_id, count(*) from execution_admission_project_charges
|
||||
where company_id = ${first.companyId} and project_id in (${a}, ${b}) group by project_id`);
|
||||
});
|
||||
expect(JSON.stringify(plan)).toContain("execution_admission_project_charges_project_idx");
|
||||
});
|
||||
|
||||
it("preserves unsettled charges through project archive and delete", async () => {
|
||||
const { companyId, runId, projectIds: [a, b] } = await fixture();
|
||||
const admissionId = randomUUID();
|
||||
await admit({ companyId, runId, admissionId, record: record(a, [b]) });
|
||||
const projectsService = projectService(db);
|
||||
await db.update(projects).set({ archivedAt: new Date() }).where(eq(projects.id, b));
|
||||
expect(await chargedProjects(admissionId)).toEqual([a, b]);
|
||||
await expect(projectsService.remove(a)).rejects.toMatchObject({ status: 409, details: { code: "project_execution_admission_pending" } });
|
||||
await expect(db.delete(projects).where(eq(projects.id, b)))
|
||||
.rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_company_project_fk" } });
|
||||
expect(await chargedProjects(admissionId)).toEqual([a, b]);
|
||||
|
||||
await createExecutionAdmission(db).updateTaskExecutionPolicy({ companyId, projectId: a }, { maxConcurrentTasks: 1 });
|
||||
await settle(companyId, admissionId);
|
||||
expect(await projectsService.remove(a)).toMatchObject({ id: a });
|
||||
expect(await db.select().from(policies).where(eq(policies.projectId, a))).toEqual([]);
|
||||
expect(await db.select({ primaryProjectId: admissions.primaryProjectId }).from(admissions).where(eq(admissions.id, admissionId)))
|
||||
.toEqual([{ primaryProjectId: a }]);
|
||||
});
|
||||
|
||||
it("guards agent deletion while unsettled and removes admissions with the company", async () => {
|
||||
const { companyId, agentId, runId, projectIds: [a] } = await fixture();
|
||||
const admissionId = randomUUID();
|
||||
await admit({ companyId, runId, admissionId, record: record(a, []) });
|
||||
await expect(withAccountingTransaction(db, companyId, tx => deleteAgentDependencies(tx, companyId, agentId)))
|
||||
.rejects.toMatchObject({ status: 409, details: { code: "agent_execution_admission_pending" } });
|
||||
expect(await chargedProjects(admissionId)).toEqual([a]);
|
||||
await settle(companyId, admissionId);
|
||||
await withAccountingTransaction(db, companyId, tx => deleteAgentDependencies(tx, companyId, agentId));
|
||||
expect(await db.select().from(admissions).where(eq(admissions.runId, runId))).toEqual([]);
|
||||
|
||||
const deleted = await fixture();
|
||||
const retained = await fixture();
|
||||
const retainedAdmission = randomUUID();
|
||||
await admit({ ...retained, admissionId: retainedAdmission, record: record(retained.projectIds[0], []) });
|
||||
await admit({ ...deleted, record: record(deleted.projectIds[0], deleted.projectIds) });
|
||||
await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, deleted.agentId));
|
||||
expect(await deleteCompany(db, deleted.companyId)).toMatchObject({ id: deleted.companyId });
|
||||
expect(await db.select().from(admissions).where(eq(admissions.companyId, deleted.companyId))).toEqual([]);
|
||||
expect(await chargedProjects(retainedAdmission)).toEqual([retained.projectIds[0]]);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,165 @@
|
||||
import {
|
||||
executionAdmissionProjectCharges as charges,
|
||||
executionAdmissions as admissions,
|
||||
heartbeatRuns,
|
||||
projectTaskExecutionPolicies as policies,
|
||||
projects,
|
||||
type Db,
|
||||
} from "@paperclipai/db";
|
||||
import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm";
|
||||
import type { TaskExecutionPolicyStore } from "../application/policy.js";
|
||||
import { decideSettlement, sameAdmissionRecord, type AdmissionRecord } from "../domain/admission.js";
|
||||
import { ABSENT_TASK_EXECUTION_POLICY, type TaskExecutionPolicy } from "../domain/policy.js";
|
||||
|
||||
function assertTransaction(tx: Db) {
|
||||
if (!("nestedIndex" in tx)) throw new Error("Execution admission storage requires a database transaction");
|
||||
}
|
||||
|
||||
/**
|
||||
* Global lock order step 3: locks the whole project set in ascending ID order
|
||||
* in one statement. Returns the IDs that are not projects of the company.
|
||||
*/
|
||||
export async function lockAdmissionProjects(tx: Db, companyId: string, projectIds: readonly string[]) {
|
||||
assertTransaction(tx);
|
||||
if (projectIds.length === 0) return [];
|
||||
const rows = await tx.select({ id: projects.id }).from(projects)
|
||||
.where(and(eq(projects.companyId, companyId), inArray(projects.id, [...projectIds])))
|
||||
.orderBy(asc(projects.id)).for("no key update");
|
||||
const found = new Set(rows.map(row => row.id));
|
||||
return projectIds.filter(id => !found.has(id));
|
||||
}
|
||||
|
||||
function toPolicy(row: { maxConcurrentTasks: number | null } | undefined): TaskExecutionPolicy {
|
||||
return row ? { state: "configured", maxConcurrentTasks: row.maxConcurrentTasks } : ABSENT_TASK_EXECUTION_POLICY;
|
||||
}
|
||||
|
||||
export function createPostgresTaskExecutionPolicyStore(db: Db): TaskExecutionPolicyStore {
|
||||
return {
|
||||
async read({ companyId, projectId }) {
|
||||
const [row] = await db.select({ configured: policies.projectId, maxConcurrentTasks: policies.maxConcurrentTasks })
|
||||
.from(projects).leftJoin(policies, eq(policies.projectId, projects.id))
|
||||
.where(and(eq(projects.companyId, companyId), eq(projects.id, projectId)));
|
||||
if (!row) return null;
|
||||
return row.configured ? toPolicy(row) : ABSENT_TASK_EXECUTION_POLICY;
|
||||
},
|
||||
|
||||
update: ({ companyId, projectId }, change) => db.transaction(async transaction => {
|
||||
const tx = transaction as unknown as Db;
|
||||
if ((await lockAdmissionProjects(tx, companyId, [projectId])).length > 0) return null;
|
||||
const [row] = await tx.select({ maxConcurrentTasks: policies.maxConcurrentTasks }).from(policies)
|
||||
.where(eq(policies.projectId, projectId));
|
||||
const current = toPolicy(row);
|
||||
const next = change(current);
|
||||
if (next === current) return current;
|
||||
if (next.state === "absent") throw new Error("A stored task execution policy cannot become absent");
|
||||
await tx.insert(policies).values({ projectId, companyId, maxConcurrentTasks: next.maxConcurrentTasks })
|
||||
.onConflictDoUpdate({ target: policies.projectId, set: { maxConcurrentTasks: next.maxConcurrentTasks, updatedAt: sql`now()` } });
|
||||
return next;
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export type RecordAdmissionInput = { companyId: string; runId: string; admissionId: string; record: AdmissionRecord };
|
||||
|
||||
export type RecordAdmissionOutcome =
|
||||
| { kind: "recorded" | "replayed"; attempt: number }
|
||||
| { kind: "projects_unavailable"; projectIds: string[] }
|
||||
| { kind: "run_not_found" }
|
||||
| { kind: "run_already_admitted"; admissionId: string }
|
||||
| { kind: "admission_settled" }
|
||||
| { kind: "admission_conflict" };
|
||||
|
||||
/**
|
||||
* Storage transition only: persists one admission attempt and its charges in
|
||||
* the caller's transaction. It neither checks capacity nor claims work; the
|
||||
* composed admission command must do both under the same locks. Takes lock
|
||||
* steps 3 (projects) and 6 (run); composed callers take them earlier.
|
||||
*/
|
||||
export async function recordAdmission(tx: Db, input: RecordAdmissionInput): Promise<RecordAdmissionOutcome> {
|
||||
assertTransaction(tx);
|
||||
const { companyId, runId, admissionId, record } = input;
|
||||
const missing = await lockAdmissionProjects(tx, companyId, record.projectIds);
|
||||
if (missing.length > 0) return { kind: "projects_unavailable", projectIds: missing };
|
||||
const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).for("no key update");
|
||||
if (!run) return { kind: "run_not_found" };
|
||||
|
||||
const [existing] = await tx.select().from(admissions).where(eq(admissions.id, admissionId));
|
||||
if (existing) {
|
||||
if (existing.companyId !== companyId || existing.runId !== runId) return { kind: "admission_conflict" };
|
||||
if (existing.settledAt) return { kind: "admission_settled" };
|
||||
const charged = await tx.select({ projectId: charges.projectId }).from(charges)
|
||||
.where(and(eq(charges.companyId, companyId), eq(charges.admissionId, admissionId))).orderBy(asc(charges.projectId));
|
||||
const stored = { primaryProjectId: existing.primaryProjectId, contextRevision: existing.contextRevision,
|
||||
projectIds: charged.map(row => row.projectId) };
|
||||
return sameAdmissionRecord(stored, record) ? { kind: "replayed", attempt: existing.attempt } : { kind: "admission_conflict" };
|
||||
}
|
||||
const [live] = await tx.select({ id: admissions.id }).from(admissions)
|
||||
.where(and(eq(admissions.runId, runId), isNull(admissions.settledAt)));
|
||||
if (live) return { kind: "run_already_admitted", admissionId: live.id };
|
||||
|
||||
const [{ attempt }] = await tx.select({ attempt: sql<number>`coalesce(max(${admissions.attempt}), 0)::int + 1` })
|
||||
.from(admissions).where(eq(admissions.runId, runId));
|
||||
await tx.insert(admissions).values({ id: admissionId, companyId, runId, attempt,
|
||||
primaryProjectId: record.primaryProjectId, contextRevision: record.contextRevision });
|
||||
if (record.projectIds.length > 0) {
|
||||
await tx.insert(charges).values(record.projectIds.map(projectId => ({ admissionId, companyId, projectId })));
|
||||
}
|
||||
return { kind: "recorded", attempt };
|
||||
}
|
||||
|
||||
export type SettleAdmissionOutcome =
|
||||
| { kind: "settled"; releasedProjectIds: string[] }
|
||||
| { kind: "already_settled" | "settled_by_other" | "not_found" };
|
||||
|
||||
/**
|
||||
* Storage transition only: releases the admission's whole charge set once.
|
||||
* Callers must first verify authoritative lifecycle facts. Takes no project lock.
|
||||
*/
|
||||
export async function settleAdmission(tx: Db, input: { companyId: string; admissionId: string; settlementId: string }):
|
||||
Promise<SettleAdmissionOutcome> {
|
||||
assertTransaction(tx);
|
||||
const { companyId, admissionId, settlementId } = input;
|
||||
const [row] = await tx.select({ settlementId: admissions.settlementId }).from(admissions)
|
||||
.where(and(eq(admissions.companyId, companyId), eq(admissions.id, admissionId))).for("no key update");
|
||||
if (!row) return { kind: "not_found" };
|
||||
const decision = decideSettlement(row.settlementId, settlementId);
|
||||
if (decision !== "settle") return { kind: decision };
|
||||
const released = await tx.delete(charges)
|
||||
.where(and(eq(charges.companyId, companyId), eq(charges.admissionId, admissionId)))
|
||||
.returning({ projectId: charges.projectId });
|
||||
await tx.update(admissions).set({ settlementId, settledAt: sql`now()` }).where(eq(admissions.id, admissionId));
|
||||
return { kind: "settled", releasedProjectIds: released.map(charge => charge.projectId).sort() };
|
||||
}
|
||||
|
||||
/** Active charge counts per project. Every stored charge is unsettled. */
|
||||
export async function countActiveProjectCharges(db: Db, companyId: string, projectIds: readonly string[]) {
|
||||
const counts = new Map(projectIds.map(id => [id, 0]));
|
||||
if (projectIds.length === 0) return counts;
|
||||
const rows = await db.select({ projectId: charges.projectId, active: sql<number>`count(*)::int` }).from(charges)
|
||||
.where(and(eq(charges.companyId, companyId), inArray(charges.projectId, [...projectIds])))
|
||||
.groupBy(charges.projectId);
|
||||
for (const row of rows) counts.set(row.projectId, row.active);
|
||||
return counts;
|
||||
}
|
||||
|
||||
/** Project deletion holds the project row lock, so no charge can be added concurrently. */
|
||||
export async function hasUnsettledProjectCharges(db: Db, scope: { companyId: string; projectId: string }) {
|
||||
const [row] = await db.select({ admissionId: charges.admissionId }).from(charges)
|
||||
.where(and(eq(charges.companyId, scope.companyId), eq(charges.projectId, scope.projectId))).limit(1);
|
||||
return Boolean(row);
|
||||
}
|
||||
|
||||
/** Removes settled history of an agent's runs before they are deleted; refuses while any admission is unsettled. */
|
||||
export async function deleteAgentAdmissionHistory(tx: Db, scope: { companyId: string; agentId: string }) {
|
||||
assertTransaction(tx);
|
||||
const agentRuns = tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns)
|
||||
.where(and(eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.agentId, scope.agentId)));
|
||||
const [live] = await tx.select({ id: admissions.id }).from(admissions)
|
||||
.where(and(eq(admissions.companyId, scope.companyId), inArray(admissions.runId, agentRuns), isNull(admissions.settledAt)))
|
||||
.limit(1);
|
||||
if (live) return { kind: "unsettled" as const };
|
||||
await tx.delete(admissions).where(and(eq(admissions.companyId, scope.companyId), inArray(admissions.runId, agentRuns)));
|
||||
return { kind: "deleted" as const };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import {
|
||||
applyTaskExecutionPolicyPatch,
|
||||
parseTaskExecutionPolicyPatch,
|
||||
type TaskExecutionPolicy,
|
||||
} from "../domain/policy.js";
|
||||
|
||||
export type ProjectScope = { companyId: string; projectId: string };
|
||||
|
||||
export interface TaskExecutionPolicyStore {
|
||||
/** Null when the project is not in the company. */
|
||||
read(scope: ProjectScope): Promise<TaskExecutionPolicy | null>;
|
||||
/**
|
||||
* Locks the project in admission lock order and stores `change(current)`.
|
||||
* Returning `current` itself writes nothing. Null when the project is not in the company.
|
||||
*/
|
||||
update(scope: ProjectScope, change: (current: TaskExecutionPolicy) => TaskExecutionPolicy): Promise<TaskExecutionPolicy | null>;
|
||||
}
|
||||
|
||||
export type PolicyReadResult = { kind: "found"; policy: TaskExecutionPolicy } | { kind: "project_not_found" };
|
||||
|
||||
export type PolicyUpdateResult =
|
||||
| { kind: "updated" | "unchanged"; policy: TaskExecutionPolicy }
|
||||
| { kind: "invalid"; field: "policy" | "maxConcurrentTasks"; reason: string }
|
||||
| { kind: "project_not_found" };
|
||||
|
||||
/**
|
||||
* Internal policy commands. No route, tool or import calls `update` until
|
||||
* admission enforces the maximum; a cap written earlier would mislead users.
|
||||
*/
|
||||
export function createTaskExecutionPolicyCommands(store: TaskExecutionPolicyStore) {
|
||||
return {
|
||||
async readTaskExecutionPolicy(scope: ProjectScope): Promise<PolicyReadResult> {
|
||||
const policy = await store.read(scope);
|
||||
return policy ? { kind: "found", policy } : { kind: "project_not_found" };
|
||||
},
|
||||
|
||||
async updateTaskExecutionPolicy(scope: ProjectScope, input: unknown): Promise<PolicyUpdateResult> {
|
||||
const parsed = parseTaskExecutionPolicyPatch(input);
|
||||
if (parsed.kind === "invalid") return parsed;
|
||||
let changed = false;
|
||||
const policy = await store.update(scope, current => {
|
||||
const next = applyTaskExecutionPolicyPatch(current, parsed.patch);
|
||||
changed = next !== current;
|
||||
return next;
|
||||
});
|
||||
if (!policy) return { kind: "project_not_found" };
|
||||
return { kind: changed ? "updated" : "unchanged", policy };
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
export { executionAdmissionCompanyDeletion } from "./adapters/company-deletion.js";
|
||||
@@ -0,0 +1,53 @@
|
||||
/** Referenced projects are rejected beyond this bound instead of admitting a subset. */
|
||||
export const MAX_ADMISSION_PROJECTS = 100;
|
||||
const MAX_CONTEXT_REVISION_LENGTH = 256;
|
||||
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
export type AdmissionRecordInput = {
|
||||
primaryProjectId: string | null;
|
||||
referencedProjectIds: readonly string[];
|
||||
contextRevision: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* `projectIds` is the complete distinct set in canonical lock order. Lowercase
|
||||
* UUID text order matches PostgreSQL uuid order. An empty set is valid and
|
||||
* charges no project.
|
||||
*/
|
||||
export type AdmissionRecord = {
|
||||
primaryProjectId: string | null;
|
||||
projectIds: string[];
|
||||
contextRevision: string;
|
||||
};
|
||||
|
||||
export type AdmissionRecordResult =
|
||||
| { kind: "ok"; record: AdmissionRecord }
|
||||
| { kind: "invalid_project_id" }
|
||||
| { kind: "too_many_projects"; count: number; limit: number }
|
||||
| { kind: "invalid_context_revision" };
|
||||
|
||||
export function prepareAdmissionRecord(input: AdmissionRecordInput): AdmissionRecordResult {
|
||||
const contextRevision = input.contextRevision.trim();
|
||||
if (!contextRevision || contextRevision.length > MAX_CONTEXT_REVISION_LENGTH) return { kind: "invalid_context_revision" };
|
||||
const primaryProjectId = input.primaryProjectId?.toLowerCase() ?? null;
|
||||
const candidates = [...input.referencedProjectIds, ...(primaryProjectId === null ? [] : [primaryProjectId])].map(id => id.toLowerCase());
|
||||
if (candidates.some(id => !UUID.test(id))) return { kind: "invalid_project_id" };
|
||||
const projectIds = [...new Set(candidates)].sort();
|
||||
if (projectIds.length > MAX_ADMISSION_PROJECTS) {
|
||||
return { kind: "too_many_projects", count: projectIds.length, limit: MAX_ADMISSION_PROJECTS };
|
||||
}
|
||||
return { kind: "ok", record: { primaryProjectId, projectIds, contextRevision } };
|
||||
}
|
||||
|
||||
export function sameAdmissionRecord(left: AdmissionRecord, right: AdmissionRecord) {
|
||||
return left.primaryProjectId === right.primaryProjectId && left.contextRevision === right.contextRevision
|
||||
&& left.projectIds.length === right.projectIds.length && left.projectIds.every((id, index) => id === right.projectIds[index]);
|
||||
}
|
||||
|
||||
export type SettlementDecision = "settle" | "already_settled" | "settled_by_other";
|
||||
|
||||
/** A settlement identity can be replayed; a second, different settlement cannot take over. */
|
||||
export function decideSettlement(existingSettlementId: string | null, settlementId: string): SettlementDecision {
|
||||
if (existingSettlementId === null) return "settle";
|
||||
return existingSettlementId === settlementId ? "already_settled" : "settled_by_other";
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { decideSettlement, MAX_ADMISSION_PROJECTS, prepareAdmissionRecord } from "./admission.js";
|
||||
import { ABSENT_TASK_EXECUTION_POLICY, applyTaskExecutionPolicyPatch, parseTaskExecutionPolicyPatch, type TaskExecutionPolicy } from "./policy.js";
|
||||
|
||||
const configured = (maxConcurrentTasks: number | null): TaskExecutionPolicy => ({ state: "configured", maxConcurrentTasks });
|
||||
const apply = (current: TaskExecutionPolicy, input: unknown) => {
|
||||
const parsed = parseTaskExecutionPolicyPatch(input);
|
||||
if (parsed.kind !== "ok") throw new Error(parsed.reason);
|
||||
return applyTaskExecutionPolicyPatch(current, parsed.patch);
|
||||
};
|
||||
|
||||
describe("task execution policy", () => {
|
||||
it("keeps never-configured distinct from explicit unlimited", () => {
|
||||
expect(apply(ABSENT_TASK_EXECUTION_POLICY, null)).toEqual(configured(null));
|
||||
expect(apply(ABSENT_TASK_EXECUTION_POLICY, { maxConcurrentTasks: null })).toEqual(configured(null));
|
||||
expect(apply(ABSENT_TASK_EXECUTION_POLICY, { maxConcurrentTasks: 3 })).toEqual(configured(3));
|
||||
});
|
||||
|
||||
it("preserves the current value for omitted policies and fields", () => {
|
||||
for (const current of [ABSENT_TASK_EXECUTION_POLICY, configured(null), configured(2)]) {
|
||||
expect(apply(current, undefined)).toBe(current);
|
||||
expect(apply(current, {})).toBe(current);
|
||||
expect(apply(current, { maxConcurrentTasks: undefined })).toBe(current);
|
||||
}
|
||||
const two = configured(2);
|
||||
expect(apply(two, { maxConcurrentTasks: 2 })).toBe(two);
|
||||
const unlimited = configured(null);
|
||||
expect(apply(unlimited, null)).toBe(unlimited);
|
||||
});
|
||||
|
||||
it("clears or replaces a configured maximum", () => {
|
||||
expect(apply(configured(2), null)).toEqual(configured(null));
|
||||
expect(apply(configured(2), { maxConcurrentTasks: null })).toEqual(configured(null));
|
||||
expect(apply(configured(null), { maxConcurrentTasks: 1 })).toEqual(configured(1));
|
||||
});
|
||||
|
||||
it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2_147_483_648, "2", true, {}])(
|
||||
"rejects maximum %s", value => {
|
||||
expect(parseTaskExecutionPolicyPatch({ maxConcurrentTasks: value })).toMatchObject({ kind: "invalid", field: "maxConcurrentTasks" });
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects malformed policies and unknown fields", () => {
|
||||
for (const input of [1, "unlimited", [], { maxConcurrentTasks: 1, perTaskOverride: 3 }]) {
|
||||
expect(parseTaskExecutionPolicyPatch(input)).toMatchObject({ kind: "invalid", field: "policy" });
|
||||
}
|
||||
expect(parseTaskExecutionPolicyPatch({ maxConcurrentTasks: 2_147_483_647 })).toEqual({ kind: "ok", patch: { maxConcurrentTasks: 2_147_483_647 } });
|
||||
});
|
||||
});
|
||||
|
||||
describe("admission record", () => {
|
||||
const a = "0b9d8e52-6f2e-4bb0-9a4a-1f0a7f2e0a01";
|
||||
const b = "7c1f3d9a-2b4e-4d8f-8a6b-3e5c7d9f1b02";
|
||||
const c = "f3a2b1c0-9d8e-4f7a-8b6c-5d4e3f2a1b03";
|
||||
|
||||
it("unions the primary with referenced projects in canonical order", () => {
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: c, referencedProjectIds: [b, a, b.toUpperCase(), c], contextRevision: " r1 " }))
|
||||
.toEqual({ kind: "ok", record: { primaryProjectId: c, projectIds: [a, b, c], contextRevision: "r1" } });
|
||||
});
|
||||
|
||||
it("keeps a primaryless set and an empty set without a project bucket", () => {
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [b], contextRevision: "r" }))
|
||||
.toEqual({ kind: "ok", record: { primaryProjectId: null, projectIds: [b], contextRevision: "r" } });
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [], contextRevision: "r" }))
|
||||
.toEqual({ kind: "ok", record: { primaryProjectId: null, projectIds: [], contextRevision: "r" } });
|
||||
});
|
||||
|
||||
it("rejects invalid references, oversized sets and missing context identity instead of narrowing", () => {
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [a, "Design"], contextRevision: "r" }))
|
||||
.toEqual({ kind: "invalid_project_id" });
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: "", referencedProjectIds: [a], contextRevision: "r" }))
|
||||
.toEqual({ kind: "invalid_project_id" });
|
||||
const many = Array.from({ length: MAX_ADMISSION_PROJECTS + 1 }, (_, index) =>
|
||||
`00000000-0000-4000-8000-${index.toString(16).padStart(12, "0")}`);
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: many, contextRevision: "r" }))
|
||||
.toEqual({ kind: "too_many_projects", count: MAX_ADMISSION_PROJECTS + 1, limit: MAX_ADMISSION_PROJECTS });
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: a, referencedProjectIds: [], contextRevision: " " }))
|
||||
.toEqual({ kind: "invalid_context_revision" });
|
||||
expect(prepareAdmissionRecord({ primaryProjectId: a, referencedProjectIds: [], contextRevision: "x".repeat(257) }))
|
||||
.toEqual({ kind: "invalid_context_revision" });
|
||||
});
|
||||
|
||||
it("settles once and replays only the same settlement", () => {
|
||||
expect(decideSettlement(null, a)).toBe("settle");
|
||||
expect(decideSettlement(a, a)).toBe("already_settled");
|
||||
expect(decideSettlement(a, b)).toBe("settled_by_other");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
/**
|
||||
* A project's task-execution policy. `absent` was never configured, so a later
|
||||
* legacy migration may still choose a value. `configured` with a null maximum
|
||||
* is an explicit "no limit" that such a migration must preserve.
|
||||
*/
|
||||
export type TaskExecutionPolicy =
|
||||
| { state: "absent"; maxConcurrentTasks: null }
|
||||
| { state: "configured"; maxConcurrentTasks: number | null };
|
||||
|
||||
/** Null clears the cap explicitly. An omitted field preserves its value. */
|
||||
export type TaskExecutionPolicyPatch = { maxConcurrentTasks?: number | null } | null;
|
||||
|
||||
export type PolicyPatchResult =
|
||||
| { kind: "ok"; patch: TaskExecutionPolicyPatch }
|
||||
| { kind: "invalid"; field: "policy" | "maxConcurrentTasks"; reason: string };
|
||||
|
||||
export const ABSENT_TASK_EXECUTION_POLICY: TaskExecutionPolicy = { state: "absent", maxConcurrentTasks: null };
|
||||
|
||||
// The column is a PostgreSQL integer.
|
||||
const MAX_STORED_MAXIMUM = 2_147_483_647;
|
||||
|
||||
/** Undefined (an omitted policy) preserves everything; null clears the cap. */
|
||||
export function parseTaskExecutionPolicyPatch(input: unknown): PolicyPatchResult {
|
||||
if (input === undefined) return { kind: "ok", patch: {} };
|
||||
if (input === null) return { kind: "ok", patch: null };
|
||||
if (typeof input !== "object" || Array.isArray(input)) {
|
||||
return { kind: "invalid", field: "policy", reason: "Policy must be an object or null" };
|
||||
}
|
||||
const unknownKey = Object.keys(input).find(key => key !== "maxConcurrentTasks");
|
||||
if (unknownKey) return { kind: "invalid", field: "policy", reason: `Unknown policy field ${unknownKey}` };
|
||||
if (!("maxConcurrentTasks" in input)) return { kind: "ok", patch: {} };
|
||||
const value = (input as { maxConcurrentTasks: unknown }).maxConcurrentTasks;
|
||||
if (value === undefined) return { kind: "ok", patch: {} };
|
||||
if (value === null) return { kind: "ok", patch: { maxConcurrentTasks: null } };
|
||||
if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > MAX_STORED_MAXIMUM) {
|
||||
return { kind: "invalid", field: "maxConcurrentTasks", reason: "Maximum concurrent tasks must be a positive whole number" };
|
||||
}
|
||||
return { kind: "ok", patch: { maxConcurrentTasks: value } };
|
||||
}
|
||||
|
||||
/** Applies a validated patch. Returns the current policy when nothing changes. */
|
||||
export function applyTaskExecutionPolicyPatch(current: TaskExecutionPolicy, patch: TaskExecutionPolicyPatch): TaskExecutionPolicy {
|
||||
if (patch !== null && patch.maxConcurrentTasks === undefined) return current;
|
||||
const maxConcurrentTasks = patch === null ? null : patch.maxConcurrentTasks ?? null;
|
||||
if (current.state === "configured" && current.maxConcurrentTasks === maxConcurrentTasks) return current;
|
||||
return { state: "configured", maxConcurrentTasks };
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { createPostgresTaskExecutionPolicyStore } from "./adapters/postgres.js";
|
||||
import { createTaskExecutionPolicyCommands } from "./application/policy.js";
|
||||
|
||||
export type { TaskExecutionPolicy, TaskExecutionPolicyPatch } from "./domain/policy.js";
|
||||
export type { PolicyReadResult, PolicyUpdateResult, ProjectScope } from "./application/policy.js";
|
||||
// Lifecycle guards run inside the caller's deletion transaction.
|
||||
export { deleteAgentAdmissionHistory, hasUnsettledProjectCharges } from "./adapters/postgres.js";
|
||||
|
||||
/**
|
||||
* Execution admission owns project task-execution policy and the durable
|
||||
* admission epochs that charge project capacity. Admission and settlement
|
||||
* commands are not composed yet; nothing enforces the maximum.
|
||||
*/
|
||||
export function createExecutionAdmission(db: Db) {
|
||||
return createTaskExecutionPolicyCommands(createPostgresTaskExecutionPolicyStore(db));
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { agents, authUsers, companyMemberships, type Db } from "@paperclipai/db";
|
||||
import { agents, authUsers, companyMemberships, issues, type Db } from "@paperclipai/db";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
|
||||
const TOAST_ACTIONS = new Set(["issue.created", "issue.updated", "issue.comment_added"]);
|
||||
|
||||
@@ -11,8 +12,26 @@ export async function activityToastIdentity(
|
||||
) {
|
||||
const actorType = typeof payload.actorType === "string" ? payload.actorType : null;
|
||||
const actorId = typeof payload.actorId === "string" ? payload.actorId : null;
|
||||
const identity = { actorType, actorId };
|
||||
if (!actorId || !TOAST_ACTIONS.has(String(payload.action))) return identity;
|
||||
const actorIdentity = { actorType, actorId };
|
||||
if (!TOAST_ACTIONS.has(String(payload.action))) return actorIdentity;
|
||||
|
||||
const issueId = payload.entityType === "issue" && typeof payload.entityId === "string"
|
||||
? payload.entityId : null;
|
||||
let issueIdentifier: string | null = null;
|
||||
if (issueId) {
|
||||
try {
|
||||
const [issue] = await db
|
||||
.select({ identifier: issues.identifier })
|
||||
.from(issues)
|
||||
.where(and(eq(issues.companyId, companyId), eq(issues.id, issueId)))
|
||||
.limit(1);
|
||||
issueIdentifier = issue?.identifier ?? null;
|
||||
} catch (err) {
|
||||
logger.warn({ err, companyId, issueId }, "task toast identifier lookup failed");
|
||||
}
|
||||
}
|
||||
const identity = { ...actorIdentity, ...(issueId ? { issueIdentifier } : {}) };
|
||||
if (!actorId) return identity;
|
||||
|
||||
if (actorType === "agent") {
|
||||
const [agent] = await db
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { LiveEvent } from "@paperclipai/shared";
|
||||
import { createLiveEventDelivery, LIVE_DELIVERY_MAX_BYTES, LIVE_DELIVERY_MAX_ITEMS } from "./live-event-delivery.js";
|
||||
import { idleWorkSnapshot } from "../services/task-admission.js";
|
||||
|
||||
const event = (seq = 1, runId = "run-1"): LiveEvent => ({ id: seq, companyId: "company", type: "heartbeat.run.log",
|
||||
createdAt: "2026-10-24T00:00:00Z", payload: { runId, agentId: "agent", seq, chunk: "output" } });
|
||||
@@ -116,7 +117,7 @@ it.each([false, true])("preserves the overflow deadline across resubscription un
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
if (settled) expect(f.socket.close).not.toHaveBeenCalled();
|
||||
else expect(f.socket.close).toHaveBeenCalledWith(1013, "persistent event backlog");
|
||||
} finally { f.delivery.dispose(); }
|
||||
} finally { f.delivery.dispose(); release(event()); await settle(); }
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
it("recovers failed delivery reads through resync rather than silently claiming healthy streaming", async () => {
|
||||
@@ -146,6 +147,7 @@ it("rejects unbounded interest sets and stale revision commands", () => {
|
||||
it.each([false, true].flatMap(negotiated => ["authorize", "decorate"].map(stage => ({ negotiated, stage }))))(
|
||||
"closes a stalled $stage read below the queue bound (negotiated=$negotiated)", async ({ negotiated, stage }) => {
|
||||
vi.useFakeTimers();
|
||||
const baseline = idleWorkSnapshot().active;
|
||||
let release!: (value: LiveEvent) => void;
|
||||
const pending = new Promise<LiveEvent>(resolve => { release = resolve; });
|
||||
const f = fixture(stage === "authorize" ? vi.fn(() => pending) : undefined);
|
||||
@@ -168,11 +170,13 @@ it.each([false, true].flatMap(negotiated => ["authorize", "decorate"].map(stage
|
||||
expect(f.socket.close).toHaveBeenCalledWith(1013, "persistent event backlog");
|
||||
expect(f.delivery.metrics().retainedItems).toBe(1);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(idleWorkSnapshot().active).toBe(baseline + 1);
|
||||
release(source); await settle();
|
||||
expect(idleWorkSnapshot().active).toBe(baseline);
|
||||
expect(f.delivery.metrics()).toMatchObject({ retainedItems: 0, retainedBytes: 0 });
|
||||
expect(f.frames.filter(frame => frame.type.startsWith("heartbeat."))).toEqual([]);
|
||||
expect(f.authorize).toHaveBeenCalledOnce();
|
||||
} finally { f.delivery.dispose(); other.delivery.dispose(); }
|
||||
} finally { f.delivery.dispose(); other.delivery.dispose(); release(event()); await settle(); }
|
||||
});
|
||||
|
||||
it.each(["resolve", "dispose"])("clears a pending-read deadline on %s without idle work", async ending => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { LiveEvent } from "@paperclipai/shared";
|
||||
import { beginIdleTrackedWork } from "../services/task-admission.js";
|
||||
|
||||
export const LIVE_DELIVERY_MAX_ITEMS = 1_000;
|
||||
export const LIVE_DELIVERY_MAX_BYTES = 1024 * 1024;
|
||||
@@ -90,6 +91,7 @@ export function createLiveEventDelivery(input: {
|
||||
async function drain() {
|
||||
if (draining) return;
|
||||
draining = true;
|
||||
const finishIdleWork = beginIdleTrackedWork();
|
||||
try {
|
||||
while (queued.length && isOpen()) {
|
||||
const item = queued.shift()!;
|
||||
@@ -121,7 +123,10 @@ export function createLiveEventDelivery(input: {
|
||||
retainedBytes -= item.bytes; retainedItems -= 1;
|
||||
}
|
||||
}
|
||||
} finally { draining = false; resync(); }
|
||||
} finally {
|
||||
try { draining = false; resync(); }
|
||||
finally { finishIdleWork(); }
|
||||
}
|
||||
}
|
||||
return {
|
||||
receive(raw: string) {
|
||||
|
||||
@@ -48,7 +48,8 @@ import type {
|
||||
} from "@paperclipai/adapter-utils";
|
||||
import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { badRequest, forbidden } from "../errors.js";
|
||||
import { badRequest, forbidden, HttpError } from "../errors.js";
|
||||
import { isCloudManagedInstance } from "../services/cloud-instance.js";
|
||||
import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js";
|
||||
import { getHiddenSettings } from "../services/settings-visibility.js";
|
||||
import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js";
|
||||
@@ -281,7 +282,17 @@ export function adapterRoutes(options: {
|
||||
if (typeof companyId === "string") {
|
||||
assertCompanyAccess(_req, companyId);
|
||||
if (!options.resolveRunnerTarget) throw badRequest("Scoped adapter discovery is unavailable.");
|
||||
target = await options.resolveRunnerTarget(companyId, typeof environmentId === "string" ? environmentId : undefined);
|
||||
try {
|
||||
target = await options.resolveRunnerTarget(companyId, typeof environmentId === "string" ? environmentId : undefined);
|
||||
} catch (error) {
|
||||
// API controllers do not need a Computer. Keep their catalog visible
|
||||
// when only the implicit managed Computer is unavailable. This target
|
||||
// grants no local execution support to any workspace harness.
|
||||
if (environmentId !== undefined || !(error instanceof HttpError) || error.status !== 422
|
||||
|| (error.details as { code?: unknown } | undefined)?.code !== "managed_sandbox_unavailable") throw error;
|
||||
target = { driver: "unavailable", ...((error.details as { executionMode?: unknown }).executionMode === "kubernetes"
|
||||
? { executionMode: "kubernetes" as const } : {}) };
|
||||
}
|
||||
}
|
||||
await waitForExternalAdapters();
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { assertProviderApiPlacementPolicy, managedCursorApiDefaultEnabled, resolveProviderApiPlacement } from "../services/native-runtime/provider-api-placement.js";
|
||||
import { readTaskPanelActivations } from "../services/plugin-task-panels.js";
|
||||
import { enqueueNativeRuntimeResponse } from "../services/native-runtime/native-runtime-response-delivery.js";
|
||||
import { latestFailedRunsByAgent } from "@paperclipai/shared";
|
||||
@@ -1961,6 +1962,9 @@ export function agentRoutes(
|
||||
return input.nextAdapterConfig;
|
||||
}
|
||||
if (["openai_dot", "muse"].includes(String(input.nextAdapterConfig.provider))) return input.nextAdapterConfig;
|
||||
// A provisional setup harness supplies no Cursor model or permissions.
|
||||
// Legacy Cursor conversion still retains its provider-specific defaults.
|
||||
if (input.nextAdapterConfig.provider === "cursor_cloud" && input.previousAdapterType !== "cursor_cloud") return input.nextAdapterConfig;
|
||||
const defaults = paperclipRunnerTransitionConfig(input.previousAdapterType, input.previousAdapterConfig.model, input.nextAdapterConfig.provider);
|
||||
if (!["claude_local", "codex_local", "opencode_local", "pi_local", "grok_local"].includes(input.previousAdapterType)
|
||||
&& !isPaperclipRunnerProvider(input.nextAdapterConfig.provider)) {
|
||||
@@ -2206,6 +2210,7 @@ export function agentRoutes(
|
||||
);
|
||||
const incompleteModelSetup = input.setupOnly && input.adapterType === "paperclip_runner"
|
||||
&& (normalizedAdapterConfig.provider === "opencode"
|
||||
|| normalizedAdapterConfig.provider === "cursor_cloud"
|
||||
|| (normalizedAdapterConfig.provider === "acpx" && ["pi", "kimi"].includes(String(normalizedAdapterConfig.acpxAgent))))
|
||||
&& (normalizedAdapterConfig.model === undefined || normalizedAdapterConfig.model === "");
|
||||
// Inactive drafts can precede model selection. Test/hire always validate the complete profile.
|
||||
@@ -3275,7 +3280,25 @@ export function agentRoutes(
|
||||
}
|
||||
const testingSetup = savedAgent ? isAgentSetupDraft(savedAgent) : false;
|
||||
const poolBinding = requestedBinding ?? (savedAgent?.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(savedAgent.runtimeConfig.aiConnection) : undefined);
|
||||
const requestedEnvironmentId = await resolveAdapterTestEnvironmentId(
|
||||
const prospectiveCursorDefault = type === "cursor_cloud" && !savedAgent
|
||||
&& (req.body.runner === undefined || req.body.runner === "auto")
|
||||
&& managedCursorApiDefaultEnabled((await instanceSettings.get()).experimental);
|
||||
const testRunner = prospectiveCursorDefault ? "paperclip" : req.body.runner;
|
||||
let providerApiTest;
|
||||
try {
|
||||
// An explicit Legacy switch must resolve its Computer and credentials
|
||||
// before runner selection converts the saved native adapter.
|
||||
providerApiTest = testRunner === "legacy" ? null : resolveProviderApiPlacement({ agent: {
|
||||
adapterType: type === "cursor_cloud" && testRunner === "paperclip" ? "paperclip_runner" : type,
|
||||
adapterConfig: type === "cursor_cloud" && testRunner === "paperclip"
|
||||
? { ...inputAdapterConfig, provider: "cursor_cloud" } : inputAdapterConfig,
|
||||
} });
|
||||
} catch (error) {
|
||||
if (error instanceof PaperclipRunnerProviderProfileError) throw unprocessable(error.message, { code: error.code });
|
||||
throw error;
|
||||
}
|
||||
if (providerApiTest) assertProviderApiPlacementPolicy(await instanceSettings.getGeneral());
|
||||
const requestedEnvironmentId = providerApiTest ? null : await resolveAdapterTestEnvironmentId(
|
||||
companyId,
|
||||
// Omission tests the saved selection. Explicit null tests a prospective
|
||||
// change back to the instance default.
|
||||
@@ -3321,7 +3344,7 @@ export function agentRoutes(
|
||||
? { adapterType: type, adapterConfig: adapterConfigForTest }
|
||||
: await resolveNewAgentRunnerForCompany(db, companyId, {
|
||||
adapterType: type, adapterConfig: adapterConfigForTest,
|
||||
runner: req.body.runner,
|
||||
runner: testRunner,
|
||||
defaultEnvironmentId: requestedEnvironmentId,
|
||||
});
|
||||
type = selection.adapterType;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { verifiedIdleCapabilities } from "../middleware/cloud-control.js";
|
||||
import { trackIdleWork } from "../services/task-admission.js";
|
||||
import { Router, type Request } from "express";
|
||||
import { companies, type Db } from "@paperclipai/db";
|
||||
@@ -336,7 +337,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag
|
||||
typeof req.query.ownerId === "string" ? req.query.ownerId : undefined,
|
||||
diagnostic => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup), diagnostic }),
|
||||
pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors,
|
||||
...(req.query.idleSleepDiagnostics === "1" ? [{ collectDiagnostics: true }] as const : [] as const));
|
||||
{ incomingWake: verifiedIdleCapabilities(req), deadlineCapability: verifiedIdleCapabilities(req),
|
||||
...(req.query.idleSleepDiagnostics === "1" ? { collectDiagnostics: true } : {}) });
|
||||
res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
import { and, eq } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { and, desc, eq, inArray, isNotNull } from "drizzle-orm";
|
||||
import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { parseObject } from "../adapters/utils.js";
|
||||
import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
||||
import { hasNativeLocalProcessStop } from "./native-local-process-stop.js";
|
||||
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
|
||||
import { validatePrpEvent } from "../vendor/paperclip-runner/index.js";
|
||||
import { nativeSha256 } from "./native-runtime/canonical.js";
|
||||
import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js";
|
||||
|
||||
/** A server-recorded run-only Stop must not manufacture a recovery incident. */
|
||||
export function hasAcknowledgedNativeRunStopReceipt(run: {
|
||||
@@ -37,6 +42,49 @@ export function isAcknowledgedNativeStop(run: Parameters<typeof hasAcknowledgedN
|
||||
return run.status === "cancelled" && hasAcknowledgedNativeStopIntent(run);
|
||||
}
|
||||
|
||||
/** API-owned execution has no Computer lease. Its provider terminal and retired
|
||||
* local controller must both be recorded before publishing shared Stop proof. */
|
||||
export async function acknowledgedNativeApiStopExecutionHasStopped(db: Db, run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "agentId" | "nativeIssueId" | "status" | "resultJson" | "processPid" | "processGroupId" |
|
||||
"runtimeMode" | "driverKind" | "runnerProfileJson" | "nativeSessionId" | "runnerInstanceId">): Promise<boolean> {
|
||||
if (run.runtimeMode !== "native" || run.driverKind !== "cursor_cloud_api" || !isAcknowledgedNativeStop(run) ||
|
||||
run.processPid !== null || run.processGroupId !== null || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return false;
|
||||
const intent = parseObject(run.resultJson?.nativeCancellation);
|
||||
const requestedAt = typeof intent.recordedAt === "string" ? Date.parse(intent.recordedAt) : NaN;
|
||||
if (!Number.isFinite(requestedAt) || !Array.isArray(intent.effects) || intent.effects.length !== 1 ||
|
||||
intent.effects[0] !== "release_run_resources" ||
|
||||
!String(intent.intentAuditId ?? "").trim() || !String(intent.acknowledgementAuditId ?? "").trim()) return false;
|
||||
const profile = parseObject(run.runnerProfileJson);
|
||||
const execution = parseObject(profile.nativeExecutionInput);
|
||||
const binding = parseObject(execution.binding), session = parseObject(execution.session);
|
||||
if (profile.executionPlacement === undefined || profile.backend !== "cursor_cloud_api" ||
|
||||
binding.companyId !== run.companyId || binding.runId !== run.id || binding.issueId !== run.nativeIssueId ||
|
||||
binding.agentId !== run.agentId || session.normalizedSessionId !== run.nativeSessionId || session.driverKind !== "cursor_cloud_api") return false;
|
||||
try {
|
||||
if (!resolveProviderApiPlacement({ run, agent: { adapterType: "process", adapterConfig: {} } })) return false;
|
||||
} catch { return false; }
|
||||
const [lease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and(
|
||||
eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id),
|
||||
)).limit(1);
|
||||
if (lease || !await hasNativeLocalProcessStop(db, run.companyId, run.id)) return false;
|
||||
const [terminal] = await db.select().from(heartbeatRunEvents).where(and(
|
||||
eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id),
|
||||
isNotNull(heartbeatRunEvents.sourceEventId),
|
||||
inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]),
|
||||
)).orderBy(desc(heartbeatRunEvents.seq)).limit(1);
|
||||
const validated = validatePrpEvent(parseObject(terminal?.payload).prpEvent);
|
||||
if (!terminal || !validated.ok || terminal.sourcePayloadSha256 !== nativeSha256(validated.event)) return false;
|
||||
const event = validated.event;
|
||||
return isConfirmedCursorCloudTerminalObservation({ eventType: terminal.eventType, payload: event.payload }) && terminal.agentId === run.agentId &&
|
||||
event.schema === "paperclip.prp.event.v1" && [1, 2].includes(Number(event.schemaVersion)) &&
|
||||
terminal.protocolSchemaVersion === event.schemaVersion && event.eventType === terminal.eventType &&
|
||||
event.sourceKind === "runner" && event.runId === run.id && event.normalizedSessionId === run.nativeSessionId &&
|
||||
event.sourceInstanceId === run.runnerInstanceId && terminal.sourceInstanceId === event.sourceInstanceId &&
|
||||
terminal.sourceEventId === event.sourceEventId && terminal.sourceSeq === event.sourceSeq &&
|
||||
Number.isSafeInteger(event.sourceSeq) && Number(event.sourceSeq) > 0 &&
|
||||
typeof event.turnId === "string" && Boolean(event.turnId);
|
||||
}
|
||||
|
||||
/** A Stop acknowledgement alone does not prove provider cleanup completed. */
|
||||
export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: Pick<typeof heartbeatRuns.$inferSelect,
|
||||
"id" | "companyId" | "nativeIssueId" | "status" | "resultJson" | "processPid" | "processGroupId">) {
|
||||
|
||||
@@ -91,11 +91,26 @@ export function activityService(db: Db) {
|
||||
`.as("usageJson");
|
||||
// A denied user needs the selected connection's name to repair their task.
|
||||
// Other readers need the connection's existing human sharing permission.
|
||||
// A malformed present Stop receipt must stay unconfirmed, not disappear.
|
||||
const summarizedResultJson = (viewerUserId: string | null) => sql<Record<string, unknown> | null>`
|
||||
case
|
||||
when ${heartbeatRuns.resultJson} is null then null
|
||||
else jsonb_strip_nulls(jsonb_build_object(
|
||||
'conversationReset', ${heartbeatRuns.resultJson} -> 'conversationReset',
|
||||
'executionCancellation', case when ${heartbeatRuns.resultJson} ? 'executionCancellation'
|
||||
then jsonb_build_object('state', case
|
||||
when jsonb_typeof(${heartbeatRuns.resultJson} -> 'executionCancellation') = 'object'
|
||||
and ${heartbeatRuns.resultJson} #>> '{executionCancellation,state}' in ('requested', 'acknowledged')
|
||||
then ${heartbeatRuns.resultJson} #>> '{executionCancellation,state}' else 'unknown' end) end,
|
||||
'nativeCancellation', case
|
||||
when jsonb_typeof(${heartbeatRuns.resultJson} -> 'nativeCancellation') = 'object'
|
||||
and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,schema}' = 'paperclip.native-cancellation.v1'
|
||||
and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,runId}' = ${heartbeatRuns.id}::text
|
||||
and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,companyId}' = ${heartbeatRuns.companyId}::text
|
||||
and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,dispatchState}' in ('pending', 'acknowledged')
|
||||
then jsonb_build_object('schema', 'paperclip.native-cancellation.v1',
|
||||
'runId', ${heartbeatRuns.id}, 'companyId', ${heartbeatRuns.companyId},
|
||||
'dispatchState', ${heartbeatRuns.resultJson} #>> '{nativeCancellation,dispatchState}') end,
|
||||
'configurationIncomplete', case
|
||||
when ${heartbeatRuns.resultJson} -> 'configurationIncomplete' ->> 'reason' = 'workspace_base_ref_unresolved'
|
||||
then jsonb_build_object('reason', 'workspace_base_ref_unresolved')
|
||||
|
||||
@@ -8,7 +8,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js";
|
||||
|
||||
const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() }));
|
||||
const legacy = vi.hoisted(() => ({ extraTypes: [] as string[], get: vi.fn(() => [] as string[]), set: vi.fn() }));
|
||||
vi.mock("./adapter-plugin-store.js", () => ({
|
||||
getDisabledAdapterTypes: legacy.get,
|
||||
setAdapterDisabled: legacy.set,
|
||||
@@ -16,7 +16,7 @@ vi.mock("./adapter-plugin-store.js", () => ({
|
||||
getAdapterPluginByType: () => undefined,
|
||||
}));
|
||||
vi.mock("../adapters/registry.js", () => ({
|
||||
listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }],
|
||||
listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }, ...legacy.extraTypes.map(type => ({ type }))],
|
||||
findServerAdapter: (type: string) => ({ type }),
|
||||
findActiveServerAdapter: (type: string) => ({ type }),
|
||||
isOverridePaused: () => false,
|
||||
@@ -56,6 +56,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos
|
||||
beforeEach(async () => {
|
||||
directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-"));
|
||||
await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1"));
|
||||
legacy.extraTypes = [];
|
||||
legacy.get.mockClear();
|
||||
legacy.set.mockClear();
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token");
|
||||
@@ -188,6 +189,58 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos
|
||||
expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local");
|
||||
});
|
||||
|
||||
it("refreshes API-only Cursor policy from PostgreSQL and authoritative operator declarations", async () => {
|
||||
legacy.extraTypes = ["cursor_cloud"];
|
||||
await service().seedIfAbsent({ disabledTypes: [] });
|
||||
await db.insert(instanceSettings).values({ singletonKey: "default", general: {},
|
||||
experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } })
|
||||
.onConflictDoUpdate({ target: instanceSettings.singletonKey, set: { general: {},
|
||||
experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } } });
|
||||
const draft = { adapterType: "cursor_cloud", runner: "auto" as const, setupOnly: true, adapterConfig: {}, disabledTypes: [] };
|
||||
const saved = { adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/acme/repo", lifecycleMode: "per_turn" } };
|
||||
const admitted = async (input = draft as typeof draft | typeof saved) => {
|
||||
expect(await resolveNewAgentRunnerForCompany(db, "company", input))
|
||||
.toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } });
|
||||
};
|
||||
const denied = async (input = draft as typeof draft | typeof saved) => {
|
||||
await expect(resolveNewAgentRunnerForCompany(db, "company", input))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
|
||||
};
|
||||
await admitted(); // The default managed tenant has no Computer or CLI probe.
|
||||
for (const type of ["cursor_cloud", "paperclip_runner"]) {
|
||||
await adapterSettingsService(peer, managed).setDisabled(type, true);
|
||||
await denied();
|
||||
await denied(saved);
|
||||
await adapterSettingsService(peer, managed).setDisabled(type, false);
|
||||
await admitted();
|
||||
}
|
||||
await adapterSettingsService(peer, managed).setDisabled("cursor_cloud", true);
|
||||
const stored = await service().inspectStored();
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", JSON.stringify([
|
||||
{ adapterType: "cursor_cloud", enabled: true }, { adapterType: "paperclip_runner", enabled: true },
|
||||
]));
|
||||
await admitted(); // Operator allowlist supersedes, rather than unions, saved disables.
|
||||
await admitted(saved);
|
||||
expect(await service().inspectStored()).toEqual(stored);
|
||||
for (const declaration of [
|
||||
[{ adapterType: "cursor_cloud", enabled: true }],
|
||||
[{ adapterType: "paperclip_runner", enabled: true }],
|
||||
[{ adapterType: "cursor_cloud", enabled: false }, { adapterType: "paperclip_runner", enabled: true }],
|
||||
[{ adapterType: "cursor_cloud", enabled: true }, { adapterType: "paperclip_runner", enabled: false }],
|
||||
]) {
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", JSON.stringify(declaration));
|
||||
await denied();
|
||||
await denied(saved);
|
||||
}
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", "invalid operator declaration");
|
||||
await expect(resolveNewAgentRunnerForCompany(db, "company", draft)).rejects.toThrow();
|
||||
vi.stubEnv("PAPERCLIP_ADAPTERS", undefined);
|
||||
await denied(); // Removing the overlay restores the saved choice.
|
||||
expect(await service().inspectStored()).toEqual(stored);
|
||||
expect(legacy.get).not.toHaveBeenCalled();
|
||||
expect(legacy.set).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rolls back the setting when its audit write fails", async () => {
|
||||
await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed");
|
||||
expect(await service().inspectStored()).toBeNull();
|
||||
|
||||
@@ -14,6 +14,7 @@ import { agents, agentApiKeys, agentRuntimeState, agentTaskSessions, agentWakeup
|
||||
caseHandoffs, principalPermissionGrants, type Db } from "@paperclipai/db";
|
||||
import { conflict } from "../errors.js";
|
||||
import { issueThreadInteractionService } from "./issue-thread-interactions.js";
|
||||
import { deleteAgentAdmissionHistory } from "../modules/execution-admission/index.js";
|
||||
|
||||
export async function deleteAgentDependencies(tx: Db, companyId: string, id: string) {
|
||||
const [retainedHandoff] = await tx.select({ id: caseHandoffs.id }).from(caseHandoffs).where(and(
|
||||
@@ -63,6 +64,11 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str
|
||||
await notifyInstructionCollection(tx, { companyId: companyId });
|
||||
await notifyCancelledWorkspacePreservation(tx, { companyId: companyId });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId });
|
||||
if ((await deleteAgentAdmissionHistory(tx, { companyId, agentId: id })).kind === "unsettled") {
|
||||
throw conflict("Wait for this agent's admitted task executions to settle before deleting it", {
|
||||
code: "agent_execution_admission_pending",
|
||||
});
|
||||
}
|
||||
await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.agentId, id));
|
||||
await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id));
|
||||
await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id));
|
||||
|
||||
@@ -10,6 +10,8 @@ import { aiConnectionService } from "./ai-connections.js";
|
||||
import { withManagedAiProbe, stripAiAuthBindings } from "./ai-connection-runtime.js";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { secretService } from "./secrets.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { assertProviderApiPlacementPolicy, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
||||
import { museRunnerBroker } from "./muse-runner-broker.js";
|
||||
import { dotRunnerBroker } from "./dot-runner-broker.js";
|
||||
import type { PluginWorkerManager } from "./plugin-worker-manager.js";
|
||||
@@ -51,6 +53,8 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan
|
||||
const binding = await museRunnerBroker(db).bindingForAgent(agent.companyId,agent.id);
|
||||
return binding?.status === "ready" && binding.paired && binding.receiverDetected && binding.backgroundReplyVerified && binding.id === agent.adapterConfig.museBindingId ? "complete" : "pending";
|
||||
}
|
||||
const providerApi = resolveProviderApiPlacement({ agent });
|
||||
if (providerApi) assertProviderApiPlacementPolicy(await instanceSettingsService(db).getGeneral());
|
||||
const [company] = await db.select().from(companies).where(eq(companies.id, agent.companyId));
|
||||
const responsibleUserId = agent.lifecycleOperation?.responsibleUserId ?? company.defaultResponsibleUserId;
|
||||
const context = { consumerType: "agent" as const, consumerId: agent.id, actorType: "system" as const,
|
||||
@@ -66,7 +70,7 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan
|
||||
}
|
||||
if (binding) config = { ...config, env: stripAiAuthBindings(config.env) };
|
||||
config = (await secrets.resolveAdapterConfigForRuntime(agent.companyId, config, context, { adapterType: agent.adapterType })).config;
|
||||
const environmentId = await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId, agent.adapterType === "paperclip_runner" && config.provider === "acpx" && config.acpxAgent === "copilot");
|
||||
const environmentId = providerApi ? null : await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId, agent.adapterType === "paperclip_runner" && config.provider === "acpx" && config.acpxAgent === "copilot");
|
||||
if (environmentId) {
|
||||
await tests.assertAdapterTestEnvironmentForCompany(agent.companyId, environmentId, agent.adapterType);
|
||||
const environment = await environments.getById(environmentId);
|
||||
@@ -81,7 +85,9 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan
|
||||
adapterType: agent.adapterType, model: config.model, runnerProvider: config.provider,
|
||||
acpxAgent: config.acpxAgent, binding,
|
||||
});
|
||||
const target = await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId });
|
||||
const target = providerApi
|
||||
? { executionTarget: null, environmentName: null, fallbackChecks: [], release: async (_status?: "released" | "failed") => {} }
|
||||
: await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId });
|
||||
let status: "released" | "failed" = "failed";
|
||||
try {
|
||||
if (target.fallbackChecks.length) throw Object.assign(new Error("The test environment is unavailable"), { code: "test_environment_unavailable" });
|
||||
|
||||
@@ -17,6 +17,8 @@ import { agentCredentialService } from "./agent-credentials.js";
|
||||
import { agentIdentityService } from "./agent-identity.js";
|
||||
import { initializePrimaryAgent } from "./primary-agent.js";
|
||||
import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { assertProviderApiPlacementPolicy, isProviderApiSetupDraft, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
||||
|
||||
export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise<CreateAgentData> {
|
||||
await notifyAttentionRetentionWork(db, { companyId });
|
||||
@@ -49,7 +51,11 @@ export async function prepareAgentHire(db: Db, companyId: string, input: CreateA
|
||||
? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType })
|
||||
: {};
|
||||
const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig);
|
||||
if (adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) &&
|
||||
const profile = { adapterType, adapterConfig };
|
||||
const providerApiDraft = options?.setupOnly === true && isProviderApiSetupDraft(profile);
|
||||
const providerApi = providerApiDraft ? null : resolveProviderApiPlacement({ agent: profile });
|
||||
if (providerApi) assertProviderApiPlacementPolicy(await instanceSettingsService(db).getGeneral());
|
||||
if (!providerApiDraft && !providerApi && adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) &&
|
||||
(await resolveAgentRunnerTaskDriverForCompany(db, companyId, data.defaultEnvironmentId))?.warmTaskVersion === 1) {
|
||||
adapterConfig.lifecycleMode = "warm";
|
||||
}
|
||||
|
||||
@@ -24,13 +24,15 @@ import { trackIdleWork } from "./task-admission.js";
|
||||
import { environmentService } from "./environments.js";
|
||||
import { instanceSettingsService } from "./instance-settings.js";
|
||||
import { pluginEnvironmentRequiresAgent } from "./plugin-environment-driver.js";
|
||||
import { isProviderApiSetupDraft } from "./native-runtime/provider-api-placement.js";
|
||||
|
||||
export { scheduleAgentLifecycle, processAgentLifecycle } from "../modules/agent-lifecycle/index.js";
|
||||
export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle/index.js";
|
||||
|
||||
export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects {
|
||||
return {
|
||||
async setupRequiresPreparation(db, companyId, environmentId) {
|
||||
async setupRequiresPreparation(db, companyId, environmentId, profile) {
|
||||
if (isProviderApiSetupDraft(profile)) return false;
|
||||
const id = environmentId ?? (await instanceSettingsService(db).get()).defaultEnvironmentId;
|
||||
const environment = id ? await environmentService(db).getById(id) : null;
|
||||
if (!environment || !await pluginEnvironmentRequiresAgent(db, environment)) return false;
|
||||
|
||||
@@ -46,18 +46,96 @@ describe("server-owned Codex runner selection", () => {
|
||||
const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo", repoStartingRef: "feature", runtimeEnvType: "cloud", env: { CURSOR_API_KEY: { type: "secret_ref", secretId: "saved-key", version: "latest" } }, instructionsFilePath: "/assigned/AGENTS.md", autoCreatePR: false };
|
||||
expect(agentRunnerAvailability("cursor_cloud", target)).toEqual({ supportedRunners: ["paperclip", "legacy"], defaultRunner: "legacy" });
|
||||
expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, target })).toEqual({ adapterType: "cursor_cloud", adapterConfig: config });
|
||||
for (const runner of [undefined, "auto"] as const) {
|
||||
for (const driver of ["unavailable", "sandbox", "ssh"]) {
|
||||
expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner, target: { driver } }))
|
||||
.toThrow(expect.objectContaining({ status: 422, details: { code: "agent_runner_unavailable" } }));
|
||||
}
|
||||
}
|
||||
expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "legacy", target: { driver: "sandbox" } }))
|
||||
.toEqual({ adapterType: "cursor_cloud", adapterConfig: config });
|
||||
const converted = resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target });
|
||||
expect(converted).toEqual({ adapterType: "paperclip_runner", adapterConfig: { model: config.model, repoUrl: config.repoUrl, repoRef: "feature", cloudEnvType: "cloud", env: config.env, instructionsFilePath: config.instructionsFilePath, autoCreatePR: false, provider: "cursor_cloud", lifecycleMode: "per_turn" } });
|
||||
expect(resolveNewAgentRunner({ ...converted, runner: "legacy", target })).toEqual({ adapterType: "cursor_cloud", adapterConfig: config });
|
||||
});
|
||||
it("rejects unsupported Cursor Cloud conversions and remote controllers without changing the original", () => {
|
||||
it("rejects unsupported Cursor Cloud settings and keeps API placement independent of the selected Computer", () => {
|
||||
const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo" };
|
||||
for (const patch of [{ promptTemplate: "custom instructions" }, { runtimeEnvType: "pool" }, { model: "" }]) {
|
||||
expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: { ...config, ...patch }, runner: "paperclip", target })).toThrow();
|
||||
}
|
||||
expect(config).toEqual({ model: "caller-model", repoUrl: "https://github.com/acme/repo" });
|
||||
expect(agentRunnerAvailability("cursor_cloud", { ...target, driver: "ssh" })).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target: { ...target, driver: "ssh" } })).toThrow("unavailable");
|
||||
expect(agentRunnerAvailability("cursor_cloud", { ...target, driver: "ssh" })).toEqual({ supportedRunners: ["paperclip"], defaultRunner: "paperclip" });
|
||||
expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target: { ...target, driver: "ssh" } })).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud" } });
|
||||
});
|
||||
it("saves native Cursor Cloud without probing or binding a managed Computer", async () => {
|
||||
state.settings = { defaultEnvironmentId: "managed", experimental: { enableManagedSandboxOnly: true } };
|
||||
const saved = await resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType: "cursor_cloud", runner: "paperclip", defaultEnvironmentId: "managed",
|
||||
adapterConfig: { model: "caller-model", repoUrl: "https://github.com/acme/repo" },
|
||||
});
|
||||
expect(saved).toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } });
|
||||
expect(state.getEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.ssh).not.toHaveBeenCalled();
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
});
|
||||
it("refreshes managed adapter policy before API-only Cursor admission without a Computer", async () => {
|
||||
vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-cloud-test-token");
|
||||
state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
state.managedEnvironment.mockResolvedValue(null);
|
||||
const input = { adapterType: "cursor_cloud", runner: "auto" as const, setupOnly: true, adapterConfig: {}, disabledTypes: [] };
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", input))
|
||||
.toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } });
|
||||
for (const disabled of ["cursor_cloud", "paperclip_runner"]) {
|
||||
state.disabled = [disabled];
|
||||
await expect(resolveNewAgentRunnerForCompany({} as never, "company", input))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
|
||||
}
|
||||
await expect(resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/acme/repo", lifecycleMode: "per_turn" },
|
||||
})).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } });
|
||||
expect(state.managedEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.getEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.resolveEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.ssh).not.toHaveBeenCalled();
|
||||
expect(state.sandboxQualification).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([undefined, "auto"] as const)("defaults a new managed Cursor setup draft to the native API controller without a Computer (%s)", async runner => {
|
||||
state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
state.managedEnvironment.mockResolvedValue(null);
|
||||
const saved = await resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType: "cursor_cloud", runner, setupOnly: true, adapterConfig: {},
|
||||
});
|
||||
expect(saved).toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null,
|
||||
adapterConfig: { provider: "cursor_cloud", lifecycleMode: "per_turn" } });
|
||||
expect(state.managedEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.getEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
it.each([
|
||||
{ setupOnly: false, enabled: true },
|
||||
{ setupOnly: true, enabled: false },
|
||||
])("does not use the new-draft Cursor API default outside its admission scope (%j)", async ({ setupOnly, enabled }) => {
|
||||
state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: enabled } };
|
||||
state.managedEnvironment.mockResolvedValue(null);
|
||||
await expect(resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType: "cursor_cloud", runner: "auto", setupOnly, adapterConfig: {},
|
||||
})).rejects.toMatchObject({ status: 422, details: { code: "managed_sandbox_unavailable" } });
|
||||
});
|
||||
it("preserves an explicit Legacy choice on a held Cursor setup draft", async () => {
|
||||
state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } };
|
||||
expect(await resolveNewAgentRunnerForCompany({} as never, "company", {
|
||||
adapterType: "cursor_cloud", runner: "legacy", setupOnly: true, adapterConfig: {},
|
||||
})).toEqual({ adapterType: "cursor_cloud", adapterConfig: {} });
|
||||
expect(state.managedEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
it("refuses native Cursor save and availability under Kubernetes-only policy without provisioning", async () => {
|
||||
state.settings = { general: { executionMode: "kubernetes" } };
|
||||
await expect(resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "cursor_cloud", runner: "paperclip",
|
||||
adapterConfig: { model: "caller-model", repoUrl: "https://github.com/acme/repo" } }))
|
||||
.rejects.toMatchObject({ status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" } });
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
|
||||
expect(agentRunnerAvailability("cursor_cloud", selected).supportedRunners).toEqual([]);
|
||||
expect(state.getEnvironment).not.toHaveBeenCalled();
|
||||
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
|
||||
});
|
||||
it("keeps legacy Cursor Cloud environment bindings and rejects their unsupported native conversion", () => {
|
||||
const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo",
|
||||
@@ -389,7 +467,7 @@ describe("server-owned Codex runner selection", () => {
|
||||
it("advertises Legacy for a forced Kubernetes target before lazy environment creation", async () => {
|
||||
state.settings = { general: { executionMode: "kubernetes" } };
|
||||
const selected = await resolveAgentRunnerTargetForCompany({} as never, "company");
|
||||
expect(selected).toEqual({ driver: "sandbox" });
|
||||
expect(selected).toEqual({ driver: "sandbox", executionMode: "kubernetes" });
|
||||
expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" });
|
||||
expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company");
|
||||
expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { assertProviderApiPlacementPolicy, managedCursorApiDefaultEnabled, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { agentHarnessType, paperclipRunnerProfileForHarness, isEnvironmentDriverSupportedForAdapter, paperclipRunnerSupportsPlatform, type AgentRunnerChoice, type AgentRunnerAvailability } from "@paperclipai/shared";
|
||||
import { normalizeLegacyRunnerProvider, PAPERCLIP_RUNNER_DEFAULT_MODELS } from "@paperclipai/adapter-utils";
|
||||
@@ -16,6 +17,7 @@ import { localAgentRunnerTarget } from "../lib/agent-runner-platform.js";
|
||||
import { normalizeClaudeRunnerOptions, type AgentConfigurationWarning } from "@paperclipai/shared";
|
||||
import { usesManagedCloudSandboxImage } from "./sandbox-snapshot-qualification.js";
|
||||
import { environmentCustomImageTemplateMatchesBaseConfig } from "./environment-custom-image-runtime.js";
|
||||
import type { ExecutionPolicy } from "./execution-allowlist.js";
|
||||
|
||||
export type RunnerTarget = {
|
||||
driver: string; platform?: string; architecture?: string; libc?: "glibc" | "musl" | "unknown"; glibcVersion?: string;
|
||||
@@ -23,6 +25,7 @@ export type RunnerTarget = {
|
||||
runnerTasks?: boolean;
|
||||
managedCloudImage?: boolean;
|
||||
warmTaskVersion?: 1;
|
||||
executionMode?: ExecutionPolicy["executionMode"];
|
||||
};
|
||||
export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi";
|
||||
export interface AgentRunnerSelection {
|
||||
@@ -52,10 +55,15 @@ function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes:
|
||||
|
||||
export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability {
|
||||
if (harness === "cursor_cloud") {
|
||||
const supported = target.driver === "local" && runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
&& (!target.platform || paperclipRunnerSupportsPlatform(harness, target.platform, target.architecture ?? "", target.libc, target.glibcVersion));
|
||||
return { supportedRunners: supported ? ["paperclip", "legacy"] : ["legacy"], defaultRunner: "legacy" };
|
||||
if (target.executionMode === "kubernetes") return { supportedRunners: [], defaultRunner: "legacy" };
|
||||
const controller = localAgentRunnerTarget();
|
||||
const supported = runnerAdaptersAvailable(harness, "local", disabledTypes)
|
||||
&& paperclipRunnerSupportsPlatform(harness, controller.platform, controller.architecture, controller.libc, controller.glibcVersion);
|
||||
const legacy = isEnvironmentDriverSupportedForAdapter(harness, target.driver);
|
||||
return { supportedRunners: [...(supported ? ["paperclip" as const] : []), ...(legacy ? ["legacy" as const] : [])],
|
||||
defaultRunner: legacy || !supported ? "legacy" : "paperclip" };
|
||||
}
|
||||
if (target.driver === "unavailable") return { supportedRunners: [], defaultRunner: "legacy" };
|
||||
if (target.runnerTasks || target.managedCloudImage) {
|
||||
return runnerAdaptersAvailable(harness, target.driver, disabledTypes)
|
||||
? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" }
|
||||
@@ -187,10 +195,16 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
|
||||
if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) };
|
||||
const availability = agentRunnerAvailability(harness, input.target, disabledTypes);
|
||||
const choice = input.runner ?? "auto";
|
||||
const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip");
|
||||
const useNative = choice === "paperclip" || (choice === "auto" && harness !== "cursor_cloud" && availability.defaultRunner === "paperclip");
|
||||
if (!useNative) {
|
||||
if (harness === "cursor_cloud") {
|
||||
if (!native) return { adapterType, adapterConfig: config };
|
||||
if (!native) {
|
||||
if (choice !== "legacy" && !availability.supportedRunners.includes("legacy")) throw unprocessable(
|
||||
"Automatic Cursor selection is unavailable for this environment. Choose a supported runner in Advanced.",
|
||||
{ code: "agent_runner_unavailable" },
|
||||
);
|
||||
return { adapterType, adapterConfig: config };
|
||||
}
|
||||
if (config.lifecycleMode !== undefined && config.lifecycleMode !== "per_turn") incompatible(["lifecycleMode"]);
|
||||
for (const [current, legacy] of [["cloudEnvType", "runtimeEnvType"], ["cloudEnvName", "runtimeEnvName"], ["repoRef", "repoStartingRef"], ["prUrl", "repoPullRequestUrl"]]) {
|
||||
if (config[current] !== undefined) config[legacy] = config[current];
|
||||
@@ -363,7 +377,33 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string;
|
||||
}
|
||||
|
||||
export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) {
|
||||
input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() };
|
||||
const disabledTypes = await adapterSettingsService(db).getDisabledTypes();
|
||||
input = { ...input, disabledTypes };
|
||||
const cursor = agentHarnessType(input.adapterType ?? "process", input.adapterConfig) === "cursor_cloud";
|
||||
const nativeCursor = cursor && input.runner !== "legacy"
|
||||
&& (input.adapterType === "paperclip_runner" || input.runner === "paperclip");
|
||||
const cursorSetupDefault = cursor && input.setupOnly === true
|
||||
&& (input.runner === undefined || input.runner === "auto");
|
||||
if (nativeCursor || cursorSetupDefault) {
|
||||
const { instanceSettingsService } = await import("./instance-settings.js");
|
||||
const settings = await instanceSettingsService(db).get();
|
||||
// Only a held new setup may adopt the API-only managed default. Existing
|
||||
// legacy agents still require an explicit conversion decision.
|
||||
const useSetupDefault = cursorSetupDefault && managedCursorApiDefaultEnabled(settings.experimental);
|
||||
if (nativeCursor || useSetupDefault) {
|
||||
if (disabledTypes.includes("cursor_cloud") || disabledTypes.includes("paperclip_runner")) {
|
||||
throw unprocessable("Cursor Cloud is disabled on this instance. Enable its adapter or choose another harness.", { code: "agent_runner_unavailable" });
|
||||
}
|
||||
assertProviderApiPlacementPolicy(settings.general ?? {});
|
||||
const selected = resolveNewAgentRunner({ ...input, ...(useSetupDefault ? { runner: "paperclip" as const } : {}), target: localAgentRunnerTarget() });
|
||||
try { if (!input.setupOnly) resolveProviderApiPlacement({ agent: selected }); }
|
||||
catch (error) {
|
||||
if (error instanceof PaperclipRunnerProviderProfileError) throw unprocessable(error.message, { code: error.code });
|
||||
throw error;
|
||||
}
|
||||
return { ...selected, defaultEnvironmentId: null };
|
||||
}
|
||||
}
|
||||
if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig))
|
||||
|| input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input);
|
||||
const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes);
|
||||
@@ -372,7 +412,8 @@ export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string,
|
||||
|
||||
export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise<RunnerTarget> {
|
||||
await waitForExternalAdapters();
|
||||
const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
const { environment, driver, executionMode } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId);
|
||||
const withPolicy = (value: RunnerTarget): RunnerTarget => ({ ...value, ...(executionMode ? { executionMode } : {}) });
|
||||
let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget()
|
||||
: { driver };
|
||||
if (driver === "plugin" && environment) {
|
||||
@@ -386,16 +427,16 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
environmentId: environment.id,
|
||||
provider: typeof environment.config.provider === "string" ? environment.config.provider : null,
|
||||
});
|
||||
if (!template?.templateRef) return { driver, managedCloudImage: true };
|
||||
if (!template?.templateRef) return withPolicy({ driver, managedCloudImage: true });
|
||||
try {
|
||||
const parsed = parseEnvironmentDriverConfig(environment);
|
||||
if (parsed.driver === "sandbox" && !environmentCustomImageTemplateMatchesBaseConfig({
|
||||
template,
|
||||
baseConfig: parsed.config,
|
||||
secretRefExcludePaths: await resolveSandboxProviderSecretRefPaths(db, template.provider),
|
||||
})) return { driver, managedCloudImage: true };
|
||||
})) return withPolicy({ driver, managedCloudImage: true });
|
||||
} catch {
|
||||
return { driver };
|
||||
return withPolicy({ driver });
|
||||
}
|
||||
}
|
||||
const { readSandboxPlatformQualification } = await import("./environment-probe.js");
|
||||
@@ -413,7 +454,7 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri
|
||||
const [os = "", arch = "", libc, glibcVersion] = output.trim().split(/\r?\n/);
|
||||
target = { driver, platform: os === "Linux" ? "linux" : os === "Darwin" ? "darwin" : "unknown", architecture: arch === "x86_64" ? "x64" : ["aarch64", "arm64"].includes(arch) ? "arm64" : arch, ...(os === "Linux" && libc !== undefined ? { libc: libc === "glibc" || libc === "musl" ? libc : "unknown", ...(libc === "glibc" && glibcVersion !== undefined ? { glibcVersion } : {}) } : {}) };
|
||||
}
|
||||
return target;
|
||||
return withPolicy(target);
|
||||
}
|
||||
|
||||
export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) {
|
||||
@@ -432,16 +473,19 @@ async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defa
|
||||
const { instanceSettingsService } = await import("./instance-settings.js");
|
||||
const { environmentService } = await import("./environments.js");
|
||||
const settings = await instanceSettingsService(db).get();
|
||||
const executionMode = settings.general?.executionMode;
|
||||
const envs = environmentService(db);
|
||||
const environmentId = defaultEnvironmentId ?? settings.defaultEnvironmentId;
|
||||
let environment = environmentId ? await envs.getById(environmentId) : null;
|
||||
if (settings.experimental?.enableManagedSandboxOnly && (!environment || environment.driver === "local")) {
|
||||
environment = await envs.findManagedSandboxEnvironment(companyId);
|
||||
if (!environment) throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", { code: "managed_sandbox_unavailable" });
|
||||
if (!environment) throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", {
|
||||
code: "managed_sandbox_unavailable", ...(executionMode ? { executionMode } : {}),
|
||||
});
|
||||
}
|
||||
if (settings.general?.executionMode === "kubernetes") {
|
||||
environment = await envs.findKubernetesEnvironment(companyId);
|
||||
if (!environment) return { environment: null, driver: "sandbox" };
|
||||
if (!environment) return { environment: null, driver: "sandbox", executionMode };
|
||||
}
|
||||
if (environmentId && !environment) throw unprocessable("Environment not found.");
|
||||
if (environment) {
|
||||
@@ -449,7 +493,7 @@ async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defa
|
||||
if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" });
|
||||
if (environment.status === "archived") throw unprocessable("Environment is archived.");
|
||||
}
|
||||
return { environment, driver: environment?.driver ?? "local" };
|
||||
return { environment, driver: environment?.driver ?? "local", executionMode };
|
||||
}
|
||||
|
||||
// The plugin's manifest declaration decides whether its environment runs
|
||||
|
||||
@@ -3,7 +3,7 @@ import { withDatabaseOperation } from "@paperclipai/db";
|
||||
|
||||
type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover"
|
||||
| "budget.accounting.settlement" | "budget.accounting.transaction"
|
||||
| "budget.admission.evaluate" | "budget.admission.reserve"
|
||||
| "budget.admission.evaluate" | "budget.admission.inspect" | "budget.admission.reconcile" | "budget.admission.reserve"
|
||||
| "budget.cancellation.deliver" | "budget.enforcement.deliver"
|
||||
| "budget.enforcement.scope" | "budget.exposure.evaluate";
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ import type {
|
||||
} from "@paperclipai/shared";
|
||||
import { HttpError, notFound, unprocessable } from "../errors.js";
|
||||
import { logger } from "../middleware/logger.js";
|
||||
import { withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js";
|
||||
import { computeObservedSpend, computeObservedSpends, type ObservedSpend } from "./budget-exposure.js";
|
||||
import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js";
|
||||
@@ -198,11 +198,66 @@ export function budgetPoliciesForRun(companyId: string, agentId: string | null,
|
||||
);
|
||||
}
|
||||
|
||||
type AdmissionContext = { issueId?: string | null; projectId?: string | null };
|
||||
|
||||
async function readAdmissionScopes(db: Db, companyId: string, agentId: string | null, context?: AdmissionContext) {
|
||||
const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [
|
||||
{ scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []),
|
||||
...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []),
|
||||
];
|
||||
const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId)));
|
||||
if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found");
|
||||
const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId),
|
||||
budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null)));
|
||||
return { scopes, records, policies };
|
||||
}
|
||||
|
||||
function scopePauseBlock(scopeType: BudgetScopeType, scopeId: string, scope: ScopeRecord, hasPolicies: boolean) {
|
||||
if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") {
|
||||
return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` };
|
||||
}
|
||||
if (scope.paused && scope.pauseReason === "budget" && !hasPolicies) {
|
||||
return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function inspectInvocationEligibility(db: Db, companyId: string, agentId: string | null, context?: AdmissionContext) {
|
||||
const { scopes, records, policies } = await readAdmissionScopes(db, companyId, agentId, context);
|
||||
if (policies.length > 0) {
|
||||
if (records.some(scope => scope.budgetHeld)) return { needsReconciliation: true } as const;
|
||||
const observations = await computeObservedSpends(db, policies);
|
||||
if (policies.some((policy, index) => observedBlocks(policy, observations[index])
|
||||
|| (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observations[index].recoveringRunCount > 0))) {
|
||||
return { needsReconciliation: true } as const;
|
||||
}
|
||||
const incidents = await db.select({ policyId: budgetIncidents.policyId, thresholdType: budgetIncidents.thresholdType })
|
||||
.from(budgetIncidents).where(and(eq(budgetIncidents.companyId, companyId),
|
||||
inArray(budgetIncidents.policyId, policies.map(policy => policy.id)), eq(budgetIncidents.status, "open")));
|
||||
const byId = new Map(policies.map((policy, index) => [policy.id, { policy, observed: observations[index] }]));
|
||||
if (incidents.some(incident => {
|
||||
const { policy, observed } = byId.get(incident.policyId)!;
|
||||
return incident.thresholdType === "hard" || shouldResolveSoftIncident(policy, observed);
|
||||
})) return { needsReconciliation: true } as const;
|
||||
}
|
||||
for (const [index, { scopeType, scopeId }] of scopes.entries()) {
|
||||
const block = scopePauseBlock(scopeType, scopeId, records[index],
|
||||
policies.some(policy => policy.scopeType === scopeType && policy.scopeId === scopeId));
|
||||
if (block) return { needsReconciliation: false, block } as const;
|
||||
}
|
||||
return { needsReconciliation: false, block: null } as const;
|
||||
}
|
||||
|
||||
function unpricedUsageBlocks(policy: PolicyRow, observed: ObservedSpend) {
|
||||
return policy.unpricedUsagePolicy !== "allow"
|
||||
&& (observed.unpricedEventCount > 0 || observed.unknownBrowserRunCount > 0);
|
||||
}
|
||||
|
||||
function shouldResolveSoftIncident(policy: PolicyRow, observed: ObservedSpend) {
|
||||
return !policy.isActive || !policy.notifyEnabled
|
||||
|| compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0;
|
||||
}
|
||||
|
||||
function observedBlocks(policy: PolicyRow, observed: Awaited<ReturnType<typeof computeObservedSpend>>) {
|
||||
if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false;
|
||||
return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || unpricedUsageBlocks(policy, observed);
|
||||
@@ -654,18 +709,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
// A closed UTC window, inactive policy, or budget raise releases only
|
||||
// budget-owned pauses. Manual pauses remain operator decisions.
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard");
|
||||
if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) {
|
||||
if (shouldResolveSoftIncident(policy, observed)) {
|
||||
await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft");
|
||||
}
|
||||
}
|
||||
// Agent status has its own admission gate and error code. Preserve a
|
||||
// manual pause without misclassifying it as a budget failure.
|
||||
if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") {
|
||||
return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` };
|
||||
}
|
||||
if (scope.paused && scope.pauseReason === "budget" && !policies.length) {
|
||||
return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." };
|
||||
}
|
||||
const pauseBlock = scopePauseBlock(scopeType, scopeId, scope, policies.length > 0);
|
||||
if (pauseBlock) return pauseBlock;
|
||||
if (scope.budgetHeld && policies[0]) await resumeScopeFromBudget(policies[0]);
|
||||
if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name,
|
||||
reason: "New work must wait while a native run recovers its unfinished accounting." };
|
||||
@@ -748,14 +797,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica
|
||||
|
||||
async function evaluateAdmissionInTransaction(companyId: string, agentId: string | null,
|
||||
context?: { issueId?: string | null; projectId?: string | null }) {
|
||||
const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [
|
||||
{ scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []),
|
||||
...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []),
|
||||
];
|
||||
const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId)));
|
||||
if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found");
|
||||
const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId),
|
||||
budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null)));
|
||||
const { scopes, records, policies } = await readAdmissionScopes(db, companyId, agentId, context);
|
||||
const observations = await computeObservedSpends(db, policies);
|
||||
const reservationIndexes = policies.map((policy, i) => policy.isActive && policy.hardStopEnabled && policy.metric === "billed_cents" && policy.amount > 0 ? i : -1).filter(i => i >= 0);
|
||||
const admission = (block: Awaited<ReturnType<typeof reconcileScope>>) => ({ block,
|
||||
@@ -1188,8 +1230,13 @@ export function budgetService(db: Db, _hooks: BudgetServiceHooks = {}) {
|
||||
upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) =>
|
||||
mutate(companyId, service => service.upsertPolicy(companyId, input, actorUserId)),
|
||||
evaluateCostEvent: (event: typeof costEvents.$inferSelect) => mutate(event.companyId, service => service.evaluateCostEvent(event)),
|
||||
getInvocationBlock: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) =>
|
||||
mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context)),
|
||||
getInvocationBlock: async (companyId: string, agentId: string | null, context?: AdmissionContext) => {
|
||||
const eligibility = await withBudgetSpan("budget.admission.inspect", () =>
|
||||
withAccountingReadSnapshot(db, companyId, tx => inspectInvocationEligibility(tx, companyId, agentId, context)));
|
||||
if (!eligibility.needsReconciliation) return eligibility.block;
|
||||
return withBudgetSpan("budget.admission.reconcile", () =>
|
||||
mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context)));
|
||||
},
|
||||
resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) =>
|
||||
mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId)),
|
||||
};
|
||||
|
||||
@@ -136,6 +136,7 @@ import { notifyChatDeliveryWork, notifyChatPublicationWork } from "./chat-work-n
|
||||
|
||||
import { voiceSessionStore } from "./voice/voice-session-store.js";
|
||||
import { createChatRuntime, type ChatRuntime } from "./chat-runtime.js";
|
||||
import { registerIdleLocalWorkProbe } from "./idle-local-work.js";
|
||||
import type { ChatEndpointRuntime } from "./voice/voice-runtime.js";
|
||||
|
||||
import { fastResponseService, fastResponseSourceCurrent } from "./fast-responses.js";
|
||||
@@ -517,6 +518,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) {
|
||||
version: string;
|
||||
}
|
||||
>();
|
||||
// SDK initialize() and the final authority read continue after the registry's
|
||||
// replacement tail settles. Neither that work nor a failed retirement may
|
||||
// disappear just because the archived endpoint is no longer returned by get().
|
||||
const unregisterRuntimeIdleProbe = registerIdleLocalWorkProbe(() =>
|
||||
runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle() ? "present" : "none");
|
||||
const { recordCurrentMicrosoftTeamsRoute } = createTeamsRoutes(db, { runtime, runtimeCallbackEndpoint });
|
||||
const persistence = createChatSdkStatePersistence(db);
|
||||
const fetchImpl = options.fetch ?? globalThis.fetch;
|
||||
@@ -3622,12 +3628,17 @@ return {
|
||||
// Stop every provider runtime before releasing leader rows. This gives a
|
||||
// standby an immediate takeover path without overlapping Gateway sockets.
|
||||
await runtime.shutdown();
|
||||
await Promise.allSettled([...runtimeInitializations.values()].map(initialization => initialization.promise));
|
||||
await slackRegistration.close();
|
||||
await Promise.all(
|
||||
ownedDiscordGateways.map((ownership) =>
|
||||
releaseDiscordGatewayLeaseRow(ownership),
|
||||
),
|
||||
);
|
||||
if (runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle()) {
|
||||
throw new Error("Chat runtime lifecycle is still unsettled after shutdown");
|
||||
}
|
||||
unregisterRuntimeIdleProbe();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -23,6 +23,11 @@ export class ChatRuntime {
|
||||
return [...this.endpoints.values()];
|
||||
}
|
||||
|
||||
/** Retirement failures retain their owner even after get/list hide it. */
|
||||
hasUnsettledLifecycle(): boolean {
|
||||
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
|
||||
}
|
||||
|
||||
private enqueueLifecycle<T>(
|
||||
endpointId: string,
|
||||
operation: () => Promise<T>,
|
||||
|
||||
@@ -241,6 +241,8 @@ import {
|
||||
type CreateChatSdkEndpointRuntimeOptions,
|
||||
} from "./chat-sdk-runtime.js";
|
||||
|
||||
import { createChatRuntime } from "./chat-runtime.js";
|
||||
|
||||
const persistence: ChatSdkStatePersistence = {
|
||||
async compareAndSet() {
|
||||
return true;
|
||||
@@ -1152,6 +1154,31 @@ describe("Chat SDK endpoint runtime", () => {
|
||||
await registry.shutdown();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "SDK", create: createChatSdkRuntime },
|
||||
{ name: "shared", create: createChatRuntime },
|
||||
])("keeps $name lifecycle work visible while hidden retirement fails, until its owner retries", async ({ create }) => {
|
||||
const registry = create();
|
||||
const first = await registry.replaceEndpoint(registryOptions());
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(false);
|
||||
const held = deferred();
|
||||
const shutdown = vi.spyOn(first, "shutdown").mockReturnValueOnce(held.promise).mockResolvedValue(undefined);
|
||||
const removal = registry.removeEndpoint("endpoint-1");
|
||||
const result = removal.catch((error: Error) => error.message);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce());
|
||||
expect(registry.get("endpoint-1")).toBeNull();
|
||||
expect(registry.list()).toEqual([]);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
held.reject(new Error("retirement uncertain"));
|
||||
expect(await result).toBe("retirement uncertain");
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(true);
|
||||
await registry.removeEndpoint("endpoint-1");
|
||||
expect(shutdown).toHaveBeenCalledTimes(2);
|
||||
expect(registry.hasUnsettledLifecycle()).toBe(false);
|
||||
await registry.shutdown();
|
||||
});
|
||||
|
||||
it("retains failed retirement ownership and only constructs a replacement after an explicit successful retry", async () => {
|
||||
const registry = createChatSdkRuntime();
|
||||
const first = await registry.replaceEndpoint(registryOptions());
|
||||
|
||||
@@ -3195,6 +3195,11 @@ export class ChatSdkRuntime {
|
||||
return [...this.endpoints.values()];
|
||||
}
|
||||
|
||||
/** Retirement failures retain their owner even after get/list hide it. */
|
||||
hasUnsettledLifecycle(): boolean {
|
||||
return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0;
|
||||
}
|
||||
|
||||
private enqueueLifecycle<T>(
|
||||
endpointId: string,
|
||||
operation: () => Promise<T>,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { parseIdleCapabilities, type IdleCapabilities } from "./idle-incoming-wake-capability.js";
|
||||
import { createPublicKey, timingSafeEqual, verify, type JsonWebKey } from "node:crypto";
|
||||
import { eq } from "drizzle-orm";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
@@ -491,6 +492,7 @@ export type CloudControlClaims = {
|
||||
sub: string;
|
||||
action: CloudControlAction;
|
||||
requestId: string;
|
||||
idleCapabilities?: IdleCapabilities;
|
||||
iat: number;
|
||||
exp: number;
|
||||
};
|
||||
@@ -572,6 +574,10 @@ export function verifyCloudControlAssertion(input: {
|
||||
) {
|
||||
throw new Error("Cloud control assertion is expired or has an invalid lifetime");
|
||||
}
|
||||
if (payload.idleCapabilities !== undefined) {
|
||||
if (payload.action !== "task-drain:read") throw new Error("Idle capabilities require task-drain:read");
|
||||
payload.idleCapabilities = parseIdleCapabilities(payload.idleCapabilities);
|
||||
}
|
||||
// Consumed LAST, only after every other check passed: a rejected
|
||||
// assertion must not burn its request id, or an attacker could deny a
|
||||
// legitimate call by replaying a mangled copy of it first.
|
||||
|
||||
@@ -20,6 +20,7 @@ import { eq, inArray } from "drizzle-orm";
|
||||
import { createAgentLifecycle, processAgentLifecycle } from "./agent-lifecycle.js";
|
||||
import { requestRestorePointDeletion } from "./git-restore-point-ledger.js";
|
||||
import { agentLifecycleCompanyDeletion } from "../modules/agent-lifecycle/company-deletion.js";
|
||||
import { executionAdmissionCompanyDeletion } from "../modules/execution-admission/company-deletion.js";
|
||||
import {
|
||||
companies,
|
||||
agents,
|
||||
@@ -117,6 +118,7 @@ export async function deleteCompany(db: Db, id: string) {
|
||||
await notifyInstructionCollection(tx, { companyId: id });
|
||||
await notifyCancelledWorkspacePreservation(tx, { companyId: id });
|
||||
await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: id });
|
||||
await executionAdmissionCompanyDeletion.deleteCompanyData(tx, id);
|
||||
await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id));
|
||||
await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, id));
|
||||
await tx.delete(agentApiKeys).where(eq(agentApiKeys.companyId, id));
|
||||
|
||||
@@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean {
|
||||
wake();
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Inspection never emits a wake or polls the broker. */
|
||||
export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner);
|
||||
@@ -1,4 +1,5 @@
|
||||
import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js";
|
||||
import { isPreDispatchReviewWait, neverStartedDispatchCancellationPredicate } from "./pre-dispatch-review-wait.js";
|
||||
import { canonicalExecutionUuid } from "./execution-identity.js";
|
||||
import { preDispatchExecutionWaitRecoveryActionPredicate } from "./pre-dispatch-execution-wait.js";
|
||||
import { and, desc, eq, gt, inArray, not, or, sql } from "drizzle-orm";
|
||||
import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js";
|
||||
@@ -97,6 +98,17 @@ export function executionBlockerPredicate() {
|
||||
return and(
|
||||
not(conversationRecoveryActionPredicate()!),
|
||||
not(preDispatchExecutionWaitRecoveryActionPredicate()),
|
||||
sql`not (${issueRecoveryActions.cause} = 'legacy_execution_requires_reconciliation' and exists (
|
||||
select 1 from ${heartbeatRuns}
|
||||
where ${heartbeatRuns.companyId} = ${issueRecoveryActions.companyId}
|
||||
and ${heartbeatRuns.id} = ${canonicalExecutionUuid(sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`)}
|
||||
and ${issueRecoveryActions.sourceIssueId} = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)}
|
||||
and (${heartbeatRuns.issueId} is null or ${heartbeatRuns.issueId} = ${issueRecoveryActions.sourceIssueId})
|
||||
and ${neverStartedDispatchCancellationPredicate()}
|
||||
and ${heartbeatRuns.errorCode} <> 'issue_continuation_waiting_on_review'
|
||||
and coalesce(${issueRecoveryActions.evidence}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive'
|
||||
and coalesce(${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}
|
||||
))`,
|
||||
or(unreconciledNativeExportPredicate(), and(
|
||||
inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]),
|
||||
or(inArray(issueRecoveryActions.status, ["active", "escalated"]),
|
||||
|
||||
@@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record<string, unknown> {
|
||||
return compact(base);
|
||||
}
|
||||
|
||||
function bindingRows(rows: Array<{
|
||||
export function githubConnectionEventBindings(rows: Array<{
|
||||
grant: typeof connectionGrants.$inferSelect;
|
||||
connection: typeof toolConnections.$inferSelect;
|
||||
}>): GitHubBinding[] {
|
||||
@@ -259,7 +259,7 @@ export function githubConnectionEventService(
|
||||
eq(toolConnections.enabled, true),
|
||||
sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`,
|
||||
));
|
||||
return bindingRows(rows);
|
||||
return githubConnectionEventBindings(rows);
|
||||
}
|
||||
|
||||
async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise<void>>) {
|
||||
|
||||
@@ -89,7 +89,10 @@ export async function appendHeartbeatRunEvent(
|
||||
input: AppendHeartbeatRunEventInput,
|
||||
): Promise<AppendHeartbeatRunEventResult> {
|
||||
const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX));
|
||||
const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType);
|
||||
// Terminal proof may arrive after controller retirement and a quiet Stop
|
||||
// check. Register the scoped wake before writing it, in the same transaction.
|
||||
const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType)
|
||||
|| Boolean(input.nativeSource && ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(input.eventType));
|
||||
const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null;
|
||||
const projectsProgress = isSafeNativeChatProgressEventType(input.eventType);
|
||||
const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent);
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js";
|
||||
import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js";
|
||||
import { reconcileTaskChatRetryReceipts } from "./task-retry.js";
|
||||
import { registerIssueRecoveryWork } from "./recovery/work.js";
|
||||
@@ -2266,7 +2267,8 @@ export function heartbeatService(
|
||||
const defaultIsolatedWorkspacesEnabled =
|
||||
isolatedWorkspacesEnabled &&
|
||||
experimentalInstanceSettings.enableIsolatedWorkspacesByDefault;
|
||||
const externalProvider = heartbeatExternalProvider({ run, agent });
|
||||
const providerApiPlacement = resolveProviderApiPlacement({ run, agent });
|
||||
const externalProvider = heartbeatExternalProvider({ run, agent }) ?? providerApiPlacement?.provider;
|
||||
const [taskWorkspaceIntentRow] = issueId && !externalProvider ? await db.select({ intent: issues.workspaceSelection }).from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) : [];
|
||||
const freshWorkspace = await readFreshWorkspaceReadiness(db, run);
|
||||
@@ -2968,7 +2970,7 @@ export function heartbeatService(
|
||||
runtimeConfig = { ...runtimeConfig, env: gitExecutionEnv };
|
||||
for (const key of MANAGED_GITHUB_TOKEN_KEYS) secretKeys.add(key);
|
||||
context.githubAuthenticationMode = useHostGitHub ? "host" : "managed";
|
||||
if (!useHostGitHub) {
|
||||
if (!useHostGitHub && !providerApiPlacement) {
|
||||
const githubLaunchers = await timePreparation("github_launchers", () => prepareHeartbeatGitHubLaunchers({
|
||||
native: agent.adapterType === "paperclip_runner",
|
||||
githubConfigured: githubSelection.configured,
|
||||
@@ -2990,7 +2992,7 @@ export function heartbeatService(
|
||||
runtimeConfig = { ...runtimeConfig, env: githubLaunchers.env };
|
||||
secretKeys.add("PAPERCLIP_GITHUB_BROKER_TOKEN");
|
||||
}
|
||||
context.paperclipEnvironment = {
|
||||
context.paperclipEnvironment = selectedEnvironment && activeEnvironmentLease ? {
|
||||
id: selectedEnvironment.id,
|
||||
name: selectedEnvironment.name,
|
||||
driver: selectedEnvironment.driver,
|
||||
@@ -3017,7 +3019,7 @@ export function heartbeatService(
|
||||
: undefined,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} : { placement: providerApiPlacement };
|
||||
await db
|
||||
.update(heartbeatRuns)
|
||||
.set({
|
||||
@@ -3244,8 +3246,8 @@ export function heartbeatService(
|
||||
executionWorkspace,
|
||||
persistedExecutionWorkspace,
|
||||
executionTarget,
|
||||
environmentDriver: selectedEnvironment.driver,
|
||||
leaseMetadata: activeEnvironmentLease.lease.metadata,
|
||||
environmentDriver: selectedEnvironment?.driver ?? "provider_api",
|
||||
leaseMetadata: activeEnvironmentLease?.lease.metadata ?? {},
|
||||
}));
|
||||
const adapterEnv = Object.fromEntries(
|
||||
Object.entries(parseObject(runtimeConfig.env)).filter(
|
||||
@@ -3932,6 +3934,9 @@ export function heartbeatService(
|
||||
await settleNativeRunProcess(db, run.id).catch(error => {
|
||||
logger.warn({ err: error, runId: run.id }, "failed to release settled native process identity");
|
||||
});
|
||||
await acknowledgeRemoteStop(run.id, run.companyId).catch(error => {
|
||||
logger.warn({ err: error, runId: run.id }, "failed to acknowledge Stop after native process settlement");
|
||||
});
|
||||
}
|
||||
await db.update(heartbeatRuns).set({
|
||||
executionStage: sql`case when ${heartbeatRuns.status} = 'cancelled' then 'settled' else ${heartbeatRuns.executionStage} end`,
|
||||
|
||||
@@ -6,7 +6,7 @@ import { DELIVERY_QUEUES } from "../delivery-work-notifications.js";
|
||||
import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js";
|
||||
import { registerScopedDeadlineWork } from "../scoped-deadline-work.js";
|
||||
|
||||
type Entry = { id: string; companyId: string; dueAt: number };
|
||||
type Entry = { id: string; companyId: string; dueAt: number; immediate: boolean };
|
||||
type NativeIntentOptions = DeferredExecutionScope & {
|
||||
onDeferredDeadline(at: number): void;
|
||||
onError(error: unknown): void;
|
||||
@@ -39,6 +39,11 @@ export function registerDeferredReceiptWork(
|
||||
},
|
||||
decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint),
|
||||
matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId),
|
||||
// A busy company can invalidate a paginated scan before the pending
|
||||
// batch drains. Keep unread ready receipts ahead of freshly reloaded
|
||||
// receipts already checked; actual wait/claim deadlines still replace.
|
||||
mergeEntry: (pending, loaded) => pending.immediate && loaded.immediate
|
||||
? { ...loaded, dueAt: Math.min(pending.dueAt, loaded.dueAt) } : loaded,
|
||||
load: async scope => {
|
||||
const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*";
|
||||
if (!enabled) { cursors.delete(key); return []; }
|
||||
@@ -76,12 +81,15 @@ export function registerDeferredReceiptWork(
|
||||
)).orderBy(asc(agentWakeupRequests.id)).limit(100);
|
||||
const complete = rows.length < 100;
|
||||
if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id);
|
||||
return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId,
|
||||
dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
const now = Date.now();
|
||||
return { rows: rows.map(row => {
|
||||
const deadline = kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000
|
||||
: kind === "native" && row.error ? row.updatedAt.getTime() + 30_000
|
||||
: kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000
|
||||
: Date.now(),
|
||||
})), complete };
|
||||
: null;
|
||||
const immediate = deadline === null || deadline <= now;
|
||||
return { id: row.id, companyId: row.companyId, dueAt: immediate ? now : deadline, immediate };
|
||||
}), complete };
|
||||
},
|
||||
check: async entry => {
|
||||
if (!enabled) return null;
|
||||
@@ -101,7 +109,7 @@ export function registerDeferredReceiptWork(
|
||||
: kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy
|
||||
&& (current.payload?.queuedCommentInterrupt || current.payload?.executionWait)
|
||||
? current.updatedAt.getTime() + 30_000 : null;
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due };
|
||||
if (due !== null && due > Date.now()) return { ...entry, dueAt: due, immediate: false };
|
||||
}
|
||||
const errors: unknown[] = [];
|
||||
let next: number | null = null;
|
||||
@@ -117,7 +125,7 @@ export function registerDeferredReceiptWork(
|
||||
catch (error) { failed(error); }
|
||||
}
|
||||
if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`);
|
||||
return next === null ? null : { ...entry, dueAt: next };
|
||||
return next === null ? null : { ...entry, dueAt: next, immediate: false };
|
||||
},
|
||||
}, canRun);
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { assertProviderApiPlacementPolicy, resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js";
|
||||
import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./external-provider.js";
|
||||
import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db";
|
||||
import { and, eq } from "drizzle-orm";
|
||||
@@ -106,16 +107,18 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
: null;
|
||||
// This retained name covers both external bridges; neither adopts task files.
|
||||
const isDotRun = heartbeatExternalProvider({ run, agent }) !== null;
|
||||
const providerApiPlacement = resolveProviderApiPlacement({ run, agent });
|
||||
const controllerOnly = isDotRun || providerApiPlacement !== null;
|
||||
const persistedNativeExecutionWorkspaceId =
|
||||
persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null;
|
||||
providerApiPlacement ? null : persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null;
|
||||
const fresh = input.task.freshWorkspace?.mode === "fresh_isolated_workspace";
|
||||
const requestedExecutionWorkspaceId = isDotRun ? null :
|
||||
const requestedExecutionWorkspaceId = controllerOnly ? null :
|
||||
persistedNativeExecutionWorkspaceId ??
|
||||
readNonEmptyString(issueRef?.executionWorkspaceId);
|
||||
let existingExecutionWorkspace = requestedExecutionWorkspaceId
|
||||
? await executionWorkspacesSvc.getStoredById(agent.companyId, requestedExecutionWorkspaceId)
|
||||
: null;
|
||||
if (existingExecutionWorkspace && issueId && !isDotRun && !fresh
|
||||
if (existingExecutionWorkspace && issueId && !controllerOnly && !fresh
|
||||
&& existingExecutionWorkspace.id === readNonEmptyString(issueRef?.executionWorkspaceId)
|
||||
&& (existingExecutionWorkspace.status === "archived" || existingExecutionWorkspace.status === "cleanup_failed")
|
||||
&& (readGitCacheExpiration(existingExecutionWorkspace as Parameters<typeof readGitCacheExpiration>[0])?.workspaceRemoved
|
||||
@@ -154,7 +157,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
const workspaceReuseRequest =
|
||||
resolveExecutionWorkspaceReuseRequestForIssue({
|
||||
issueExecutionWorkspaceId: requestedExecutionWorkspaceId,
|
||||
issueExecutionWorkspacePreference: isDotRun ? null : nativeRecoveryExecutionWorkspaceId
|
||||
issueExecutionWorkspacePreference: controllerOnly ? null : nativeRecoveryExecutionWorkspaceId
|
||||
? "reuse_existing"
|
||||
: (issueRef?.executionWorkspacePreference ?? null),
|
||||
existingExecutionWorkspaceStatus:
|
||||
@@ -168,11 +171,11 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
: null;
|
||||
const workspaceAuthorizationActor = { type: "agent" as const, agentId: agent.id, companyId: agent.companyId,
|
||||
source: "agent_jwt" as const, runId: run.id, onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId };
|
||||
if (!isDotRun && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
if (!controllerOnly && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
const workspaceRequiringAccess = reusableExistingExecutionWorkspace ?? boundSourceWorkspace;
|
||||
if (workspaceRequiringAccess && issueId) {
|
||||
await assertTaskWorkspaceExecutionAccess(db, workspaceAuthorizationActor, agent.companyId, { issueId, workspaceId: workspaceRequiringAccess.id });
|
||||
} else if (!isDotRun && executionProjectId && (hasSelectedWorkspaceSource || workspaceProjectContext?.hasWorkspace)) {
|
||||
} else if (!controllerOnly && executionProjectId && (hasSelectedWorkspaceSource || workspaceProjectContext?.hasWorkspace)) {
|
||||
await assertTaskWorkspaceSourceProjectAccess(db, workspaceAuthorizationActor, agent.companyId, executionProjectId);
|
||||
}
|
||||
}
|
||||
@@ -191,10 +194,12 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
const managedSandboxOnly =
|
||||
(await instanceSettings.getExperimental()).enableManagedSandboxOnly ===
|
||||
true;
|
||||
const managedSandboxEnvironment = managedSandboxOnly
|
||||
const managedSandboxEnvironment = managedSandboxOnly && !providerApiPlacement
|
||||
? await environmentsSvc.findManagedSandboxEnvironment(agent.companyId)
|
||||
: null;
|
||||
const environmentResolution = resolveExecutionWorkspaceEnvironmentId({
|
||||
const environmentResolution = providerApiPlacement
|
||||
? { environmentId: null, source: "default" as const }
|
||||
: resolveExecutionWorkspaceEnvironmentId({
|
||||
lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review"
|
||||
? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, {includeEnvironmentId: true})?.environmentId : null,
|
||||
agentDefaultEnvironmentId: agent.defaultEnvironmentId,
|
||||
@@ -218,8 +223,9 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
};
|
||||
const executionForcedToKubernetes =
|
||||
isExecutionForcedToKubernetes(executionPolicy);
|
||||
if (providerApiPlacement) assertProviderApiPlacementPolicy(executionPolicy);
|
||||
let selectedEnvironmentId = environmentResolution.environmentId;
|
||||
if (executionForcedToKubernetes) {
|
||||
if (executionForcedToKubernetes && !providerApiPlacement) {
|
||||
let kubernetesEnvironment =
|
||||
await environmentsSvc.findKubernetesEnvironment(agent.companyId);
|
||||
if (!kubernetesEnvironment) {
|
||||
@@ -292,7 +298,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
: selectedEnvironmentId
|
||||
? await environmentsSvc.getById(selectedEnvironmentId)
|
||||
: null;
|
||||
const nativeChatWorkspaceScope = isDotRun ? null : await findNativeChatWorkspaceScope(db, {
|
||||
const nativeChatWorkspaceScope = controllerOnly ? null : await findNativeChatWorkspaceScope(db, {
|
||||
adapterType: agent.adapterType,
|
||||
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
|
||||
companyId: agent.companyId,
|
||||
@@ -391,7 +397,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
// project or issue settings request serialization. Sandbox protection still uses
|
||||
// the existing holder staleness and workspace_busy retry ladder.
|
||||
if (
|
||||
!isDotRun && issueRef?.projectWorkspaceId &&
|
||||
!controllerOnly && issueRef?.projectWorkspaceId &&
|
||||
effectiveExecutionWorkspaceMode === "shared_workspace"
|
||||
) {
|
||||
const workspaceHolder = await findSharedWorkspaceHolder({
|
||||
@@ -472,18 +478,18 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!isDotRun && taskWorkspaceIntent && !boundSourceWorkspace && !nativeRecoveryExecutionWorkspaceId) {
|
||||
if (!controllerOnly && taskWorkspaceIntent && !boundSourceWorkspace && !nativeRecoveryExecutionWorkspaceId) {
|
||||
await executionWorkspacesSvc.validateStoredSelectionForRun({ companyId: agent.companyId,
|
||||
actor: { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId },
|
||||
selection: taskWorkspaceIntent.selection, issueId,
|
||||
preserveImplicitDefaultSettings: ["project", "task_default"].includes(taskWorkspaceIntent.source) });
|
||||
}
|
||||
const explicitTaskDirectory = !isDotRun && taskWorkspaceIntent?.selection.kind === "task_directory";
|
||||
const explicitTaskDirectory = !controllerOnly && taskWorkspaceIntent?.selection.kind === "task_directory";
|
||||
const explicitDirectoryStrategies = [
|
||||
...(reusableExistingExecutionWorkspace?.providerType === "adapter_managed" ? [config.workspaceStrategy] : []),
|
||||
issueAssigneeOverrides?.adapterConfig?.workspaceStrategy,
|
||||
projectExecutionWorkspacePolicy?.workspaceStrategy, issueExecutionWorkspaceSettings?.workspaceStrategy];
|
||||
const boundTaskDirectory = !isDotRun && !fresh && !nativeRecoveryExecutionWorkspaceId && issueId !== null && issueRef !== null &&
|
||||
const boundTaskDirectory = !controllerOnly && !fresh && !nativeRecoveryExecutionWorkspaceId && issueId !== null && issueRef !== null &&
|
||||
requestedShouldReuseExisting && requestedExecutionWorkspaceMode === "isolated_workspace" &&
|
||||
!workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId &&
|
||||
taskWorkspaceIntent?.selection.kind !== "configured_source" &&
|
||||
@@ -503,7 +509,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
if (retainedDirectory && !boundTaskDirectory) throw new WorkspaceValidationFailure("The saved Retry task directory is unavailable.", {
|
||||
workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId },
|
||||
});
|
||||
const useIsolatedTaskDirectory = !isDotRun && issueRef !== null && (reuseTaskDirectory || (fresh && !workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId) || shouldUseIsolatedTaskDirectory({
|
||||
const useIsolatedTaskDirectory = !controllerOnly && issueRef !== null && (reuseTaskDirectory || (fresh && !workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId) || shouldUseIsolatedTaskDirectory({
|
||||
trustPreset: trustPreset.kind,
|
||||
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
|
||||
mode: requestedExecutionWorkspaceMode,
|
||||
@@ -520,6 +526,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron
|
||||
persistedRunnerProfile,
|
||||
persistedNativeExecutionInput,
|
||||
isDotRun,
|
||||
providerApiPlacement,
|
||||
existingExecutionWorkspace,
|
||||
nativeRecoveryExecutionWorkspaceId,
|
||||
workspaceReuseRequest,
|
||||
|
||||
@@ -24,5 +24,5 @@ type TaskWorkspaceExecution = Exclude<NativeExecutionInput, {
|
||||
|
||||
export function nativeExecutionUsesTaskWorkspace(execution: NativeExecutionInput | null): execution is TaskWorkspaceExecution {
|
||||
return execution !== null && execution.schema !== "paperclip.native-execution-input.v6"
|
||||
&& execution.schema !== "paperclip.native-execution-input.v8";
|
||||
&& execution.schema !== "paperclip.native-execution-input.v8" && execution.provider.kind !== "cursor_cloud";
|
||||
}
|
||||
@@ -60,6 +60,11 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType<typeof
|
||||
: Date.parse(String((row.metadata?.legacyShutdownWorkspaceSettlement as Record<string, unknown>)?.deadlineAt)) || Date.now(),
|
||||
}));
|
||||
} else {
|
||||
// Retirement may commit before Stop acknowledgement. This is only a
|
||||
// work hint; the owner rechecks the complete provider/process proof.
|
||||
const pendingApiStop = and(eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`,
|
||||
sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`);
|
||||
const candidates = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId,
|
||||
nextAttemptAt: nativeRunFinalizations.nextAttemptAt, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt })
|
||||
.from(heartbeatRuns).leftJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id),
|
||||
@@ -76,9 +81,9 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType<typeof
|
||||
: kind === "active" ? and(eq(heartbeatRuns.status, "running"),
|
||||
sql`${heartbeatRuns.contextSnapshot}->'nativeQuestionCancellation' is null`)
|
||||
: and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]),
|
||||
isNotNull(heartbeatRuns.finishedAt), or(isNotNull(heartbeatRuns.processPid),
|
||||
isNotNull(heartbeatRuns.processGroupId), isNotNull(heartbeatRuns.processStartedAt)),
|
||||
settledNativeProcessIdentityCondition(db)),
|
||||
isNotNull(heartbeatRuns.finishedAt), or(pendingApiStop,
|
||||
and(or(isNotNull(heartbeatRuns.processPid), isNotNull(heartbeatRuns.processGroupId),
|
||||
isNotNull(heartbeatRuns.processStartedAt)), settledNativeProcessIdentityCondition(db)))),
|
||||
)).orderBy(asc(heartbeatRuns.id)).limit(100);
|
||||
rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, dueAt: kind === "native"
|
||||
? Math.max(Date.now(), row.nextAttemptAt?.getTime() ?? 0, row.leaseExpiresAt?.getTime() ?? 0) : Date.now() }));
|
||||
|
||||
@@ -2775,7 +2775,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD
|
||||
scope.onDeferredDeadline(Date.now() + 30_000);
|
||||
return;
|
||||
}
|
||||
await releaseSettledNativeRunProcess(db, scope.runId, scope);
|
||||
const outcome = await releaseSettledNativeRunProcess(db, scope.runId, scope);
|
||||
if (outcome !== "pending") await acknowledgeRemoteStop(scope.runId, scope.companyId);
|
||||
},
|
||||
orphanLease: scope => sweepOrphanedActiveLeases({ ...scope, backoffMs: 5 * 60_000 }),
|
||||
shutdown: scope => sweepLegacyShutdownWorkspaceSettlements(db, new Date(), scope),
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, caseEvents, caseHandoffs, cases, companies, environmentLeases, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { activityLog, agents, agentWakeupRequests, caseEvents, caseHandoffs, cases, companies, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db";
|
||||
import { eq, inArray, sql } from "drizzle-orm";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { runningProcesses } from "../../adapters/index.js";
|
||||
@@ -24,9 +24,18 @@ import { caseHandoffService } from "../case-handoffs.js";
|
||||
import { withQueuedCommentIdsInWakePayload } from "../issue-queued-comment-queue.js";
|
||||
import { instanceSettingsService } from "../instance-settings.js";
|
||||
import { issueService } from "../issues.js";
|
||||
import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js";
|
||||
import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";
|
||||
import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js";
|
||||
import { acknowledgedNativeApiStopExecutionHasStopped } from "../acknowledged-native-stop.js";
|
||||
import { nativeSha256 } from "../native-runtime/canonical.js";
|
||||
import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js";
|
||||
|
||||
vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() }));
|
||||
vi.mock("../acknowledged-native-stop.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../acknowledged-native-stop.js")>();
|
||||
return { ...actual, acknowledgedNativeApiStopExecutionHasStopped: vi.fn(actual.acknowledgedNativeApiStopExecutionHasStopped) };
|
||||
});
|
||||
vi.mock("../adapter-execution-control.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../adapter-execution-control.js")>();
|
||||
return {
|
||||
@@ -628,6 +637,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat
|
||||
releasedAt: proof === "missing" ? null : new Date(), cleanupStatus: proof === "missing" ? null : "success",
|
||||
metadata: proof === "missing" ? {} : { remoteExecutionTermination: receipt } });
|
||||
const control = createHeartbeatRunControl(db, callbacks(db));
|
||||
vi.mocked(publishLiveEvent).mockClear();
|
||||
await control.acknowledgeRemoteStop(run.id, company.id);
|
||||
await control.acknowledgeRemoteStop(run.id, company.id);
|
||||
const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
@@ -635,6 +645,149 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat
|
||||
if (proof === "confirmed") expect(stopped.resultJson?.executionCancellation).toMatchObject({ proof: "provider_termination_receipt" });
|
||||
expect(stopped.resultJson?.workspaceRestoreFailure).toBe("restore_failed");
|
||||
expect(stopped.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [leaseId] });
|
||||
expect(vi.mocked(publishLiveEvent).mock.calls).toEqual(proof === "confirmed" ? [[{
|
||||
companyId: company.id,
|
||||
type: "heartbeat.run.status",
|
||||
payload: expect.objectContaining({ runId, agentId: agent.id, status: "cancelled" }),
|
||||
}]] : []);
|
||||
});
|
||||
|
||||
it.each(["confirmed", "missing_process_stop", "later_start", "missing_terminal", "unproven_terminal",
|
||||
"foreign_terminal", "foreign_binding", "missing_placement", "unexpected_lease", "running_pid",
|
||||
"wrong_intent", "pending_ack", "unresolved_mode", "missing_hash", "wrong_hash", "malformed_terminal",
|
||||
"race_event_start", "race_profile_binding", "race_benign_event",
|
||||
"natural_completed", "natural_cancelled", "natural_error", "natural_expired",
|
||||
"mismatched_terminal_status", "local_failure", "unproven_cancelled",
|
||||
"pre_stop_completed", "pre_stop_cancelled", "pre_stop_error", "pre_stop_expired", "pre_stop_interrupted",
|
||||
"pre_stop_later_turn", "pre_stop_unproven",
|
||||
] as const)("publishes an API-only native Stop only after its exact provider and controller stopped: %s", async proof => {
|
||||
const { company, agent } = await fixture();
|
||||
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "API-only Stop", assigneeAgentId: agent.id }).returning();
|
||||
const runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID();
|
||||
const recordedAt = new Date(Date.now() - 1_000).toISOString();
|
||||
const nativeExecutionInput = buildNativeExecutionInput({ companyId: company.id, agentId: agent.id,
|
||||
runId: proof === "foreign_binding" ? randomUUID() : runId, provider: "cursor_cloud", model: "explicit-model",
|
||||
cursorCloud: { schema: "paperclip.cursor-cloud.config.v1", repos: [{ url: "https://github.com/example/qualification" }] },
|
||||
issue: { id: issue.id, identifier: "RC-1", title: issue.title, description: null, workMode: "standard" },
|
||||
taskPrompt: "Stop the foreground command", normalizedSessionId: sessionId,
|
||||
workspace: { id: randomUUID(), cwd: "/private/controller", repoUrl: null, repoRef: null, branchName: null },
|
||||
completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1",
|
||||
contract: { revision: "1", objective: "Stop the foreground command", criteria: [{ id: "objective", requirement: "Stop the foreground command" }] } },
|
||||
runtimeContext: nativeRuntimeContextFixture(),
|
||||
});
|
||||
const profile = { backend: "cursor_cloud_api", nativeExecutionInput,
|
||||
...(proof === "missing_placement" ? {} : { executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } }) };
|
||||
const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId: company.id, agentId: agent.id,
|
||||
nativeIssueId: issue.id, runtimeMode: proof === "unresolved_mode" ? "legacy" : "native", driverKind: "cursor_cloud_api",
|
||||
nativeSessionId: sessionId, runnerInstanceId: runnerId, runnerProfileJson: profile,
|
||||
status: "cancelled", finishedAt: new Date(), processPid: proof === "running_pid" ? process.pid : null,
|
||||
resultJson: { executionCancellation: { state: "requested", requestedAt: recordedAt },
|
||||
cancelledByActorType: "user", cancelledByUserId: "operator", nativeCancellation: {
|
||||
schema: "paperclip.native-cancellation.v1", runId: proof === "wrong_intent" ? randomUUID() : runId,
|
||||
companyId: company.id, issueId: issue.id, scope: "run", reasonCode: "cancellation_run_only",
|
||||
dispatchState: proof === "pending_ack" ? "pending" : "acknowledged", dispatched: true,
|
||||
effects: ["release_run_resources"], recordedAt, acknowledgedAt: new Date().toISOString(),
|
||||
intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID(),
|
||||
} },
|
||||
}).returning();
|
||||
if (proof === "unexpected_lease") await db.insert(environmentLeases).values({ companyId: company.id, issueId: issue.id,
|
||||
heartbeatRunId: runId, provider: "local", status: "released", leasePolicy: "retain_on_failure",
|
||||
releasedAt: new Date(), cleanupStatus: "success" });
|
||||
if (proof !== "missing_terminal") {
|
||||
const preStop = proof.startsWith("pre_stop_");
|
||||
const terminalProof = proof === "pre_stop_later_turn" || proof === "pre_stop_unproven" ? "natural_completed"
|
||||
: preStop ? `natural_${proof.slice("pre_stop_".length)}` : proof;
|
||||
const terminalAt = preStop ? new Date(Date.parse(recordedAt) - 1_000) : new Date();
|
||||
const terminalType = terminalProof === "natural_cancelled" || terminalProof === "unproven_cancelled" ? "turn.cancelled"
|
||||
: ["natural_error", "natural_expired", "local_failure"].includes(terminalProof) ? "turn.failed"
|
||||
: terminalProof === "natural_completed" || terminalProof === "mismatched_terminal_status" ? "turn.completed" : "turn.interrupted";
|
||||
const terminalError = terminalProof === "natural_error" ? { code: "cursor_cloud_error", message: "Cursor Cloud run failed" }
|
||||
: terminalProof === "natural_expired" ? { code: "cursor_cloud_expired", message: "Cursor Cloud run expired" }
|
||||
: terminalProof === "local_failure" ? { code: "process_lost", message: "Synthetic local process loss" } : null;
|
||||
const event = { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: terminalType,
|
||||
sourceKind: "runner", sourceInstanceId: runnerId, sourceEventId: `${runnerId}:${runId}:1`, sourceSeq: 1,
|
||||
runId, turnId: "api-stop-turn", normalizedSessionId: proof === "foreign_terminal" ? randomUUID() : sessionId,
|
||||
emittedAt: terminalAt.toISOString(), priority: 1,
|
||||
payload: { status: terminalProof === "mismatched_terminal_status" ? "interrupted" : terminalType.slice("turn.".length), error: terminalError,
|
||||
...(terminalProof === "natural_cancelled" ? { providerTerminalState: "completed", reason: "cancelled_after_provider_completed" } : {}),
|
||||
...(terminalProof === "unproven_cancelled" ? { providerTerminalState: "unknown", reason: "operator_cancel_requested" } : {}),
|
||||
...(proof === "unproven_terminal" || proof === "pre_stop_unproven" ? { providerTerminalObserved: false } : {}) } };
|
||||
await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId,
|
||||
eventType: event.eventType, payload: { prpEvent: event }, nativeSource: {
|
||||
sourceInstanceId: runnerId, sourceEventId: event.sourceEventId, sourceSeq: 1,
|
||||
protocolSchemaVersion: 1, canonicalPayload: event,
|
||||
} });
|
||||
if (preStop) await db.update(heartbeatRunEvents).set({ createdAt: terminalAt }).where(eq(heartbeatRunEvents.runId, runId));
|
||||
if (proof === "pre_stop_later_turn") {
|
||||
const later = { ...event, eventType: "turn.started", sourceEventId: `${runnerId}:${runId}:2`, sourceSeq: 2,
|
||||
turnId: "later-api-turn", emittedAt: new Date().toISOString(), payload: { status: "running" } };
|
||||
await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId,
|
||||
eventType: later.eventType, payload: { prpEvent: later }, nativeSource: {
|
||||
sourceInstanceId: runnerId, sourceEventId: later.sourceEventId, sourceSeq: 2,
|
||||
protocolSchemaVersion: 1, canonicalPayload: later,
|
||||
} });
|
||||
}
|
||||
if (["missing_hash", "wrong_hash", "malformed_terminal"].includes(proof)) {
|
||||
const malformed = { ...event, priority: undefined };
|
||||
await db.update(heartbeatRunEvents).set({
|
||||
sourcePayloadSha256: proof === "missing_hash" ? null : proof === "wrong_hash" ? "0".repeat(64) : nativeSha256(malformed),
|
||||
...(proof === "malformed_terminal" ? { payload: { prpEvent: malformed } } : {}),
|
||||
}).where(eq(heartbeatRunEvents.runId, runId));
|
||||
}
|
||||
}
|
||||
const control = createHeartbeatRunControl(db, callbacks(db));
|
||||
vi.mocked(publishLiveEvent).mockClear();
|
||||
if (proof === "confirmed") {
|
||||
await control.acknowledgeRemoteStop(run.id, company.id);
|
||||
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]
|
||||
.resultJson?.executionCancellation).toMatchObject({ state: "requested" });
|
||||
expect(publishLiveEvent).not.toHaveBeenCalled();
|
||||
}
|
||||
if (proof !== "missing_process_stop") await appendHeartbeatRunEvent(db, { companyId: company.id,
|
||||
agentId: agent.id, runId, eventType: "native.local_process_stopped", payload: { processPid: 424242, processGroupId: 424242 } });
|
||||
if (proof === "later_start") await appendHeartbeatRunEvent(db, { companyId: company.id,
|
||||
agentId: agent.id, runId, eventType: "native.process_start_requested", payload: { targetKind: "local" } });
|
||||
if (proof.startsWith("race_")) {
|
||||
const actual = await vi.importActual<typeof import("../acknowledged-native-stop.js")>("../acknowledged-native-stop.js");
|
||||
vi.mocked(acknowledgedNativeApiStopExecutionHasStopped).mockImplementationOnce(async (...args) => {
|
||||
const proven = await actual.acknowledgedNativeApiStopExecutionHasStopped(...args);
|
||||
expect(proven).toBe(true);
|
||||
if (proof === "race_event_start" || proof === "race_benign_event") {
|
||||
const [before] = await db.select({ cursor: heartbeatRuns.nextEventSeq }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId,
|
||||
eventType: proof === "race_event_start" ? "native.process_start_requested" : "lifecycle",
|
||||
payload: proof === "race_event_start" ? { targetKind: "local" } : { reason: "synthetic finalization log" } });
|
||||
const [after] = await db.select({ cursor: heartbeatRuns.nextEventSeq }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
|
||||
expect(after.cursor).toBe(before.cursor + 1);
|
||||
} else {
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: {
|
||||
...profile,
|
||||
nativeExecutionInput: { ...nativeExecutionInput, binding: { ...nativeExecutionInput.binding, issueId: randomUUID() } },
|
||||
} }).where(eq(heartbeatRuns.id, runId));
|
||||
}
|
||||
return proven;
|
||||
});
|
||||
}
|
||||
await control.acknowledgeRemoteStop(run.id, company.id);
|
||||
if (proof === "race_benign_event") expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]
|
||||
.resultJson?.executionCancellation).toMatchObject({ state: "acknowledged" });
|
||||
await control.acknowledgeRemoteStop(run.id, company.id);
|
||||
const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
|
||||
const acknowledged = ["confirmed", "race_benign_event", "natural_completed", "natural_cancelled", "natural_error", "natural_expired",
|
||||
"pre_stop_completed", "pre_stop_cancelled", "pre_stop_error", "pre_stop_expired", "pre_stop_interrupted"].includes(proof);
|
||||
expect(stopped.resultJson?.executionCancellation).toMatchObject({ state: acknowledged ? "acknowledged" : "requested" });
|
||||
if (acknowledged) expect(stopped.resultJson?.executionCancellation).toMatchObject({ proof: "provider_api_stop_receipt" });
|
||||
expect(vi.mocked(publishLiveEvent).mock.calls).toEqual(acknowledged ? [[{
|
||||
companyId: company.id,
|
||||
type: "heartbeat.run.status",
|
||||
payload: expect.objectContaining({ runId, agentId: agent.id, status: "cancelled" }),
|
||||
}]] : []);
|
||||
if (acknowledged) {
|
||||
const payload = vi.mocked(publishLiveEvent).mock.calls[0]![0].payload;
|
||||
expect(payload).not.toHaveProperty("resultJson");
|
||||
expect(payload).not.toHaveProperty("executionCancellation");
|
||||
expect(payload).not.toHaveProperty("runnerProfileJson");
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps pre-Stop input pending until a later user message authorizes one continuation", async () => {
|
||||
|
||||
@@ -12,6 +12,8 @@ import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, readNativeLocalProc
|
||||
import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js";
|
||||
import { pendingCaseHandoffForRun, cancelCaseHandoffsForIssue, isCaseHandoffSourceReleased } from "./case-handoff-lifecycle.js";
|
||||
import { publishActivity, type ActivityPublication } from "../activity-log.js";
|
||||
import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js";
|
||||
import { publishLiveEvent } from "../live-events.js";
|
||||
import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js";
|
||||
import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js";
|
||||
import {
|
||||
@@ -37,6 +39,7 @@ import { settleSlackConversation } from "../slack-conversation-lifecycle.js";
|
||||
import { readQueuedInteractionResponse } from "../queued-interaction-response.js";
|
||||
import {
|
||||
isAcknowledgedNativeStop,
|
||||
acknowledgedNativeApiStopExecutionHasStopped,
|
||||
acknowledgedNativeStopExecutionHasStopped,
|
||||
} from "../acknowledged-native-stop.js";
|
||||
import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js";
|
||||
@@ -506,27 +509,61 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
|
||||
completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope });
|
||||
}
|
||||
}
|
||||
const stopped = state?.status === "cancelled" ? await getRun(runId) : null;
|
||||
// Native interrupt acknowledgement is not physical containment. Publish the
|
||||
// shared Stop receipt only after every exact remote lease confirms teardown.
|
||||
// Keep workspace preservation separate: a stopped provider can still owe files.
|
||||
if (stopped?.status === "cancelled" &&
|
||||
parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&
|
||||
(stopped.runtimeMode === "native" ? isAcknowledgedNativeStop(stopped)
|
||||
: stopped.runtimeMode === "legacy" && await runUsedConversationAdapter(db, stopped)) &&
|
||||
await remoteExecutionHasStopped(db, companyId, runId)) {
|
||||
await db.update(heartbeatRuns).set({
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({
|
||||
executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),
|
||||
state: "acknowledged", acknowledgedAt: new Date().toISOString(),
|
||||
proof: "provider_termination_receipt" },
|
||||
...(stopped.runtimeMode === "legacy" ? { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } : {}),
|
||||
})}::jsonb`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`,
|
||||
stopped.runtimeMode === "native" ? sql`${heartbeatRuns.resultJson}->'nativeCancellation' = ${JSON.stringify(stopped.resultJson?.nativeCancellation)}::jsonb` : undefined));
|
||||
// Finalization can append a harmless event after proof was read. Retry
|
||||
// only a lost API compare-and-set, with fresh complete proof each time.
|
||||
for (let attempt = 0; attempt < 3; attempt++) {
|
||||
const stopped = state?.status === "cancelled" ? await getRun(runId) : null;
|
||||
// Native interrupt acknowledgement is not physical containment. Require
|
||||
// exact remote lease teardown, or the API provider's terminal receipt and
|
||||
// the local controller's retirement, before publishing shared Stop proof.
|
||||
// Keep workspace preservation separate: a stopped provider can still owe files.
|
||||
if (stopped?.status === "cancelled" && stopped.companyId === companyId &&
|
||||
parseObject(stopped.resultJson?.executionCancellation).state === "requested" &&
|
||||
(stopped.runtimeMode === "native" ? isAcknowledgedNativeStop(stopped)
|
||||
: stopped.runtimeMode === "legacy" && await runUsedConversationAdapter(db, stopped))) {
|
||||
const apiStopped = stopped.runtimeMode === "native" && await acknowledgedNativeApiStopExecutionHasStopped(db, stopped);
|
||||
if (!apiStopped && !await remoteExecutionHasStopped(db, companyId, runId)) return;
|
||||
const acknowledged = await db.update(heartbeatRuns).set({
|
||||
resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({
|
||||
executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation),
|
||||
state: "acknowledged", acknowledgedAt: new Date().toISOString(),
|
||||
proof: apiStopped ? "provider_api_stop_receipt" : "provider_termination_receipt" },
|
||||
...(stopped.runtimeMode === "legacy" ? { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } : {}),
|
||||
})}::jsonb`,
|
||||
updatedAt: new Date(),
|
||||
}).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId),
|
||||
eq(heartbeatRuns.status, "cancelled"),
|
||||
sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`,
|
||||
apiStopped ? and(eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"),
|
||||
eq(heartbeatRuns.nextEventSeq, stopped.nextEventSeq),
|
||||
eq(heartbeatRuns.agentId, stopped.agentId), eq(heartbeatRuns.nativeIssueId, stopped.nativeIssueId!),
|
||||
isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId),
|
||||
eq(heartbeatRuns.nativeSessionId, stopped.nativeSessionId!), eq(heartbeatRuns.runnerInstanceId, stopped.runnerInstanceId!),
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'executionPlacement' = ${JSON.stringify({ kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" })}::jsonb`,
|
||||
sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`,
|
||||
sql`jsonb_build_object(
|
||||
'companyId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'companyId',
|
||||
'runId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'runId',
|
||||
'agentId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'agentId',
|
||||
'issueId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'issueId'
|
||||
) = ${JSON.stringify({ companyId, runId, agentId: stopped.agentId, issueId: stopped.nativeIssueId })}::jsonb`,
|
||||
sql`jsonb_build_object(
|
||||
'normalizedSessionId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'session'->'normalizedSessionId',
|
||||
'driverKind', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'session'->'driverKind'
|
||||
) = ${JSON.stringify({ normalizedSessionId: stopped.nativeSessionId, driverKind: "cursor_cloud_api" })}::jsonb`,
|
||||
sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'provider'->>'kind' = 'cursor_cloud'`,
|
||||
sql`not exists (select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${heartbeatRuns.companyId}
|
||||
and ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id})`) : undefined,
|
||||
stopped.runtimeMode === "native" ? sql`${heartbeatRuns.resultJson}->'nativeCancellation' = ${JSON.stringify(stopped.resultJson?.nativeCancellation)}::jsonb` : undefined)).returning({ id: heartbeatRuns.id });
|
||||
if (acknowledged.length > 0) {
|
||||
publishLiveEvent({
|
||||
companyId: stopped.companyId,
|
||||
type: "heartbeat.run.status",
|
||||
payload: buildHeartbeatRunStatusLiveEventPayload(stopped),
|
||||
});
|
||||
}
|
||||
if (!apiStopped || acknowledged.length > 0) return;
|
||||
} else return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -135,7 +135,7 @@ export interface HeartbeatRuntimeExecutionInput {
|
||||
executionTarget: Selection["workspace"]["executionTarget"];
|
||||
workspaceOperationRecorder: ReturnType<ReturnType<typeof workspaceOperationService>["createRecorder"]>;
|
||||
nativeWorkspaceSync: Selected["nativeWorkspaceSync"];
|
||||
workspaceRestoreSource: Parameters<typeof recordLegacyWorkspaceRestoreFailure>[3];
|
||||
workspaceRestoreSource: Parameters<typeof recordLegacyWorkspaceRestoreFailure>[3] | null;
|
||||
remoteExecution: unknown;
|
||||
};
|
||||
config: Pick<Selection["config"], "runtimeConfig" | "resolvedConfig" | "managedAiRuntime"> & {
|
||||
@@ -937,6 +937,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe
|
||||
input.effects.onWorkspaceRestoreFailure(requiredWorkspaceRestoreEvidence);
|
||||
// Retention is the fallback even if recording this receipt fails.
|
||||
input.effects.onProviderResourceDisposition("stop_and_retain");
|
||||
if (workspaceRestoreSource === null) throw new Error("legacy_workspace_restore_source_unavailable");
|
||||
await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource);
|
||||
}
|
||||
for (const stream of ["stdout", "stderr"] as const) {
|
||||
|
||||
@@ -2,10 +2,11 @@ import { randomUUID } from "node:crypto";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { activityLog, agents, companies, completionContracts, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { activityLog, agents, companies, completionContracts, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
|
||||
import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js";
|
||||
import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../chat-control-recovery-stop.js";
|
||||
import { buildNativeRuntimeContext } from "../native-runtime/runtime-context.js";
|
||||
import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js";
|
||||
@@ -28,7 +29,8 @@ vi.mock("../native-runtime/native-workspace-sync.js", async (original) => ({
|
||||
prepareNativeWorkspaceSync: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUNTIME_SELECTION_TEST_DATABASE_URL");
|
||||
const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport();
|
||||
describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () => {
|
||||
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
|
||||
let db: Db;
|
||||
@@ -36,7 +38,9 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
|
||||
beforeAll(async () => {
|
||||
instructionRoot = await mkdtemp(join(tmpdir(), "paperclip-selection-instructions-"));
|
||||
await writeFile(join(instructionRoot, "AGENTS.md"), "Complete the assigned task.\n");
|
||||
database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-selection");
|
||||
if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl);
|
||||
database = externalDatabaseUrl ? { connectionString: externalDatabaseUrl, cleanup: async () => {} }
|
||||
: await startEmbeddedPostgresTestDatabase("paperclip-runtime-selection");
|
||||
db = createDb(database.connectionString);
|
||||
await db.execute(sql`set client_min_messages = warning`);
|
||||
}, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
|
||||
@@ -78,6 +82,65 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () =
|
||||
const coordinators = (id: string) => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, id));
|
||||
const sandbox = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", reusableLeaseConfigured: false };
|
||||
|
||||
it.each([false, true])("persists the Cursor provider API placement and completion authority without a Computer lease (ephemeral: %s)", async ephemeral => {
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", String(ephemeral));
|
||||
const { input, run, issue } = await fixture();
|
||||
input.agent = { ...input.agent, adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification" } };
|
||||
await db.update(agents).set({ adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id));
|
||||
input.workspace.activeEnvironmentLease = null;
|
||||
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig,
|
||||
agent: input.agent, issue, target: null, workspaceId: null });
|
||||
const result = await selectHeartbeatRuntime(db, input);
|
||||
expect(result.selected).toBe(true);
|
||||
if (!result.selected) throw new Error("Expected native Cursor selection");
|
||||
expect(result.nativeExecution?.provider).toMatchObject({ kind: "cursor_cloud", model: "caller-model" });
|
||||
expect(await read(run.id)).toMatchObject({ runtimeMode: "native", driverKind: "cursor_cloud_api", runnerProfileJson: {
|
||||
backend: "cursor_cloud_api", executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" },
|
||||
nativeExecutionInput: result.nativeExecution,
|
||||
} });
|
||||
expect(await db.select().from(completionContracts)).toHaveLength(1);
|
||||
expect(await coordinators(run.id)).toHaveLength(1);
|
||||
expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled();
|
||||
expect(input.stopControllerLease).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each(["cursor_cloud_api", null] as const)("restores a historical Cursor workspace binding without allocating a Computer (driver: %s)", async driverKind => {
|
||||
const { input, run, issue } = await fixture();
|
||||
input.agent = { ...input.agent, adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification" } };
|
||||
await db.update(agents).set({ adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id));
|
||||
const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId, sourceIssueId: issue.id,
|
||||
name: "Previously admitted Cursor workspace", mode: "shared_workspace", strategyType: "project_primary", cwd: "/previous-cursor-workspace" }).returning();
|
||||
input.workspace.persistedExecutionWorkspace = workspace;
|
||||
input.workspace.activeEnvironmentLease = null;
|
||||
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig,
|
||||
agent: input.agent, issue, target: null, workspaceId: workspace.id });
|
||||
const selected = await selectHeartbeatRuntime(db, input);
|
||||
if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native Cursor selection");
|
||||
expect(selected.nativeExecution.binding.executionWorkspaceId).toBe(workspace.id);
|
||||
const saved = await read(run.id);
|
||||
const historicalProfile: Record<string, unknown> = { ...saved.runnerProfileJson, sessionCheckpoint: { retained: "Cursor provider checkpoint" } };
|
||||
delete historicalProfile.executionPlacement;
|
||||
await db.update(heartbeatRuns).set({ runnerProfileJson: historicalProfile, driverKind }).where(eq(heartbeatRuns.id, run.id));
|
||||
input.run = await read(run.id);
|
||||
input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: input.run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig,
|
||||
agent: input.agent, issue, target: null, workspaceId: null });
|
||||
input.session.persistedRunnerProfile = historicalProfile;
|
||||
input.session.persistedNativeExecutionInput = selected.nativeExecution;
|
||||
input.workspace.persistedExecutionWorkspace = null;
|
||||
input.workspace.executionWorkspace.cwd = "/private-cursor-controller";
|
||||
vi.mocked(buildNativeRuntimeContext).mockClear();
|
||||
const resumed = await selectHeartbeatRuntime(db, input);
|
||||
expect(resumed).toMatchObject({ selected: true, nativeExecution: selected.nativeExecution, nativeRunnerInstanceId: selected.nativeRunnerInstanceId });
|
||||
expect(await read(run.id)).toMatchObject({ driverKind: "cursor_cloud_api" });
|
||||
expect((await read(run.id)).runnerProfileJson).toMatchObject({ ...historicalProfile,
|
||||
executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } });
|
||||
expect(await db.select().from(executionWorkspaces)).toEqual([workspace]);
|
||||
expect(await db.select().from(completionContracts)).toHaveLength(1);
|
||||
expect(await coordinators(run.id)).toHaveLength(1);
|
||||
expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled();
|
||||
expect(buildNativeRuntimeContext).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("persists and admits a standing-chat response contract without requiring the unavailable action", async () => {
|
||||
const { input, issue, agent, run } = await fixture();
|
||||
const [conversation] = await db.update(issues).set({ title: "Chat with CEO", description: null,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js";
|
||||
import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js";
|
||||
import { museRunnerBroker } from "../muse-runner-broker.js";
|
||||
import { assertEphemeralNativeLifecycle, readRemoteWorkspaceAuthority } from "../remote-workspaces.js";
|
||||
@@ -123,7 +124,7 @@ export interface HeartbeatRuntimeSelectionInput {
|
||||
persistedExecutionWorkspace: { id: string; metadata?: Record<string, unknown> | null } | null;
|
||||
executionWorkspace: Omit<Parameters<typeof buildNativeExecutionInput>[0]["workspace"], "id">;
|
||||
executionTarget: Parameters<typeof prepareNativeWorkspaceSync>[0]["target"];
|
||||
activeEnvironmentLease: Pick<Parameters<typeof prepareNativeWorkspaceSync>[0], "lease">;
|
||||
activeEnvironmentLease: Pick<Parameters<typeof prepareNativeWorkspaceSync>[0], "lease"> | null;
|
||||
isDotRun: boolean;
|
||||
projectRepositoryPaths: string[];
|
||||
onCheckpoint?: Parameters<typeof prepareNativeWorkspaceSync>[0]["onCheckpoint"];
|
||||
@@ -279,7 +280,11 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
throw new Error("native_runtime_ineligible: issue is required");
|
||||
}
|
||||
const nativeExecutionWorkspaceId =
|
||||
persistedExecutionWorkspace?.id ?? run.id;
|
||||
persistedNativeExecutionInput?.provider.kind === "cursor_cloud"
|
||||
// Older Cursor runs keep their admitted binding identity. Their task
|
||||
// files and Computer are not part of the recovered API controller.
|
||||
? persistedNativeExecutionInput.binding.executionWorkspaceId
|
||||
: persistedExecutionWorkspace?.id ?? run.id;
|
||||
const nativeReviewContext = readNativeReviewAssignmentContext(context);
|
||||
const nativeReview = nativeReviewContext ? await measureRunPhase("runtime_selection.review", () => getNativeReviewAssignment(db, {
|
||||
companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id,
|
||||
@@ -514,10 +519,10 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
((executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && !!executionTarget.runnerTask) ||
|
||||
supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution));
|
||||
const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ??
|
||||
(["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession
|
||||
(["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession
|
||||
? { mode: "per_turn" as const, idleTimeoutMs: null }
|
||||
: environmentLifecyclePolicy ?? agentLifecyclePolicy);
|
||||
if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend)) {
|
||||
if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend)) {
|
||||
assertEphemeralNativeLifecycle(effectiveLifecyclePolicy.mode);
|
||||
}
|
||||
if (
|
||||
@@ -774,7 +779,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
}),
|
||||
) ??
|
||||
`# ${issueRef.identifier ?? issueRef.id}: ${issueRef.title}`,
|
||||
!["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) && projectRepositoryPaths.length > 0
|
||||
!["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) && projectRepositoryPaths.length > 0
|
||||
? `## Project repositories\nThe task workspace also contains these editable Git repositories:\n${projectRepositoryPaths.map((repo) => `- ${repo}`).join("\n")}`
|
||||
: null,
|
||||
].filter(Boolean).join("\n\n"),
|
||||
@@ -970,6 +975,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
runnerProfileJson: {
|
||||
...nativeRuntimeResolution.profile,
|
||||
...lockedProfile,
|
||||
...(nativeExecution?.provider.kind === "cursor_cloud" ? { executionPlacement: resolveProviderApiPlacement({ agent, run }) } : {}),
|
||||
nativePreparationFencingVersion: !lockedRun.runnerInstanceId && !lockedRun.nativeSessionId
|
||||
? NATIVE_PREPARATION_FENCING_VERSION
|
||||
: lockedProfile.nativePreparationFencingVersion,
|
||||
@@ -1058,7 +1064,8 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
});
|
||||
if (!nativeSelected) return { selected: false };
|
||||
stopControllerLease();
|
||||
nativeWorkspaceSync = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null
|
||||
if (nativeExecution?.provider.kind !== "cursor_cloud" && !activeEnvironmentLease) throw new Error("native_environment_lease_unavailable");
|
||||
nativeWorkspaceSync = nativeExecution?.provider.kind === "cursor_cloud" || readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null
|
||||
: await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({
|
||||
db,
|
||||
runId: run.id,
|
||||
@@ -1066,7 +1073,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele
|
||||
workspaceId: nativeExecutionWorkspaceId,
|
||||
workspaceLocalDir: executionWorkspace.cwd,
|
||||
target: executionTarget,
|
||||
lease: activeEnvironmentLease.lease,
|
||||
lease: activeEnvironmentLease!.lease,
|
||||
restartRecovery: runOptions.nativeRestartRecovery,
|
||||
sameRunRecovery: Boolean(runOptions.nativeLeaseOwner),
|
||||
resourceDisposition: providerResourceDispositionForRun,
|
||||
|
||||
@@ -126,6 +126,56 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
};
|
||||
return { input, source, issue, run, envOrchestrator };
|
||||
}
|
||||
it.each([false, true])("places Cursor Cloud in its provider API without a Computer under managed-only policy (ephemeral: %s)", async ephemeral => {
|
||||
vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", String(ephemeral));
|
||||
const { input, source, issue, run } = await fixture();
|
||||
input.agent = { ...input.agent, adapterType: "paperclip_runner", adapterConfig: {
|
||||
provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification",
|
||||
} };
|
||||
await db.update(agents).set({ adapterType: input.agent.adapterType, adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id));
|
||||
const settingsBefore = await instanceSettingsService(db).get();
|
||||
try {
|
||||
await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true, enableNativeRunner: true });
|
||||
const selectionInput = {
|
||||
run: input.run, agent: input.agent,
|
||||
task: { issueRef: issue, issueContext: issue, issueId: issue.id, context: input.task.context,
|
||||
executionProjectId: issue.projectId, workspaceProjectContext: { hasWorkspace: true }, boundSourceWorkspace: null,
|
||||
hasSelectedWorkspaceSource: true, taskWorkspaceIntent: null, responsibleUserId: null },
|
||||
policy: { ...input.policy }, config: { config: input.agent.adapterConfig, sourceWorkspaceConfig: input.agent.adapterConfig, issueAssigneeOverrides: null },
|
||||
services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db),
|
||||
instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: async () => null, findExecutionWorkspaceHolder: async () => null },
|
||||
};
|
||||
const selection = await selectHeartbeatEnvironment(db, selectionInput);
|
||||
expect(selection).toMatchObject({ providerApiPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" },
|
||||
selectedEnvironmentId: null, selectedEnvironmentForConfig: null, executionPolicy: { managedSandboxOnly: true } });
|
||||
input.environment = { ...input.environment, selectedEnvironmentId: selection.selectedEnvironmentId,
|
||||
selectedEnvironmentForConfig: selection.selectedEnvironmentForConfig, environmentResolution: selection.environmentResolution,
|
||||
resolvedInstanceSettings: selection.resolvedInstanceSettings };
|
||||
input.policy.executionPolicy = selection.executionPolicy;
|
||||
const acquire = vi.spyOn(input.services.envOrchestrator, "acquireForRun");
|
||||
const realize = vi.spyOn(input.services.envOrchestrator, "realizeForRun");
|
||||
const resolveWorkspace = vi.spyOn(input.services, "resolveWorkspaceForRun");
|
||||
const result = await prepareHeartbeatWorkspace(db, input);
|
||||
expect(result).toMatchObject({ selectedEnvironment: null, activeEnvironmentLease: null, executionTarget: null,
|
||||
persistedExecutionWorkspace: null, projectRepositoryPaths: [], hostExecutionWorkspaceConfig: {}, skipHostProvision: true });
|
||||
expect(result.executionWorkspace.cwd).toContain(`/cursor-cloud-controllers/${input.agent.companyId}/${input.agent.id}/${issue.id}`);
|
||||
expect(result.executionWorkspace).toMatchObject({ projectId: null, repoUrl: null, repoRef: null, created: false });
|
||||
expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
|
||||
expect(acquire).not.toHaveBeenCalled(); expect(realize).not.toHaveBeenCalled(); expect(resolveWorkspace).not.toHaveBeenCalled();
|
||||
expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run.id))).toEqual([]);
|
||||
expect(await db.select().from(executionWorkspaces)).toEqual([]);
|
||||
expect((await instanceSettingsService(db).getExperimental()).enableManagedSandboxOnly).toBe(true);
|
||||
await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" });
|
||||
await expect(selectHeartbeatEnvironment(db, selectionInput)).rejects.toMatchObject({ status: 422,
|
||||
details: { code: "provider_api_kubernetes_execution_unsupported" } });
|
||||
} finally {
|
||||
await instanceSettingsService(db).updateGeneral({ executionMode: settingsBefore.general.executionMode ?? "any" });
|
||||
await instanceSettingsService(db).updateExperimental({
|
||||
enableManagedSandboxOnly: settingsBefore.experimental.enableManagedSandboxOnly,
|
||||
enableNativeRunner: settingsBefore.experimental.enableNativeRunner,
|
||||
});
|
||||
}
|
||||
});
|
||||
it.each(["unconfigured", "operator-disabled"] as const)("auto-enrollment keeps %s native sandboxes on their existing host path", async mode => {
|
||||
const { input, source, envOrchestrator } = await fixture();
|
||||
input.agent.adapterType = "paperclip_runner";
|
||||
@@ -1713,7 +1763,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
if (transport !== "daytona") {
|
||||
await expect(prepareNativeWorkspaceSync({ db, runId: input.run.id, companyId: issue.companyId,
|
||||
workspaceId: first.persistedExecutionWorkspace!.id, workspaceLocalDir: first.executionWorkspace.cwd,
|
||||
target, lease: result.activeEnvironmentLease.lease })).resolves.toBeNull();
|
||||
target, lease: result.activeEnvironmentLease!.lease })).resolves.toBeNull();
|
||||
}
|
||||
expect(admittedRoot).not.toBe(first.executionWorkspace.cwd);
|
||||
} else {
|
||||
@@ -1817,7 +1867,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace
|
||||
const result = await prepareHeartbeatWorkspace(db, input);
|
||||
expect((await readIssue(issue.id)).executionWorkspaceId).toBe(result.persistedExecutionWorkspace!.id);
|
||||
expect(result.persistedExecutionWorkspace!.name).toBe("Realized workspace");
|
||||
result.activeEnvironmentLease.lease.metadata!.remoteCwd = "/rebound";
|
||||
expect(result.workspaceRestoreSource.metadata?.remoteCwd).toBe("/original");
|
||||
result.activeEnvironmentLease!.lease.metadata!.remoteCwd = "/rebound";
|
||||
expect(result.workspaceRestoreSource!.metadata?.remoteCwd).toBe("/original");
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js";
|
||||
import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, RemoteWorkspaceWriterActiveError, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js";
|
||||
import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WorkspaceBusyDeferral } from "./retries.js";
|
||||
import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js";
|
||||
@@ -127,10 +128,10 @@ export interface HeartbeatWorkspacePreparationInput {
|
||||
executionPolicy: Parameters<typeof evaluateExecutionAllowlist>[0];
|
||||
};
|
||||
environment: {
|
||||
selectedEnvironmentId: string;
|
||||
selectedEnvironmentId: string | null;
|
||||
localEnvironment: Environment;
|
||||
selectedEnvironmentForConfig: Environment | null;
|
||||
environmentResolution: ExecutionWorkspaceEnvironmentResolution;
|
||||
environmentResolution: ExecutionWorkspaceEnvironmentResolution | { environmentId: null; source: "default" };
|
||||
resolvedInstanceSettings: Awaited<ReturnType<ReturnType<typeof instanceSettingsService>["get"]>>;
|
||||
};
|
||||
config: {
|
||||
@@ -169,13 +170,14 @@ function readNonEmptyString(value: unknown): string | null {
|
||||
export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) {
|
||||
const sourceIssueId = input.reuse.reusableExistingExecutionWorkspace?.sourceIssueId
|
||||
?? input.reuse.existingExecutionWorkspace?.sourceIssueId ?? input.task.issueId;
|
||||
if (!sourceIssueId || input.reuse.isDotRun) return prepareHeartbeatWorkspaceLocked(db, input);
|
||||
if (!sourceIssueId || input.reuse.isDotRun || resolveProviderApiPlacement(input)) return prepareHeartbeatWorkspaceLocked(db, input);
|
||||
return withIsolatedRepositorySourceLock(db, input.agent.companyId, sourceIssueId,
|
||||
() => prepareHeartbeatWorkspaceLocked(db, input));
|
||||
}
|
||||
|
||||
async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspacePreparationInput) {
|
||||
const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input;
|
||||
const providerApiPlacement = resolveProviderApiPlacement(input);
|
||||
const fresh = input.task.freshWorkspace?.mode === "fresh_isolated_workspace";
|
||||
const timeWorkspace = createHeartbeatPreparationTiming(run.id);
|
||||
const {
|
||||
@@ -217,7 +219,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
} = input.reuse;
|
||||
// An ordinary clean task directory already has a new per-attempt identity.
|
||||
// It is not a repository source and must not pass through Git copy admission.
|
||||
const freshNeedsSeed = fresh && !isDotRun && !useIsolatedTaskDirectory &&
|
||||
const controllerOnly = isDotRun || providerApiPlacement !== null;
|
||||
const freshNeedsSeed = fresh && !controllerOnly && !useIsolatedTaskDirectory &&
|
||||
!(nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId);
|
||||
const {
|
||||
envOrchestrator,
|
||||
@@ -237,14 +240,14 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
}
|
||||
}
|
||||
const enrolledRemoteAuthority = readRemoteWorkspaceAuthority((reusableExistingExecutionWorkspace ?? existingExecutionWorkspace)?.metadata) !== null;
|
||||
let remoteAuthority = !isDotRun && shouldUseRemoteWorkspaces({
|
||||
let remoteAuthority = !controllerOnly && shouldUseRemoteWorkspaces({
|
||||
adapterType: agent.adapterType,
|
||||
environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
|
||||
experimentalEnabled: (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" || resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled) && Boolean(issueId && issueRef)
|
||||
&& !projectExecutionWorkspacePolicy?.storageProviderPluginId,
|
||||
existingWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
|
||||
});
|
||||
assertEphemeralWorkspaceExecution({ externalConversation: isDotRun,
|
||||
if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun,
|
||||
adapterType: agent.adapterType,
|
||||
pluginTaskWorkspace, remoteAuthority });
|
||||
const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({
|
||||
@@ -260,7 +263,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
"Remote workspace storage requires a task.", { reason: "remote_workspace_task_required" });
|
||||
const remoteSourceState: { value: Extract<Awaited<ReturnType<typeof resolveRemoteWorkspaceSource>>, { authority: "remote" }> | null } = { value: null };
|
||||
const pluginConfig = parseObject(selectedEnvironmentForConfig?.config);
|
||||
if (fresh && !isDotRun && pluginTaskWorkspace) {
|
||||
if (fresh && !controllerOnly && pluginTaskWorkspace) {
|
||||
const plugin = typeof pluginConfig.pluginKey === "string" ? await pluginRegistryService(db).getByKey(pluginConfig.pluginKey) : null;
|
||||
const driver = plugin?.manifestJson.environmentDrivers?.find(value => value.driverKey === pluginConfig.driverKey);
|
||||
if (driver?.freshWorkspaceVersion !== 1) throw new WorkspaceValidationFailure(
|
||||
@@ -298,10 +301,11 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
}
|
||||
: null,
|
||||
resolveSelectedEnvironmentDriver: async () => {
|
||||
if (providerApiPlacement) return "provider_api";
|
||||
const preflightEnvironment = await envOrchestrator.resolveEnvironment(
|
||||
{
|
||||
companyId: agent.companyId,
|
||||
selectedEnvironmentId,
|
||||
selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id,
|
||||
localEnvironmentId: localEnvironment.id,
|
||||
},
|
||||
);
|
||||
@@ -329,7 +333,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
remoteAuthority = false;
|
||||
// Source resolution can retain a configured host folder. Check the
|
||||
// resolved authority before materializing it or acquiring a sandbox.
|
||||
assertEphemeralWorkspaceExecution({ externalConversation: isDotRun,
|
||||
if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun,
|
||||
adapterType: agent.adapterType, pluginTaskWorkspace, remoteAuthority });
|
||||
}
|
||||
else {
|
||||
@@ -354,10 +358,10 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
};
|
||||
}
|
||||
}
|
||||
if (isDotRun) {
|
||||
if (controllerOnly) {
|
||||
// This is private controller storage, never a provider filesystem.
|
||||
// External providers receive workspace.access=none and cwd=null.
|
||||
const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", (persistedNativeExecutionInput?.provider.kind ?? mergedConfig.provider) === "muse" ? "muse-controllers" : "dot-controllers", agent.companyId, run.id);
|
||||
const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", providerApiPlacement ? "cursor-cloud-controllers" : (persistedNativeExecutionInput?.provider.kind ?? mergedConfig.provider) === "muse" ? "muse-controllers" : "dot-controllers", agent.companyId, ...(providerApiPlacement ? [agent.id, issueId ?? run.id] : [run.id]));
|
||||
await fs.mkdir(cwd, { recursive: true, mode: 0o700 });
|
||||
return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null,
|
||||
workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] };
|
||||
@@ -497,7 +501,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
let providerFresh: PluginFreshWorkspace | undefined;
|
||||
let previousProviderWorkspaceId: string | null = null;
|
||||
let previousProviderBindingRevision = 0;
|
||||
if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && !isDotRun && issueId) {
|
||||
if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && !controllerOnly && issueId) {
|
||||
const [task] = await db.select({ workspaceId: issues.executionWorkspaceId, revision: issues.workspaceBindingRevision }).from(issues)
|
||||
.where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId)));
|
||||
previousProviderWorkspaceId = task?.workspaceId ?? null;
|
||||
@@ -513,7 +517,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
previousWorkspaceId: previousProviderWorkspaceId });
|
||||
}
|
||||
}
|
||||
const pluginPrepared = isDotRun || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, {
|
||||
companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId,
|
||||
issueId, runId: run.id, agentId: agent.id, responsibleUserId,
|
||||
existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null,
|
||||
@@ -544,7 +548,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
boundWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace,
|
||||
}));
|
||||
const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh" || hostProvisionDriver === "plugin";
|
||||
const hostExecutionWorkspaceConfig = isDotRun ? {} :
|
||||
const hostExecutionWorkspaceConfig = controllerOnly ? {} :
|
||||
stripHostWorkspaceProvision({
|
||||
config: fresh && !remoteAuthority ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig,
|
||||
skipHostProvision,
|
||||
@@ -726,7 +730,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
warnings: [], created: false, branchCreatedByRuntime: false,
|
||||
baseRefSha: pluginPrepared.metadata?.workspaceProviderBaseSha as string | null,
|
||||
} as RealizedExecutionWorkspace, reusedExecutionWorkspace: Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy }
|
||||
: isDotRun || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
|
||||
: controllerOnly || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
|
||||
{
|
||||
requestedShouldReuseExisting,
|
||||
existingExecutionWorkspaceId:
|
||||
@@ -854,7 +858,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
const resolvedProjectWorkspaceId =
|
||||
resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"
|
||||
? reusableExistingExecutionWorkspace.projectWorkspaceId
|
||||
: isDotRun ? null : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
|
||||
: controllerOnly ? null : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
|
||||
let persistedExecutionWorkspace: ExecutionWorkspace | null = null;
|
||||
let issueExecutionWorkspaceIdForRun =
|
||||
issueRef?.executionWorkspaceId ?? null;
|
||||
@@ -875,7 +879,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
const bindIssueToPersistedExecutionWorkspace = async (
|
||||
workspace: ExecutionWorkspace | null,
|
||||
) => {
|
||||
if (isDotRun || !issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {
|
||||
if (controllerOnly || !issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {
|
||||
return;
|
||||
}
|
||||
const nextIssueWorkspaceMode =
|
||||
@@ -996,6 +1000,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
try {
|
||||
if (remoteAuthority) {
|
||||
if (!remoteSourceState.value || !issueId) throw new Error("remote_workspace_source_missing");
|
||||
if (!selectedEnvironmentId) throw new Error("remote_workspace_environment_missing");
|
||||
await assertRemotePreparationOwned();
|
||||
const reserved = await reserveRemoteWorkspace({
|
||||
db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id,
|
||||
@@ -1053,7 +1058,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
),
|
||||
},
|
||||
))
|
||||
: !isDotRun && !persistedNativeExecutionInput && (resolvedProjectId || issueRef)
|
||||
: !controllerOnly && !persistedNativeExecutionInput && (resolvedProjectId || issueRef)
|
||||
? await timeWorkspace("workspace_record_create", () => executionWorkspacesSvc.create({
|
||||
companyId: agent.companyId,
|
||||
projectId: resolvedProjectId,
|
||||
@@ -1178,7 +1183,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
const projectRepositoryPaths: string[] = [];
|
||||
if (remoteAuthority) projectRepositoryPaths.push(...(remoteSourceState.value?.source.repositories ?? [])
|
||||
.filter(repository => !repository.readOnly).map(repository => repository.relativePath));
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !controllerOnly && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) {
|
||||
const taskRepositories = await timeWorkspace("workspace_repository_admission", () =>
|
||||
executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId,
|
||||
issueId, workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd,
|
||||
@@ -1231,6 +1236,21 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
})
|
||||
.where(eq(heartbeatRuns.id, run.id));
|
||||
}
|
||||
if (providerApiPlacement) {
|
||||
await controllerLease.assertOwned();
|
||||
return {
|
||||
resolvedWorkspace, hostExecutionWorkspaceConfig, skipHostProvision: true,
|
||||
latestWorkspaceConfigMetadata, workspaceConfigFreshness, workspaceOperationRecorder,
|
||||
executionWorkspace, reusedExecutionWorkspace, resolvedWorkspaceReusePolicy,
|
||||
persistedExecutionWorkspace: null, projectRepositoryPaths: [],
|
||||
environmentAcquireStartedAtMs: Date.now(), selectedEnvironment: null,
|
||||
activeEnvironmentLease: null,
|
||||
realizationResult: { remoteExecution: null, executionTarget: null, persistedExecutionWorkspace: null,
|
||||
lease: null, workspaceRealization: null },
|
||||
environmentRealizeEndedAtMs: Date.now(), workspaceRealization: null,
|
||||
executionTarget: null, workspaceRestoreSource: null,
|
||||
};
|
||||
}
|
||||
const environmentAcquireStartedAtMs = Date.now();
|
||||
let acquiredEnvironment: Awaited<
|
||||
ReturnType<typeof envOrchestrator.acquireForRun>
|
||||
@@ -1254,7 +1274,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
}
|
||||
acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({
|
||||
companyId: agent.companyId,
|
||||
selectedEnvironmentId,
|
||||
selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id,
|
||||
localEnvironmentId: localEnvironment.id,
|
||||
adapterType: agent.adapterType,
|
||||
adapterConfig: parseObject(agent.adapterConfig),
|
||||
@@ -1423,7 +1443,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace
|
||||
if (wake) wake.executionContinuation = continuation;
|
||||
}
|
||||
}
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
|
||||
if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !controllerOnly && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId)
|
||||
&& !(executionTarget?.kind === "remote" && (executionTarget.transport === "ssh"
|
||||
|| (executionTarget.transport === "sandbox" && executionTarget.runnerTask)))) {
|
||||
const workspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? nativeRecoveryExecutionWorkspaceId!;
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { IdleApiActivity } from "./idle-api-activity.js";
|
||||
|
||||
describe("authenticated API idle activity", () => {
|
||||
it("is passive without successful authentication and expires without timers or dependency work", () => {
|
||||
let monotonic = 1;
|
||||
const activity = new IdleApiActivity(() => monotonic);
|
||||
expect(activity.active()).toBe(false);
|
||||
activity.record();
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic += 599_999;
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic++;
|
||||
expect(activity.active()).toBe(false);
|
||||
});
|
||||
it("extends only from the actual latest successful use and ignores wall clock changes", () => {
|
||||
let monotonic = 1;
|
||||
const activity = new IdleApiActivity(() => monotonic);
|
||||
activity.record();
|
||||
monotonic += 300_000;
|
||||
activity.record();
|
||||
monotonic += 300_001;
|
||||
expect(activity.active()).toBe(true);
|
||||
monotonic += 299_999;
|
||||
expect(activity.active()).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,12 @@
|
||||
/** Authenticated API use is real activity; rejecting an unknown key is not. */
|
||||
export class IdleApiActivity {
|
||||
private until = 0;
|
||||
constructor(private readonly now: () => number = () => performance.now()) {}
|
||||
record(): void {
|
||||
this.until = Math.max(this.until, this.now() + 10 * 60_000);
|
||||
}
|
||||
active(): boolean {
|
||||
return this.now() < this.until;
|
||||
}
|
||||
}
|
||||
export const idleApiActivity = new IdleApiActivity();
|
||||
Loaded 100 of 161 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user