diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index 3b590332789..da1824cfa51 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -98,7 +98,20 @@ const stackedBase = { const routeCases = [ ["allowlisted PR", {}, awsRunner], - ["allowlisted profile label", { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: "profile-ci" }, awsRunner], + ...["profile-ci", "full-ci"].flatMap((label) => { + const event = { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: label }; + return [ + [`allowlisted ${label} label`, event, awsRunner], + [`${label} with unlisted author`, { ...event, EVENT_PR_AUTHOR_ID: "99999999" }], + [`${label} with unlisted sender`, { ...event, EVENT_SENDER_ID: "99999999" }], + [`${label} with unlisted rerun actor`, { ...event, TEST_TRIGGERING_ACTOR_ID: "99999999" }], + [`${label} on non-owner PR with owner sender and rerun`, { + ...event, EVENT_PR_AUTHOR_ID: "99999999", TEST_PR_AUTHOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]", + }], + [`${label} with non-owner sender`, { ...event, EVENT_SENDER_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + [`${label} with non-owner rerun actor`, { ...event, TEST_TRIGGERING_ACTOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ]; + }), ["unrelated label", { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: "documentation" }], ["validated stacked base", stackedBase, awsRunner], ["lagging PR API base snapshot", { TEST_PR_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, awsRunner], diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 7fc7416f369..3afa9f0859d 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -86,8 +86,8 @@ jobs: [[ "$EVENT_BASE_REPOSITORY_ID" == '1170821064' ]] || fail_closed 'unexpected base repository ID' [[ -n "$EVENT_BASE_REF" ]] || fail_closed 'missing base branch' [[ "$EVENT_ACTION" =~ ^(opened|reopened|synchronize|labeled)$ ]] || fail_closed 'unsupported pull_request action' - if [[ "$EVENT_ACTION" == 'labeled' && "${EVENT_LABEL_NAME:-}" != 'profile-ci' ]]; then - fail_closed 'only the profile-ci label can request a profiling run' + if [[ "$EVENT_ACTION" == 'labeled' && ! "${EVENT_LABEL_NAME:-}" =~ ^(profile-ci|full-ci)$ ]]; then + fail_closed 'only the profile-ci or full-ci labels can request CI' fi jq -e ' diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 0e53269a9e2..a0c4e79591f 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -757,6 +757,43 @@ blocks concurrent task or run updates. It allows audit inserts to retain their foreign-key `KEY SHARE` locks without waiting on identity acquisition. The audit foreign keys and their deletion behavior remain enforced. +## Execution admission accounting + +The execution-admission module (`server/src/modules/execution-admission/`) owns +three tables. Nothing writes admissions or caps in production yet, and no claim +path enforces a project maximum. + +- `project_task_execution_policies` stores a project's maximum concurrent tasks + apart from `projects`, so project readers never return an unenforced cap. No + row means never configured. A row with a null maximum means an explicit "no + limit", which a later legacy migration must not overwrite. Maximums are + positive. The row is deleted with its project. +- `execution_admissions` stores one admission attempt of a run. A run can be + released before dispatch and admitted again, so `(run_id, attempt)` is unique + and a partial unique index allows one unsettled attempt per run. The row keeps + the context revision, the nullable primary project as provenance (no foreign + key) and the settlement identity, which is set together with `settled_at`. +- `execution_admission_project_charges` stores one row per distinct admitted + project, including unlimited projects. An admission without projects has no + rows. Composite `(company_id, …)` foreign keys keep admissions and projects in + one company. Rows exist only while the admission is unsettled: settlement + deletes the whole set in one transaction, and `(company_id, project_id)` is + indexed for active counts, so lookups scale with active work, not history. + +Archiving a project changes no charge. Deleting a project with a charge returns +`409 project_execution_admission_pending`; the foreign key is the backstop. +Deleting an agent refuses unsettled admissions on its runs and removes settled +ones. Company deletion removes the company's admissions before its runs and +projects. + +The proposed global lock order for the admission transaction is: company row +(`FOR NO KEY UPDATE`), native finalization recovery owner, the whole project set +in ascending ID order (`FOR NO KEY UPDATE`, one statement), issues, wake +requests, the run, the external-admission advisory lock, the admission row and +its charges, then budget reservations. Policy updates take the same project +lock. Settlement takes no project lock. Only the project-set step is implemented +so far; claim paths keep their current locks until they are integrated. + ## Switching between modes The database mode is controlled by `DATABASE_URL`: diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 182afe1d850..7997238437e 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -164,6 +164,9 @@ history, changed PR heads and unverifiable ancestry still deny AWS execution. Missing or malformed settings and unverifiable authorization use GitHub-hosted runners. A maintainer rerunning an outside contributor's PR does not grant it AWS access. +Both `full-ci` and `profile-ci` label events use this same authorization path. +The labels request CI scope; they do not grant AWS access or move an authorized +CODEOWNER run to GitHub-hosted capacity. Other label events do not request CI. Authorized test jobs and the final `verify` and `e2e` aggregates use the selected AWS fleet. A stale admission roster can allocate only the short metadata gate; the fresh CODEOWNERS check still denies PR execution for a removed owner. @@ -1848,6 +1851,17 @@ host, sandbox, or Git remote. Files changed since the last save still depend on the live sandbox: losing that allocation can lose those edits. There is no periodic workspace backup in this release. +Remote workspace command timeouts report `remote_workspace_operation_timed_out` +with the operation (`prepare`, `repository`, `archive_helper`, `capture`, `restore`, +or `cleanup`) and its timeout in milliseconds. Command output is excluded from +that diagnostic. A timeout does not prove the remote process stopped or that a +save failed before publication. Reconcile the existing workspace and operation +receipts before recovery; the transport does not automatically repeat the command. +Export cleanup keeps this typed timeout diagnostic in the operation's +`metadata.exportCleanup.lastError`. Other cleanup errors retain a generic message. +The saved snapshot stays acknowledged and the cleanup obligation stays pending +until the existing reconciliation path settles it. + Files reads use the current committed saved version when remote authority is clean and has no active writer, even while its old allocation remains recorded. The saved loader verifies the company, workspace, snapshot, and archive before @@ -1878,6 +1892,34 @@ or recovery, unsafe paths, and explicitly concurrent shared writers block import. An interrupted apply retains its journal and admission hold; retry the same command to reconcile it before starting another writer. +An import preserves the workspace's lifecycle state and the task's current +workspace binding. It can import an archived or superseded workspace without +reopening the task or selecting that older workspace for future turns. A changed +task binding during the import blocks publication. + +If an archived folder has already been removed, the importer can reconstruct it +in private scratch space from its encrypted task-directory restore point or +validated Git restoration recipe. For older Git workspaces without a recipe, +it requires a successful branch-deletion receipt and the exact saved commit in +the verified source repository. It never substitutes today's default branch. +Git restore points retain unpushed history and staged and unstaged edits, even +when the source repository has pruned the saved branch. Missing, expired, +corrupt, or unbound sources block the import. Publication still requires an +independent download and restore of the new encrypted snapshot. The old folder +remains absent, and the importer does not change the original restore point. + +Shared physical folders require Cloud's offline enrollment maintenance process. +A task-level hold cannot fence another workspace record pointing at the same +folder. Maintenance verifies that legacy deployments have stopped and that no +runs are active or queued, then imports nested task directories before parent +roots. A parent can omit a runtime-created nested task folder only after an +independent restore and source comparison prove that its committed snapshot +covers those bytes. The parent snapshot records that dependency, and retention +pins the child snapshot until the parent is deleted. Changed or unavailable +nested data blocks publication. Ordinary user folders remain in the archive. +Interrupted maintenance resumes the same import journals; completed records +leave the pending set. Original folders remain on the retained volume. + Supported sources include empty task folders, repository URLs, and managed mutable repositories. Authorized mentioned projects attach all their configured Git repositories as writable copies under `.paperclip-repositories/` in the task diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index c9455c9de14..dd49412c784 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -1729,7 +1729,10 @@ Scheduler must skip invocation when: - an existing run is active - hard budget limit has been hit -Budget admission and accounting serialize under the company accounting lock. +Authoritative paid-work admission and accounting mutations serialize under the +company accounting lock. Preliminary invocation checks use a read-only snapshot +and take that lock only when a hold, incident, or enforcement transition needs +reconciliation; reconciliation rereads current state under the lock. The dispatch boundary rechecks current company, agent, and project policies and reserves capacity before paid work starts. Accounting commits budget blocks and durable stop intent without waiting for provider shutdown or reconciling unrelated diff --git a/doc/TOASTS.md b/doc/TOASTS.md index f2e0489fb00..fc984f1a640 100644 --- a/doc/TOASTS.md +++ b/doc/TOASTS.md @@ -63,6 +63,10 @@ never persisted or published on the shared company event bus. Authorized events still reach cache handling and component subscribers when eligibility is false. Lookup failure is logged but does not discard the authorized event. Existing task/run privacy checks and payload redaction remain in force. +Task activity titles, fallback text, and links use the authorized task identifier. +If the optional identifier lookup fails, the event still arrives and uses `Task` +unless the browser already has the identifier. Permission-check failures do not +use this fallback. When an authenticated comment event omits the comment ID, the visible task thread refreshes through its authorized HTTP read instead of trying to hydrate the comment. Closed sockets cannot deliver stale notifications after account/company changes. diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 325148106d2..138f2f91559 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -756,6 +756,8 @@ Normal issue-thread feedback can still wake the current assignee according to th Restart recovery revisits accepted deferred messages through the latest issue run, including historical mention queues for another agent. Ordinary messages for a former owner retain their assignment handoff. Native sources require their matching committed finalization, settled environment leases, and released finalizer and copyback ownership before recovery clears old issue claims. A successful retained warm release can satisfy the lease check. Released local bookkeeping without a provider allocation and a matching SSH bookkeeping release need no provider cleanup receipt. Plugin releases record their successful acknowledgement. Pending or failed cleanup and durable owners still hold work. A successful fenced cleanup completion clears its in-flight claim in the same database write, so a restart before retry bookkeeping cannot strand saved input. The normal release checks preserve cleanup holds, pauses, and operator Stops. Recovery checks for a competing turn under the issue lock before it promotes saved input. +Repeated eligibility notifications must not move an unchecked, ready deferred receipt behind receipts already reconsidered. Paginated reconstruction preserves pending ready receipts' dispatch priority while refreshing their current state. Receipts whose wait deadline has passed join the ready queue on reconstruction. Future wait grace periods, claimed-work deadlines, infrastructure retry backoffs, and current admission checks remain effective. + A paused recipient does not fail an accepted deferred request. Release and periodic recovery preserve its pending receipt and may process an eligible request behind it. Resuming the agent allows normal admission to reconsider the saved input. Missing, foreign, terminated, and unapproved recipients retain their rejection rules. Explicit task holds and operator Stops still prevent unauthorized execution. This does not replay historical failed requests. Accepted agent feedback must survive a child changing to `done` before its active run exits. Deferred wake promotion may reopen that completed child only for its current assignee, with explicit agent resume intent and live tracked comments from another author. Claim promotion before reopening. Cancelled tasks, deleted comments, self-authored comments, empty continuations, and agent continuations without explicit intent keep their existing suppression rules. Normal pause, ownership, authorization, and budget gates still apply. @@ -949,6 +951,8 @@ The task thread exposes the guarded Retry action for failed or timed-out convers A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed. +Other legacy queued-run gate cancellations also stay out of unknown-action reconciliation when the terminal run has no retained execution authority, session, workspace recovery, resources, or execution events. The same proof removes the effect of an old false recovery hold only for its exact company, task, and run; the audit row remains. Started or conflicting work retains its hold. Review waits retain their guarded Retry path. New work still passes the current approval, dependency, budget, pause, and ownership gates. + A successful connected run can still have newer admitted input waiting behind it. Task Retry resumes that input's existing durable chat receipt through the inbound owner, instead of replaying the successful run as a failed attempt. The original @@ -1600,8 +1604,9 @@ agent, workspace, retained session, and termination of prior execution. The checkpoint must match the exact Runner, lease, run, session, turn, and item. A missing restart claim does not remove these checks or authorize a fresh session. -After a plugin session passes provider recovery and durable run admission, its -local suspension receipt is removed. The running session then owns authority; +After a remote sandbox session passes provider recovery and durable run admission, +its local suspension receipt is removed. This applies to built-in sandbox runners +and plugin tasks. The running session then owns authority; the old receipt must not conflict with a later warm run's journal identity. Failed admission retains the receipt. Normal suspension writes a new receipt. @@ -3027,7 +3032,12 @@ resumption path. This does not consume a provider attempt or stop the retained sandbox. A locked run/coordinator check rejects Stop, changed ownership or any provider progress. Startup restores even an overdue wait to the same scheduler; ordinary resumption rechecks current permissions and task authority. No idle -scan is added. Per-turn continuation reconstructs a missing suspended-state +scan is added. The same wait applies when a compatible successful predecessor +still has a live finalization lease, even with full local controller projections. +An expired or undated owner, incomplete canonical commands and incompatible +execution bindings retain their existing reconciliation holds. Once the owner +releases its lease, the existing resumption claim admits the same run once. +Per-turn continuation reconstructs a missing suspended-state receipt from the admitted sandbox or prepared-task owner, including after a previous read failed with only a PRP projection written. diff --git a/doc/idle-incoming-wake.md b/doc/idle-incoming-wake.md new file mode 100644 index 00000000000..05dfac779c8 --- /dev/null +++ b/doc/idle-incoming-wake.md @@ -0,0 +1,118 @@ +# Authenticated incoming-wake proof + +The Cloud control signer can attach `idleCapabilities.incomingWake` to an owned +`task-drain:read` assertion. The tenant verifies the existing stack, action, +audience, expiry and one-use request ID, then validates the closed capability +shape. A tenant query, browser session, API key or forwarded identity header +cannot install this capability. Unknown versions and routes fail closed. + +The report uses outer version 3 and includes the exact version-1 incoming proof. +Cloud compares its requested capability and the initial/final proof before stop. +Without this capability, versions 1/2 and all existing ingress blockers remain. +Older Cloud/app combinations retain their conservative behavior. + +## Supported profiles + +`tenant-api-http-v2` allows saved agent and board API key rows to remain while +sleeping. Cloud resolves the target from its verified host registry, coalesces a +bounded wake, waits for tenant readiness, refreshes routing, then forwards the +original HTTP request once. Ordinary `/api/instance` administration routes are +included. The separately reserved Muse protocol, non-API MCP routes, upgrades, +encoded/noncanonical paths and methods outside GET/HEAD/OPTIONS/POST/PUT/PATCH/ +DELETE do not gain this wake path. + +The Cloud capability authenticates the **host's ability to receive work**. Opaque +tenant keys are authenticated only by the awakened tenant, under current key, +expiry, user, membership, company, agent and resource permissions. No raw key or original +credential hash is copied into Cloud. A signed final-drain projection supplies +only per-cycle derived wake fingerprints. Unknown inputs cannot consume +provider-start quota. Tenant authentication still rejects any later revocation. + +Successful current board/agent key authentication followed by a successful HTTP +response retains ten minutes of local API activity, measured with a monotonic +clock. Further successful use extends the deadline. Rejection, expiry, revoked +membership, failed requests and drain refusals do not extend it. The existing +local-work probe observes this timestamp; there is no timer, SQL, renewal or +network work from the hold, and browser-entry metrics are unchanged. + +`github-broker-events-v1` uses the durable Cloud connector queue from the existing +connector-event wake protocol. The signed proof binds the current enrolled +connector instance and canonical fingerprint of all current GitHub bindings. +The tenant reconstructs those bindings through the original connection-event +owner and exact completed GitHub bot registration ownership. An active broker +bot can retain its endpoint and paired channel connection only when setup has +finished and every binding matches. Missing, foreign, changed, unsupported or +unsettled binding state prevents sleep. Original GitHub signature verification +and current endpoint/connection authorization still run on delivery. + +Personal GitHub MCP connections additionally owe credential continuity work. +This change exposes their required connection IDs but keeps them blocked. The +stacked domain-deadline change must supply its inspected future continuity proof +before those rows can be exempted. A signed catalog name alone is insufficient. +Legacy scheduled alarms remain conservative when combined with ingress until +that downstream composition is present. + +## Delivery and admission guarantees + +Synchronous HTTP is not durable acceptance. A request arriving during stopping +or cold start is held only before forwarding, under finite capacity/time bounds. +An active-route race with tenant drain returns its real 503 and Retry-After; +clients retry through their normal API/domain idempotency contract. A streamed +mutation is never automatically replayed after an ambiguous upstream outcome. +Wake/readiness failure, manual suspension, deployment recovery holds, wrong +hosts and exhausted admission return non-success without acknowledging work. + +Broker events have stronger persistence: the existing queue target survives a +lost wake hint, failed wake, process restart and lease expiry. A 202 hint does not +settle delivery. Only the current event lease acknowledgment does. Duplicate +transport delivery uses the domain's existing receipts/atomic claims. + +Cloud API admission is explicitly bounded per process: 64 waiting requests, +8 per stack, 4 per source, 32 pending provider preparations; source request +budget 24/minute, source wake budget 3/10 minutes, stack wake budget 8/hour, +and aggregate wake budget 32/minute. Readiness waiting is at most 90 seconds. +A timed-out provider preparation remains coalesced until it actually settles. +The stack budget accommodates ordinary ten-minute quiet cycles with boundary +headroom. These are availability limits, not an unrestricted API promise. +Only the existing explicitly configured trusted-proxy hop contract may resolve +a forwarded source IP; otherwise the socket address is used. A shared proxy/NAT +therefore shares its source limits and can receive truthful retry refusals. + +## Explicit remaining coverage + +Direct GitHub, routine, Slack, voice and X webhooks retain their existing +configured-ingress blockers until their exact response budget, retry/retention, +authenticated wake and replay contracts are qualified. Cloud X forwarding by +itself does not establish wake. Saved outbound HTTP MCP remains passive under +its existing narrow rule; public MCP/SSE subscriptions, Dot, Muse and gateway +resources retain their own blockers. Discord Gateway, local MCP stdio, and other +required persistent sockets need owner-proved stop, reconnect and sequence replay. +Telegram/Teams/Photon transports require their own actual producer/lifecycle +classification. Hosted previews belong to their separate ingress changes. + +The source/loopback/PostgreSQL tests establish the exercised protocol boundaries; +they do not establish a fleet deployment, actual provider/Neon suspension, +provider startup latency or cost savings. Deployed combined qualification remains +separate and must include every enabled transport. + + +## Pre-wake API admission + +The v2 API profile requires an authenticated per-sleep-cycle key projection before compute stops. The signed challenge is derived from the newly random owned idle hold and stack. In the same read-only snapshot, the app projects up to 1000 active key rows into domain-separated salted fingerprints; original key hashes and raw keys stay private. Board-key expiry is carried and enforced before wake. Unknown, duplicate, malformed or overflow projection fails closed. The complete initial and final receipts must match; changing a credential or challenge refuses stop. + +Cloud stores the final receipt in one internal stack column under the sleep operation lock. Unknown requests cannot consume provider-start quota. Possession of a matching snapshot key only admits a bounded wake; tenant authentication and current permissions still run on delivery. A key revoked after the snapshot can therefore receive no tenant authority, even if it once qualified for wake. Wake claims recheck the exact cycle, matching key and expiry atomically. Normal next sleep re-reads revocations. Successful wake, manual/archive/delete or a known sleep abort clears the receipt; uncertain stop and a same-cycle failed wake retain recovery. A restarted host reads the durable receipt. No global TTL strands nonexpiring saved keys during long sleep, and no periodic credential refresh worker is added. + + +Removed GitHub endpoints are retained history only after the ordinary removal +writer archives the endpoint with its timestamp and generation, archives and +disables the same-company channel connection, clears its credential references, +and releases endpoint leases. Any retained lease, incomplete registration or +malformed cleanup state still blocks. This exemption covers only GitHub; archived +persistent transports remain blocked. Actions, deliveries and publications keep +their separate completion checks even when their endpoint has been removed. + +The service also keeps its existing local-work probe active while runtime +initialization or retirement is unsettled. Both supported runtime registries +report failed retirement even after `get` and `list` hide the runtime. Successful +owner retry clears that debt; a failed full service shutdown retains the probe. +These checks read process-local state and add no polling, network or SQL. diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index a07fc4d9f6f..9b71b4cf034 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -644,3 +644,13 @@ retry dates and package/object/native-asset retention timestamps are earliest maintenance eligibility, not promised delivery deadlines. Workers reconstruct persisted rows and filesystem access leases on startup, and use the existing coordinator's drain/standby/shutdown controls. No external Cloud alarm or provider suspension is established by these tests. + + +## Negotiated incoming work + +A signed, stack/owner-scoped Cloud incoming-wake capability can produce a version-3 +safety report for saved API keys and exactly bound brokered GitHub bot endpoints. +Tenant authorization still runs after wake. Personal GitHub credential continuity +and unsupported transports retain their blockers. See [authenticated incoming +wake proof](idle-incoming-wake.md) for scope, HTTP versus durable-event guarantees, +rate/availability bounds, compatibility and remaining transport dependencies. diff --git a/doc/observability.md b/doc/observability.md index abadb0e6538..c04aef97125 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -45,10 +45,15 @@ Keep credential values out of logs and diagnostic reports. ## Budget admission, accounting, and enforcement The operator-gated `paperclip.budgets` tracer records +`budget.admission.inspect`, `budget.admission.reconcile`, `budget.admission.evaluate`, `budget.admission.reserve`, `budget.accounting.transaction`, `budget.accounting.settlement`, `budget.accounting.lock_wait`, `budget.exposure.evaluate`, `budget.enforcement.deliver`, `budget.enforcement.scope`, `budget.cancellation.deliver`, and `budget.accounting.recover`. +Inspection spans cover the preliminary read-only snapshot. Reconciliation spans +exist only when that inspection finds a transition requiring the company lock; +compare their counts to measure how often preliminary checks bypass the lock. +Authoritative reservation and transactional evaluation retain their own spans. Transaction spans include commit time. Lock-wait spans cover acquisition of the company accounting lock. Settlement spans cover the ledger and reservation transaction. Enforcement spans measure asynchronous delivery; cancellation diff --git a/doc/plans/2026-10-28-cursor-cloud-native-runner.md b/doc/plans/2026-10-28-cursor-cloud-native-runner.md index fbdecbf71c0..6264582fb0c 100644 --- a/doc/plans/2026-10-28-cursor-cloud-native-runner.md +++ b/doc/plans/2026-10-28-cursor-cloud-native-runner.md @@ -15,6 +15,36 @@ and dependent [product PR #18069](https://github.com/paperclipai/paperclip/pull/ Neither PR is merged. Product PR #18069's Verification section owns the latest candidate, check status and final acceptance decision. +## Hosted placement correction — 2026-10-30 + +Native Cursor Cloud uses a provider API controller. Cursor hosts the model and +commands; Paperclip runs only its qualified API client and bounded private artifact +stager. This placement has no Paperclip Computer, lease, task checkout or host +provisioning command. It is derived from the validated native Cursor profile and +persisted with the admitted run; recovery retains that backend and placement. +Earlier Cursor profiles retain their admitted workspace binding as identity; +recovery does not reopen that workspace or allocate a Computer. Known backend +and driver contradictions fail closed, while historical missing driver values +remain recoverable. +The generic managed-sandbox-only policy and local Computer denial stay intact. +Kubernetes-only execution remains unsupported in save, Test, inventory and run +admission. An unavailable Computer does not advertise local Legacy execution. +Legacy direct agents retain their existing selection and execution behavior. + +Ordinary native setup clears the Computer binding and explains Cursor's hosted +environment. Save/reopen and Test use the same account/configuration contract as +run admission. The Cloud front door must pass the exact authenticated native-run +callback to the tenant; Core still enforces live run authority and per-turn scope. + +The ordinary Cloud card is available only with native rollout and a qualified +controller. A server-held new setup draft can use that managed API default before +model selection. It keeps its setup hold and does not inherit another harness's +model or permissions. Unsaved automatic Test uses the advertised native default; +saved Legacy agents still require an explicit conversion. + +This correction requires fresh hosted acceptance on its final image. Local and +synthetic checks do not qualify that deployed journey or erase prior failures. + ## Review correction checkpoint — 2026-10-29 20:25 UTC All six original acceptance gates have passed on their recorded candidates; diff --git a/doc/plans/2026-11-03-accounting-contention-and-budget-enforcement.md b/doc/plans/2026-11-03-accounting-contention-and-budget-enforcement.md new file mode 100644 index 00000000000..5ab838d2f97 --- /dev/null +++ b/doc/plans/2026-11-03-accounting-contention-and-budget-enforcement.md @@ -0,0 +1,400 @@ +# Accounting contention and best-effort budget enforcement + +Date: 2026-11-03 +Status: Investigation complete; first admission optimization implemented locally, remaining stages proposed +Source revision: `367f29478d14003cc5c0f161b7ce8f3164993634` + +Follow-up: [Lock-free accounting design](2026-11-03-lock-free-accounting-design.md) +defines the requested target architecture. Its append-only receipt path replaces +the run-lock proposal below; this document retains the investigation evidence +and the possible behavior-preserving first migration step. + +## Recommendation + +Keep receipt persistence, deduplication, final settlement, and audit transactional. +Make budget detection and automatic stopping a measured, short-lag control. Move +frequent receipt writes to transactions scoped to their run or paid operation. +Evaluate budget policies in coalesced batches outside those transactions. + +First remove avoidable admission transactions and measure the remaining critical +sections. Do not begin with distributed spending allocations or a new accounting +service. A short lock at an infrequent boundary can be acceptable; serializing +every company's receipt stream and admission preflight is the scaling concern. + +The first implementation now inspects preliminary eligibility in a repeatable-read, +read-only snapshot. Healthy checks and checks with no policies bypass the company +writer lock. Holds, hard blocks, recovering accounting and incidents needing +resolution fall back to the existing mutating owner, which revalidates current +state under the lock. Transactional admission, reservations and receipt accounting +retain their authority. Separate inspection/reconciliation spans expose the bypass +ratio. This does not implement the later append-only accounting architecture. + +Configured-policy tradeoffs: a healthy check adds an incident lookup, and an +unheld blocking/recovering scope can compute exposure once during inspection +and again during locked revalidation. Already-held scopes request reconciliation +before computing exposure. Qualify configured-policy workloads as well as the +predominant no-policy case; the change is not a claim of lower CPU everywhere. + +The first milestone's budget-service suite passes 18 tests against real +PostgreSQL. A controlled writer holds the company accounting lock while routine +checks finish; authoritative admission on the same reader instead reaches its +500 ms lock timeout. Coverage includes healthy/warning policies, manual and +orphaned pauses, hold release, incident cleanup and cross-company rejection. +This establishes the lock-bypass contract, not deployed latency or CPU savings. +Seven additional accounting/enforcement/replay/tracing suites pass 55 tests, +for 73 passing targeted tests across eight files with no skips. +The full workspace typecheck passes via `node scripts/typecheck-low-memory.mjs`, +and `node scripts/check-module-boundaries.mjs` passes. +The full workspace build passes via `PAPERCLIP_TYPECHECK_LOW_MEMORY=1 pnpm -r +--workspace-concurrency=1 build`, including the current-source Runner release +binary, server, UI and CLI. The initial incomplete local Copilot build artifact +was repaired with the pinned installer before the successful canonical retry. + +This proposes a product-contract change for the later stages. It does not change +the current spec or silently redefine existing configured budgets. The first +implementation stage below preserves existing enforcement semantics. + +## What the investigation established on the source revision above + +### The lock protects several different responsibilities + +[`withAccountingTransaction`](../../server/src/services/accounting-transaction.ts) +takes `FOR NO KEY UPDATE` on the company row before invoking its callback. It +holds that lock until the transaction ends. Different companies can proceed +independently; writers for one company serialize. Ordinary report reads already +use a read-only repeatable-read snapshot and do not take this writer lock. + +| Path | Work sharing the company lock before this change | +| --- | --- | +| Usage receipt | Lock run, validate source/sequence and replay hash, insert immutable receipt, update running usage, evaluate affected budgets if monetary exposure changed | +| Admission | Read company/agent/project state and policies, calculate exposure, reconcile budget holds/incidents, reserve estimated capacity before dispatch | +| Final settlement | Recover durable receipt files, validate completion, settle reservation, insert ledger entries, update projections and runtime totals, acknowledge the run, evaluate budgets | +| Other mutations | Direct cost reports, billing imports/corrections, accounting repairs, paid browser/decision/fast-response/classifier operations, and agent lifecycle operations | + +The wrapper was unconditional. Having no configured budget did not bypass the +lock. The public `getInvocationBlock` method also used this mutating wrapper. +The implementation above separates its preliminary reads while preserving +the reconciliation and resume effects that some callers need. + +Primary owners are [usage receipts](../../server/src/services/usage-receipts.ts), +[budgets](../../server/src/services/budgets.ts), +[reservations](../../server/src/services/budget-reservations.ts), and +[run settlement](../../server/src/services/run-cost-accounting.ts). +Queue, dispatch, and lifecycle code should consume their contracts rather than +implement another budgeting system. + +### The current guarantee is already narrower than a dollar cap + +Reservations serialize admission against known exposure and estimated holds. +They do not restrict how much an already-dispatched provider can charge. +`reservationCents` defaults to zero. A zero reservation still prevents duplicate +dispatch, but does not reserve a positive allowance for future model spend. + +The [implementation spec](../SPEC-implementation.md) explicitly acknowledges that +mid-run enforcement depends on available provider reports or supported prices. +Providers reporting only at completion cannot provide exact mid-call detection. +Browser work can pass a provider cost cap; preserve such supported limits. + +The current rules are approximately: + +- Admission counts ledger spend plus the maximum of each unfinished operation's + reservation and observed spend, plus separate browser exposure. +- Threshold stopping uses ledger plus observed running spend, rather than + treating reservation estimates as money already spent. +- Pending terminal accounting and configured unknown-price restrictions can + independently block work. +- Final settlement replaces live exposure with the ledger charge atomically. + +Accurate recording of received evidence and instantaneous control of external +spend are separate contracts. Some prices remain estimates or unknown even when +their receipt is recorded correctly. + +### The critical section can do substantial work + +When applicable policies exist, [exposure evaluation](../../server/src/services/budget-exposure.ts) +executes four sequential aggregate statements over ledger entries, held +reservations joined to runs, pending/live runs, and browser work. Policies are +batched into those statements; it is not four statements per policy. With no +billable policies, these four queries are skipped. + +A changed monetary receipt can invoke this evaluation synchronously. Duplicate, +stale, unchanged, and subscription receipts take different branches; not every +receipt performs the aggregate scans. Native delivery already suppresses +accepted source-event replays in its receiver, so duplicate suppression is a +measurement question rather than an assumed large win. + +The hot inputs are usage and turn-lifecycle events, not every streamed text +token. Native capture handles usage reports and turn start/terminal events. +Legacy capture also durably spools before its database attempt. Recovery can +still read receipt files while the settlement transaction holds the company +lock, although it builds an initial file-identity index outside the lock. + +[Monthly spend projections](../../server/src/services/costs.ts) update +`companies.spentMonthlyCents` for each ledger insert, and the applicable agent +counter. Merely deleting the explicit company lock would leave an implicit +shared-row write bottleneck and break assumptions around settlement and repair. +PostgreSQL updates acquire conflicting row locks too. See the +[PostgreSQL locking contract](https://www.postgresql.org/docs/current/explicit-locking.html#LOCKING-ROWS). + +### Live traces justify a focused optimization, not a CPU diagnosis + +Read-only observability queries examined one live host. The matched one-hour +sample is **2026-11-03 16:45–17:45 UTC**, service revision +`3910352c923afda320b2a5134e523932ee1d0633`. +The aggregate measurements are reproduced below. Private trace links and host +identifiers remain outside the repository. + +| Observed span | Count | p50 | p95 | p99 | Maximum | +| --- | ---: | ---: | ---: | ---: | ---: | +| Company lock acquisition | 9,421 | 13.00 ms | 206.85 ms | 725.14 ms | 4,301.53 ms | +| Accounting transaction | 9,421 | 87.16 ms | 486.95 ms | 1,147.10 ms | 5,594.19 ms | +| Admission evaluation | 8,082 | 43.45 ms | 166.84 ms | 342.13 ms | 1,212.92 ms | +| Final settlement | 216 | 355.90 ms | 1,894.05 ms | 3,149.67 ms | 5,594.33 ms | + +These are span counts; nested spans must not be added as independent operations. +There were 23 `budget.admission.reserve` spans in the same window. The much larger +admission-evaluation count warrants caller attribution, not an assumption that +all evaluations represent new paid dispatches. + +Of 8,292 exposure evaluations on that host in that hour, **8,280 had zero policies** +and 12 had one. That is 99.86% with zero policies at this call boundary; it does +not prove that every company has no budgets. These empty evaluations are cheap, +but their surrounding admission/accounting work still takes the company lock. + +One inspected admission transaction took 4.183 seconds, with 4.090 seconds in +lock acquisition. Its exposure child had zero policies and took about 0.026 ms. +A separate settlement transaction took 5.594 seconds, including 4.302 seconds +in acquisition and a zero-policy exposure evaluation. +Both inspected traces report mean sample rate 1. + +Limits of this evidence: + +- The acquisition span measures the complete awaited lock query. It includes + database/network round trips and Node scheduling; it is not a PostgreSQL + lock-wait measurement. Transaction duration also includes transaction setup + and commit. Do not subtract unrelated percentiles to estimate hold time. +- The spans do not identify the locking company, blocker, receipt frequency, + or operation caller. They do not establish one company's throughput ceiling + or the cause of the previously observed CPU utilization. +- The sample identifies one host. It does not establish fleet-wide behavior or + independently qualify the public instance serving each request. +- The local accounting wrapper, receipt, exposure, budget service, and run-cost + files match the traced revision in Git. Other deployed components and database + workload have not been qualified against this checkout. +- The existing [budget benchmark](../../scripts/benchmark-budget-enforcement.ts) + varies unrelated idle-agent count. It does not test concurrent receipt writers, + 10,000 active runs, or growing ledger history. It was inspected, not run here. + +## Proposed behavior + +| Responsibility | Proposed contract | +| --- | --- | +| Receipt acceptance | Durable commit before acknowledgement; source ownership, sequence, hash conflicts and company scope remain enforced | +| Ledger and settlement | Idempotent transaction scoped to the run/invocation; no lost charges, duplicate application, or transient live-plus-final double counting | +| Budget detection | Coalesced evaluation from committed evidence; a measurable delay is allowed | +| New work | Check current authoritative pause/Stop and known budget-block state at dispatch; concurrent admission against a not-yet-reflected spend increase can overshoot | +| Automatic cancellation | Commit durable stop intent, then use the existing scoped cancellation worker; provider shutdown remains a separate obligation | +| Manual Stop and authorization | Preserve current execution-authority checks; budget approximation does not weaken them | +| Unknown accounting | Retain unresolved obligations and existing configured unknown-usage restrictions; missing evidence is never a zero-cost receipt | + +Suggested initial service targets, to validate before adopting as promises: + +- Coalesce dirty budget work over approximately one second while work exists. +- p99 from committed priced receipt to durable threshold-stop intent at most + five seconds under the qualified workload. Measure provider reporting delay + and cancellation latency separately. +- For applicable spend limits, stop admitting additional paid work when the + oldest unevaluated accounting change exceeds fifteen seconds. This is a + proposed protection against a failed evaluator, not a hard spending cap. + An idle company's old evaluation timestamp alone must never trigger it. +- With no dirty changes, unresolved retries, or actual budget deadlines, the + evaluator performs no recurring database scans. + +Best effort needs a stated overspend model. As an illustration, 10,000 runs each +spending $0.001/second consume $10/second; five seconds of extra detection delay +adds about $50, before unreported and uncancellable in-flight work. Those are +hypothetical rates, not a measured cost estimate or maximum. + +In general, additional spend depends on fleet burn rate multiplied by reporting, +evaluation, and stop delay, plus already-committed work. Without bounds on those +quantities, do not advertise a bounded dollar overshoot. A strict end-to-end cap +would require provider-enforced limits or preauthorized bounded work; distributed +allocations alone cannot cap a provider that can exceed its allocation. + +## Implementation stages + +### 1. Attribute the cost and eliminate unnecessary admission transactions + +Owners: accounting transaction/tracing services, budget admission API, and its +queue/dispatch/lifecycle callers. + +Add a closed operation-kind label to accounting spans so receipt, finalization, +preflight, reservation, policy edit, correction, and lifecycle work can be +distinguished. Capture acquisition-query time, time from acquisition return to +transaction completion, query count, and replay/outcome categories. Keep the +operator-gated Observability contract and content-free attributes; this is not +first-party Telemetry. Label the client-observed held interval honestly because +it includes commit/return latency. Use sampled PostgreSQL wait/blocker evidence +to establish actual contention, with SQL values and credentials excluded. + +Separate read-only preliminary eligibility from mutating reconciliation. +Preflights should not take a company writer lock just to discover no applicable +policy. Keep the existing authoritative reservation and policy check at final +dispatch in this stage. Budget changes, rollover, settlement and explicit +reconciliation continue to own their necessary effects. Do not introduce a +stale cached "no budgets" result that can bypass a newly configured limit. + +Measure identical receipt replay and unchanged snapshots before prioritizing +their optimization. Any early replay return must prove the immutable receipt +and relevant projection state agree. Native event identity and lifecycle events +cannot be discarded just because their monetary amount is unchanged. + +Deliver a real-PostgreSQL benchmark using the service boundaries below, and the +small behavior-preserving optimization justified by the caller measurements. +If this removes today's significant delay, ship it independently; later stages +remain a deliberate scale/product decision. + +### 2. Persist receipts per run and evaluate budgets asynchronously + +Owners: usage receipts, native usage delivery, budget exposure, budgets, and the +existing accounting/enforcement delivery lifecycle. + +In one run-scoped transaction: + +1. Validate current source/ownership and lock the run. +2. Insert or verify the receipt with its unique identity and fingerprint. +3. Replace the run's accepted cumulative usage/exposure projection. +4. Persist a versioned pending-evaluation marker and register the normal + transaction-aware wake. +5. Commit before acknowledging native delivery or removing legacy spool evidence. + +Do not take the company accounting writer lock or evaluate all affected policies +inside this receipt transaction. Keep final and partial receipts durable; batch +derived budget evaluation rather than dropping intermediate accounting evidence. + +Use the shared delivery coordinator for coalescing, retry and deadlines. Pending +state must be scoped to the changed run/operation, or an append-only change; +updating one company dirty counter for every receipt would recreate the lock. +Consumption must compare revisions or claim durable work so a concurrent change +cannot be cleared by an older evaluator. A timestamp/sequence high-water mark +alone cannot safely assume database transactions commit in allocation order. + +Read ledger and live contributions in one consistent database snapshot, or use +transactionally consistent replacement contributions. The present four separate +read-committed aggregates are not safe to reuse unchanged once other operations +in the same company can settle concurrently: live-to-ledger +replacement must not be missed or counted twice across queries. + +Evaluate each affected policy once per batch, revalidate its revision before +committing block/resume state, and retain durable cancellation generations. +Updates arriving during evaluation remain pending. Recovery reconstructs pending +work after restart; rollback must not publish a false wake or acknowledgement. +Use short policy-state transactions where necessary. Preserve manual holds and +committed stop obligations during policy changes and price corrections. + +This is the first stage that permits an additional detection/admission race +window. Update the product contract together with implementation; removing only +the wrapper is not a valid intermediate change. + +### 3. Remove remaining company writes from frequent settlement + +Owners: run-cost accounting, costs, billing reconciliation/import, accounting +integrity, paid-service invocation owners, and DB/shared contracts. + +Settle ledger entries, reservation state, final receipt acknowledgement, and +run-specific accounting together under the operation's lock. Use database +uniqueness plus conflict/hash verification for concurrent idempotency; a +read-before-insert check previously protected by the company lock is insufficient. +Retain narrow agent coordination for shared runtime totals where needed. + +Move company monthly display-counter updates to versioned, coalesced projection +work. Update integrity inspection/repair and all counter consumers to understand +projection freshness and checkpoint identity. Reporting from ledger snapshots +remains available; intentional projection lag must not look like lost money. +Handle backdated events, credits/corrections, UTC rollover and lifetime budgets. + +For receipt-file recovery, stage immutable candidate envelopes outside the broad +transaction, then validate source/version and closure under the operation lock. +Retain disk evidence until the final owning transaction commits. A concurrent +late receipt or replaced source must still prevent premature acknowledgement. + +Apply the same boundary to browser runs, decision-model invocations, fast +responses, prompt-injection classification, and externally reported charges. +Inventory administrative users of `withAccountingTransaction` separately; +agent deletion, repair, and lifecycle safety must not depend on a lock removed +from receipt writers. Specify and test the new lock order, including issue/run +ownership and foreign-key interactions. Keep supported provider spending caps. + +Start with bounded batches and indexed aggregates. If history growth or batch +time misses the target, maintain incremental scope/window contributions. If +those counters become the next measured hot row, partition their writes and +sum a bounded set of partitions. Do not synchronously increment a single new +company total on every receipt and call the bottleneck solved. + +### 4. Qualify workload and roll out + +Extend the existing benchmark rather than using idle-agent count as scale proof. +Use real PostgreSQL with at least two independent controller processes and +explicitly recorded connection-pool limits. Simulated provider events should +exercise real receipt ingestion and settlement, followed by a smaller real +provider qualification. Keep load tests off the active staging instance. + +| Dimension | Required cases | +| --- | --- | +| Distribution | 100, 1,000, then 10,000 active runs; one company and many companies | +| Receipt cadence | One update per run per 60 seconds, 10 seconds and 1 second; record actual target-provider cadence before choosing the release load | +| Policies | None; company; agent/project only; overlapping scopes and windows | +| History | Increasing ledger history independently of active-run count | +| Event shape | Cost-changing, unchanged, subscription, duplicate, stale, conflicting and out-of-order reports; reconnect bursts | +| Races/failures | Simultaneous settlement and receipt, worker crash, failed/ambiguous commit, source replacement, policy change, manual Stop, late final charge, unknown price and UTC rollover | + +At 10,000 runs the illustrative receipt cadences imply about 167, 1,000, and +10,000 receipts/second before admission or settlement. This defines a workload, +not a claim that current or proposed code supports it. A shared critical section +held for an average of H seconds has an ideal ceiling near 1/H operations/second +for that company; do not estimate H from the unrelated percentile summaries. + +Acceptance requires durable receipt/ledger equality after quiescence, no duplicate +effect or acknowledgement loss, stable pending-work depth, correct policy/manual +hold behavior, and measured detection/cancellation/freshness targets. Record +p50/p95/p99 ingestion, admission and settlement latency, actual PostgreSQL wait +events, query count/rows/bytes, CPU/event-loop delay, and reconnect recovery time. +Prove unrelated companies remain responsive and writers for unrelated runs no +longer contend on a shared company update in the frequent path. + +Initially compute the proposed evaluation in shadow mode without stop/resume +effects and compare it against the authoritative ledger and existing decisions. +Then canary the new enforcement contract on an explicitly selected environment, +with its versioned pending work rebuilt and validated before switching. Do not +run old and new enforcement authorities simultaneously for the same company. +Rollback must drain/reconcile accepted receipts and rebuild authoritative state +before restoring the older path; flipping a flag alone is insufficient. + +Relevant existing coverage includes scoped budget enforcement, live attempts, +native usage delivery, run-cost accounting, budget cancellation, worker recovery, +and idle-accounting work. Add focused concurrency/failure tests for the changed +boundaries; run the repository's full checks before a PR-ready implementation +handoff. Browser qualification is needed if dispatch/Stop or budget UI semantics +change, not to validate this plan file. + +## Contract and delivery boundaries + +Before enabling the new behavior, update `doc/SPEC.md`, +`doc/SPEC-implementation.md` (especially scheduler and budget enforcement), the +applicable contributor invariants, shared budget contracts, and user-facing +budget explanations. The UI should say what auto-stop does and disclose possible +in-flight overspend; it should not expose internal partition mechanics. Decide +how configured existing reservations retain their estimate/visibility role +without promising atomic aggregate admission in the new mode. + +Distributed bounded spending allocations are deferred. Revisit them only if +customers need stricter aggregate admission or qualified batch evaluation cannot +meet latency/overspend needs. They require provider cost bounds as well as +replenishment, revocation and policy-version coordination to become a monetary +guarantee. They are not required for the recommended first implementation. + +Investigation validation: inspected the source and existing test/benchmark +contracts, queried live spans and two individual traces, and checked the relevant +deployed-revision source diff. No service, configuration, database, or deployed +code was changed. Runtime tests and load tests were not run for this planning +change; dependency installation and implementation belong to the next stage. diff --git a/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md new file mode 100644 index 00000000000..9bf24ecb59c --- /dev/null +++ b/doc/plans/2026-11-03-existing-workspace-ephemeral-migration.md @@ -0,0 +1,47 @@ +# Preserve existing workspaces during ephemeral enrollment + +The deployment endpoint remains a volume-free Cloud tenant, followed by a real +rolling deployment while an agent works and new work is admitted. Enrollment must +preserve existing task files and historical recovery sources. It must not reset +task workspaces, manufacture successful copyback, or weaken migration guards. + +## Evidence and remaining work + +| Required outcome | Current evidence | Remaining gap | Owner / next action | +| --- | --- | --- | --- | +| Existing active task retains files | The current import encrypts, uploads, independently restores, compares the source again and publishes under the lifecycle lock. A live completed task imported successfully with its original directory retained. | The importer only accepts an active workspace with the current task binding. | Workspace migration owner: support historical records without changing the task binding or lifecycle state. | +| Archived task retains its saved files | Retention records include encrypted task-directory restore points, Git restore points and verified Git reconstruction recipes. | The importer rejects missing local folders even when an existing restore point can reconstruct them. | Restore-point owners: reconstruct into migration scratch space, reject expired/missing/corrupt sources, and feed the verified result into the normal import publication. | +| Shared project roots preserve all task files | Legacy roots contain nested task worktrees and multiple records can reference one physical folder. | A naive root capture recursively imports those other workspaces; the current per-record writer check does not fence aliases. | Workspace migration owner: inventory physical overlaps, fence all writers, and preserve nested workspace coverage before excluding any directory. | +| Failed runs retain unsaved files | Provider inspection distinguishes surviving archived sandboxes from missing allocations. Existing recovery validation accepts some sources and rejects others for missing provenance or ambiguous allocation ownership. | Recovery must settle or durably preserve each exact source. A successful old preservation action does not necessarily update the legacy sync descriptor. | Workspace recovery owner: verify existing preservation receipts, recover eligible sources, and retain explicit unresolved obligations for unavailable/ambiguous sources. Do not replay agent work. | +| Tenant is eligible without its volume | External DB and S3 are configured; the original volume remains attached. | Workspaces, local run logs and remaining owner-maintained data must pass the packaged maintenance receipt. | Cloud enrollment owner, after workspace migration is complete. | +| Ordinary upgrades remain available | The no-drain Cloud deployment path is merged and qualified on another enrolled tenant. | Exact final app image qualification and a live deployment on this tenant. | Cloud deployment owner: qualify, enroll, then prove reads, submissions and active-run adoption through an actual replacement. | + +## Acceptance boundaries + +The first implementation slice supports archived and historical records, +encrypted task-directory restoration, Git restoration recipes, and legacy +branch-deletion receipts. PostgreSQL and real-file tests prove cold restoration +of saved bytes, index state, and unpushed commits, including after pruning the +old Git object store. Invalid or unavailable retained data still blocks import. +The next slice adds leaf-first offline migration of overlapping folders, verified +nested snapshot dependencies, interruption recovery, and dependency retention. +Recovery-obligation reconciliation remains a separate gap; these slices do not +yet make the whole tenant eligible for enrollment. + +- Import existing files or a verified retained source. A missing folder alone is + not permission to publish an empty workspace or choose a newer Git branch. +- Preserve archived status, task bindings, task history, unpushed Git history, + staged and unstaged edits, and saved recovery copies. +- Preserve original files and the detached original volume until verified + migration receipts and the retention interval permit removal. +- Parent roots may omit a nested workspace only with durable, inspectable + coverage of that exact nested directory. Arbitrary exclusion globs are not an + acceptable data migration. +- Resume interrupted imports with their original identity; cancellation may + release an empty reservation only after the producer is stopped and no upload + was journaled. Preserve an audit receipt. +- Verify restored file bytes independently, including after making the old + source unavailable in an isolated test. Exercise current and historical + bindings, archived sources, overlap/writer races, and unavailable sources. +- Unrelated tenants, production services, and agent task execution remain outside + the migration's mutation scope. diff --git a/doc/plans/2026-11-03-lock-free-accounting-design.md b/doc/plans/2026-11-03-lock-free-accounting-design.md new file mode 100644 index 00000000000..4c0ed915adc --- /dev/null +++ b/doc/plans/2026-11-03-lock-free-accounting-design.md @@ -0,0 +1,432 @@ +# Lock-free accounting and best-effort budgets + +Date: 2026-11-03 +Status: Target architecture proposed; preliminary admission optimization implemented locally +Scope: Accounting ingestion, settlement, budget projections and budget admission +Evidence: [Accounting contention investigation](2026-11-03-accounting-contention-and-budget-enforcement.md) + +## Decision + +Make the frequent accounting write path append-only. A receipt writes its own +immutable fact and commits. It does not lock or update the company, agent, run, +budget, aggregate counter, or worker checkpoint. Independent runs use independent +keys, including when they belong to the same company or agent. + +Asynchronous workers derive operation state, ledger entries and partitioned spend +totals. Budget admission reads a recent view without making a synchronized debit +against a company balance. Threshold detection and cancellation may lag; durable +evidence, deterministic accounting and execution authorization remain required. + +"Lock-free" here means no application mutex, advisory lock, or `SELECT FOR +UPDATE` in the receipt or budget-check path, and no shared accounting row written +for each receipt. It is not a formal nonblocking progress guarantee: PostgreSQL +still uses internal locks, unique-index arbitration, foreign-key checks and +transaction commit. In particular, two inserts with the same unique key can +wait for each other. See [PostgreSQL INSERT](https://www.postgresql.org/docs/current/sql-insert.html). + +Background batches and infrequent policy changes use optimistic conditional +writes. Those also take database row locks internally. Moving a compare-and-swap +onto the same company counter for every receipt would fail this design. + +Existing issue ownership, provider-dispatch fencing, manual Stop and resource +ownership are outside this lock-removal scope. Accounting cannot replace those +contracts with an eventually consistent spend view. + +## Problem fit and first milestone + +This design removes the identified company-wide accounting serialization point +if receipt ingestion, admission, settlement and shared counters all migrate. +It does not establish that accounting is the current main-loop CPU culprit or +that the resulting system supports 10,000 active agents. Those remain separate +claims requiring measurements. + +The matched investigation sample contained 8,280 zero-policy exposure evaluations +out of 8,292. Those exposure evaluations returned almost immediately; the +surrounding admission path still read state and policies inside the accounting +transaction. This makes separating eligibility reads from reconciliation the +most directly supported first change. It does not show that large exposure +scans or a high receipt rate caused the sampled delays. + +The 9,421 accounting transactions in that hour average about 2.6 per second +across the observed host. Burst rate, company distribution and actual lock hold +time are missing. The acquisition spans include database/network time and Node +scheduling, so their long tails alone do not prove PostgreSQL row-lock saturation. + +| Problem | What this design changes | Remaining condition | +| --- | --- | --- | +| Eligibility checks enter a mutating company transaction | Read-only admission, separate reconciliation | Classify callers and preserve manual holds, resume behavior and execution fencing | +| Independent receipts serialize by company | Independent immutable inserts | Qualify actual receipt rate, index/FK waits and durable ingestion capacity | +| Repeated exposure work and shared counters limit growth | Coalesced evaluation and partitioned projections | Show less work per receipt and sufficient projection throughput | +| Main-loop CPU/event-loop contention | Some synchronous work leaves the request path | Same-process reducers still consume CPU; batching must reduce work or processing must move to another runtime | + +The immutable identity model, closure changes and partition publication protocol +are target-architecture work, not prerequisites for the first latency improvement. +An asynchronous pipeline also adds backlog and recovery obligations: its stale +decision gate can become an admission outage if reducers cannot keep up. Merely +moving identical work behind an ACK would conceal latency without solving load. + +Start with an independently shippable milestone: attribute admission calls and +transaction work, provide a pure eligibility read (including a no-policy fast +path), and move required reconciliation to its explicit owner. Retain the current +final admission/reservation authority during this initial behavior-preserving +step. Having no configured policy must not bypass manual company/project pauses, +existing holds, unknown-accounting restrictions or duplicate-dispatch protection. + +Compare end-to-end dispatch latency, accounting transactions and queries per +dispatch, database waits, CPU and event-loop delay under the same workload. +Then qualify receipt coalescing and the remaining single-company bottleneck. +Use the full migration below when that evidence or the agreed capacity target +justifies it. The accounting rewrite does not cover the other independently +identified work in run-event persistence, recovery, snapshots or fanout. + +## Architecture + +```mermaid +flowchart LR + P["Runner / paid service"] --> I["Validate and append receipt"] + I --> J[("Immutable receipt journal")] + J --> A["ACK after durable commit"] + J --> R["Independent partition reducers"] + R --> L[("Ledger + operation views")] + R --> T[("Spend totals by scope / window / partition")] + T --> B["Coalesced budget evaluator"] + B --> V[("Versioned budget decisions")] + V --> D["Dispatch: read decision, record own admission"] + B --> S[("Durable stop intent")] + S --> C["Existing scoped cancellation path"] +``` + +The journal, reducers and evaluator can start in the existing server and +PostgreSQL deployment. Independent scaling is a module/runtime boundary, not a +requirement for Kafka, Redis, another service, or a separate database today. + +## 1. Give every paid operation a stable identity + +Use one accounting operation for each heartbeat run, browser invocation, +decision-model invocation, fast response, classifier invocation, or separately +reported charge. Record its company and attribution before paid dispatch. +Retries of the same external operation reuse its identity; a genuinely new paid +attempt has a distinct attempt identity beneath that operation. + +An accounting operation's immutable identity is separate from mutable run state. +Receipts reference this identity, rather than acquiring foreign-key protection +on a busy mutable `heartbeat_runs` row. The existing journal directly references +that row, so removing the explicit lock alone does not eliminate this possible +wait. Use a company-scoped composite reference to the immutable identity; keep +company-boundary validation and the existing deletion/retention obligations. +Separate any nullable live-run pointer from retained original attribution. + +Admitted accounting sources have durable identities and provenance. Controller +reconnection does not manufacture a new paid attempt. Source replacement must +retain the existing verified handoff/baseline semantics. A higher timestamp, +random UUID, or newly generated controller source ID is not proof of authority +over earlier billing evidence. + +Source handoff remains an infrequent execution-authority operation. Ordinary +checkpoints do not mutate its ownership record. Existing authentication and +company access checks still apply. Authority to report a closing receipt after +Stop is accounting-only; it never grants authority to continue execution. + +## 2. Receipt ingestion only appends evidence + +Extend/generalize the existing `run_usage_receipts` contract. The conceptual +receipt contains: + +| Field | Purpose | +| --- | --- | +| Company and operation IDs | Tenant boundary and accounting unit | +| Attempt and admitted source identity | Separate paid attempts from reconnect/replay | +| Stable source event ID and source sequence | Replay identity and ordering within that source | +| Receipt kind and normalization version | Partial usage, final usage, source handoff, closure, correction | +| Cumulative usage, price and provenance | Evidence; unknown price stays unknown | +| Payload fingerprint | Detect conflicting reuse of an identity | +| Server-assigned ingestion time and deterministic partition | Recovery, pending-work age and routing | + +Provider event time remains separate. Freshness protection never trusts a caller's +timestamp. External charge callers need a stable request identity to obtain +idempotent retries; matching dollar amounts or payload similarity cannot prove +that two otherwise unidentified charges are the same purchase. + +Within a small transaction, validate the admitted identity and input, register +the existing transaction-aware work signal, and insert the immutable receipt. +Enforce the logical identity with a unique constraint. On conflict, fetch the +committed existing receipt and compare its fingerprint; equal means replay, +different means a conflict requiring reconciliation. Do not use `DO UPDATE` +to replace receipt evidence. At read committed isolation, a conflict check may +need a subsequent statement to see the winning committed row. + +Commit before ACK. An ACK means the evidence is durable, not that the spend view +has already incorporated it. Preserve legacy durable spooling and native +redelivery when the database write or its acknowledgement is uncertain. + +There is no read-modify-write of `usageJson`, `spentMonthlyCents`, reservation +capacity, or a company-wide dirty flag here. The new receipt is itself durable +pending work. Mutable delivery metadata may be attached to it by the consumer; +its evidence fields are immutable. + +Batch inserts with count/byte bounds when useful. Keep lifecycle, final, unknown +and conflicting evidence intact. Do not drop a receipt solely because its dollar +amount matches the previous one: it may change completeness or provenance. + +## 3. Reduce independent partitions, then publish batches atomically + +Route by a stable hash of **operation ID**, not company ID. For example, a +starting benchmark can use 64 virtual partitions per company. That number is a +test parameter, not a capacity claim. One company with 10,000 active runs spreads +over all partitions; one agent with many runs spreads too. + +Persist the layout version and partition with the operation's admission. All its +receipts, closure and corrections follow that assignment. Changing partition +count requires an explicit migration; do not silently rehash active operations. + +The minimal conceptual storage is: + +| Record | Write frequency and ownership | +| --- | --- | +| Operation identity | Once per admitted operation; immutable attribution | +| Receipt journal | Append per receipt; unique identity; durable pending metadata | +| Operation view | Replaced by its partition reducer, at most once per operation per batch | +| Scope/window/partition totals | Updated in a bounded batch, never by receipt ingestion | +| Partition publication head | One conditional generation advance per batch | +| Budget policy and decision | Policy edits and coalesced evaluations; independent of receipt acceptance | +| Ledger, adjustments, incidents, stop intent | Existing domain records, adapted to operation identities and idempotent writes | + +Some records can extend existing tables. Separate immutable identities from +mutable projections; do not add a table for each timer or delivery stage. + +For a reducer batch: + +1. In one repeatable-read snapshot, capture partition generation `g`, a bounded + set of unprocessed receipt IDs, and their prior operation views. +2. Reduce those operations and compute replacement contributions outside the + publishing transaction. No provider calls or filesystem work occurs during + publication. +3. In one short transaction, conditionally advance the partition head from `g` + to `g + 1`, bulk-write operation views and aggregate deltas, insert any + qualified ledger settlement, and mark exactly the consumed receipt IDs. +4. If the conditional advance loses, roll back and retry from a fresh snapshot. + Use bounded retries with scheduler backoff under contention. + +All writers of those views, including repairs, follow this publication protocol. +A normally assigned reducer avoids duplicate work; overlapping workers or an +old worker after failover remain safe because publication requires the captured +generation. No durable exclusive lease is needed for correctness of this batch. + +An input arriving during computation remains pending. Never advance a global +`last_seen_id` or timestamp past all smaller values: sequence allocation and +transaction commit order differ. An earlier allocated receipt that commits +later must still be discoverable. Retain an indexed pending predicate. + +This has bounded coordination per partition **per batch**. It cannot stall +receipt ACKs behind the calculation, and different partitions do not write the +same aggregate rows. Shared company/agent/project totals are sums of these +partition contributions, not counters incremented by every producer. + +## 4. Define canonical money without depending on arrival order + +The reducer preserves current normalization and source-ownership rules as a +versioned pure function over admitted evidence. Within a source, order by the +producer sequence, not database arrival time. Across sources, follow verified +handoff/continuation provenance rather than arbitrarily choosing a winner. + +Cumulative reports replace earlier reports for the same accounting component. +Reports of $2, $3 and $5 represent $5, not $10. Distinct paid attempts contribute +separately; session-cumulative counters require their established baseline. +An incomplete/unknown report cannot erase known partial exposure. Final prices +and audited corrections may legitimately reduce earlier estimates. + +Each operation view owns mutually exclusive contribution categories: + +```text +unsettled: ledger = 0; live = known observed amount; hold = max(estimate, live) +settled: ledger = final; live = 0; hold = 0 +``` + +This is a contribution model, not a declaration that unknown exposure is zero. +Carry unknown-price, pending-terminal and unresolved-dispatch counts separately +and apply their configured restrictions. Hosted browser work remains a separate +paid operation from its parent run. + +Settlement and that operation's replacement contribution publish atomically. +Company/agent/project views sum one contribution per operation. A reader sees +the old or new contribution, never both the live estimate and its final charge. +UTC-month and lifetime attribution, unsettled carryover, credits, backdated +charges and corrections need explicit reducer rules matching the selected +product contract; do not assign costs to the month in which a worker ran. + +### Finalization requires closure, not merely a "complete" checkpoint + +The governing execution coordinator supplies immutable closure evidence only +after the operation cannot start another ordinary paid attempt. It references +the admitted attempts/sources and the final usage evidence or unresolved debt. +An adapter's `complete` usage flag alone does not establish that boundary. + +A closure record must prove the final cumulative coverage it names. The current +native `sourceSeq` includes non-accounting events, so gaps between usage receipts +are not automatically lost usage. Use explicit final references/coverage, or a +dedicated accounting sequence if the producer contract is extended. Do not +invent a contiguous usage sequence from today's sparse source sequence. + +Publish a settlement certificate and its ledger components in one transaction, +with one base settlement per `(company_id, operation_id)` and a verified content +fingerprint. A different closure for an already-settled operation is a conflict +or an explicit correction, not a second base settlement. +Only the transaction that inserts that certificate inserts its components; a +replay verifies the existing certificate. Ledger IDs and component keys are +stable across retries. Unknown external outcomes remain unresolved. + +Late evidence within the declared closure coverage is replay/recovery. Evidence +beyond it remains durable and creates a reconciliation obligation or an explicit +correction; it cannot silently reopen the same final charge or disappear because +the run was marked terminal. Corrections use immutable adjustment identities. +Existing mutable correction/reporting APIs need an adapter to this ledger model. + +## 5. Budget checks read; they do not reserve a shared balance + +For dispatch, read current company/agent/project controls and applicable budget +decisions in one bounded read from the authoritative database, rather than an +unbounded-lag replica or cache. A policy without a current decision must +be evaluated or treated as pending; it is not implicitly unlimited. The decision +records policy revision, budget window, projection generations, status and +freshness evidence. + +If applicable decisions allow work, record this operation's own admission with +the decision versions and estimated cost. Preserve duplicate-dispatch fencing +in the execution subsystem. An admission receipt is not proof that an external +provider call occurred exactly once; existing unknown-outcome recovery remains. + +There is no atomic decrement of company remaining budget. Estimate-fit checks +can use the same snapshot, and admitted estimates can enter the asynchronous +exposure view. Two dispatches can see the same remaining money and both start. +That race is accepted by this best-effort contract. Reservations become recorded +estimates/obligations, not exclusive claims on a shared spending allowance. + +New snapshots after a committed budget block deny additional work. Admissions +already in flight can still race that block. Manual Stop, pause, permissions, +issue ownership and dispatch authority retain their existing final gates. +Keep provider-enforced cost caps where supported. + +## 6. Coalesce budget decisions and preserve stop authority + +The evaluator reads the applicable scope/window partition totals in one +consistent snapshot, rather than scanning ledger and receipt history on each +checkpoint. Capture the policy's configuration revision and prior decision +revision in that same snapshot. + +Publish with a conditional update against both captured revisions. A policy +edit advances its configuration revision and invalidates the previous decision. +An older evaluator cannot overwrite a newer decision or a changed policy. A +losing evaluator recomputes; it never retries publication using a fresh version +number attached to stale calculated totals. + +Publish the decision, any incident/approval transition, and durable stop intent +in the same transaction. Generate idempotency identities from the winning +transition; losing or repeated evaluations produce no duplicate side effects. +Only actual state transitions need notification/audit effects. Cancellation +delivery remains at least once, scoped and recoverable. Already-committed +per-run Stop obligations survive later policy changes. Resuming a budget hold +cannot clear a manual hold. + +These occasional policy-row updates can contend with one another. Producers +and admission readers do not wait on that row to record receipts or debit money. +Do not replace them with per-receipt policy updates. + +## 7. Bound lag and recover work without an idle polling loop + +Retain the proposed healthy-load targets from the investigation: roughly +one-second coalescing and a five-second p99 target from durable priced receipt +to stop intent. Provider reporting and shutdown delay are additional. Targets +require a workload qualification; none is a guarantee on final billed dollars. + +For scopes with spending limits, reject additional paid admissions if an +unrepresented accounting change has been waiting over fifteen seconds. This +check considers pending journal inputs **and** reduced contributions not yet +covered by the decision's projection generations. Carry their oldest pending +time through publication. Consuming the journal must not hide evaluator lag. +An old decision with no subsequent relevant changes remains fresh while idle. +Policy edits and UTC-window changes are explicit new evaluation obligations. + +Durable pending rows are truth; notifications are hints. Reuse the existing +delivery coordinator for local coalescing, finite retries, actual deadlines, +drain and restart. No unconditional per-company timer scans empty queues. + +The current DB work signals are process-local by default, with optional +commit-bound PostgreSQL notifications. Separate reducers therefore require +qualified cross-process wake delivery, subscribed before the startup scan and +reconciled after reconnect. `NOTIFY` alone is not durable storage or a way to +wake stopped compute. See the [existing signal implementation](../../packages/db/src/work-signals.ts) +and [PostgreSQL notification behavior](https://www.postgresql.org/docs/current/sql-notify.html). +Do not claim distributed recovery or host sleep until the actual notification, +startup, disconnect and external-wake paths are qualified. + +Receipt acceptance continues while budgets are blocked. Otherwise stopping spend +would also stop the evidence needed to settle it. + +## Failure cases the design must pass + +| Event/race | Required outcome | +| --- | --- | +| Same receipt delivered twice | One logical receipt; equal fingerprint accepted as replay | +| Same identity, different content | Conflict retained/reported; no silent overwrite or second charge | +| Sequence 8 arrives before 7 | Correct cumulative state; 7 cannot rewind it | +| Earlier allocated receipt commits after a batch | It remains discoverable as pending work | +| Producer loses ACK after commit | Safe redelivery with the same identity | +| Reducer dies before or during publication | All batch effects commit together or remain replayable | +| Two reducers compute from generation 12 | One publishes generation 13; the other recomputes | +| Final receipt races partial receipt | Closure and canonical reduction determine one contribution, independent of arrival order | +| Final bill is lower than live estimate | Correct signed adjustment/replacement; no double counting | +| Source changes during recovery | Verified handoff retains prior debt; no guessed new baseline | +| Policy is lowered during evaluation | Old policy revision cannot publish a decision | +| Manual Stop races budget resume | Manual execution hold remains authoritative | +| Evaluator stops while receipts continue | Evidence accumulates durably; stale-view admission protection activates | +| No new accounting work | Reducers and evaluator become quiet | + +## Implementation and migration + +These steps implement the full target after the first milestone above; they are +not required as one change to improve today's eligibility path. + +1. Add immutable operation identities and the journal acceptance boundary, + initially alongside the existing authoritative path. Preserve existing native + and legacy receipts; adapt source/closure evidence before removing their + current completion fences. +2. Implement deterministic operation reduction, idempotent settlement and + partition publication. Run it in shadow mode with isolated projections and + no production ledger, stop, or incident side effects. Compare at a common + receipt/closure boundary after draining, not at arbitrary wall times. +3. Implement versioned budget decisions, freshness, read-only admission and + transition-based stop delivery. Replace shared monthly/runtime counters with + partitioned projections and update all report, repair and integrity consumers. +4. At a controlled cutover, fence the old settlement/evaluation authority, drain + committed inputs, seed verified ledger/adjustment baselines, and select one + authoritative writer path. Resume admission only with current decisions. + Continue accepting/replaying durable receipts through this transition. +5. Rollback similarly requires reconciliation and an authority handoff. Do not + let old and new settlement writers charge the same operation independently. + +Owning modules are usage receipts/native usage delivery, run-cost accounting, +costs/reconciliation/integrity, budgets and the existing enforcement worker. +Dispatch consumes a narrow decision contract; provider normalization remains +with adapters. Update DB/shared/API/UI contracts and the budget guarantees in +`doc/SPEC.md`, `doc/SPEC-implementation.md` and contributor guidance before enabling +the change. The earlier per-run-lock design is not this target's hot path. + +Qualification uses real PostgreSQL and multiple independent processes, both one +large company and many companies, overlapping scope budgets, ledger-history +growth, duplicate/reconnect bursts and the races above. Verify producer SQL has +no shared counter/head/policy writes and no company/run `FOR UPDATE`. Audit +foreign-key, trigger and uniqueness behavior too; string searches cannot prove +absence of contention. + +Measure sustained receipts and bytes per second, ACK/admission/settlement latency, +projection backlog age, stop latency, database waits and event-loop load. Ten +thousand agents at one receipt per second would create 864 million receipts per +day before batching/retention effects. Eliminating serialization does not remove +database, WAL, index, storage or provider-capacity limits. Start with bounded +batches and measured retention; introduce a different log substrate only when +its need is demonstrated. + +Design validation in this chat: inspected the existing journal, budget incident, +enforcement and work-signal contracts and checked PostgreSQL concurrency and +notification semantics. This is an architecture document, not a prototype or +load-test result. Implementation details and capacity remain to be qualified. diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index 65b00378d69..c8a2c5037b2 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -19,6 +19,14 @@ same Runner and provider history through another healthy follow-up. This is a controlled cache fault, not a reconstruction of a historical incident. Each recovery case includes another healthy turn after the recovered turn and asserts that it does not start another runner or provider process. +The `warm-suspended` case suspends the first turn, resumes the second turn, +then keeps that controller warm for the third turn. It must retire the first +turn's local suspension receipt so the third turn can reuse the resumed Runner +and provider history without a second process or repeated turn. +The `finalizer-wait` case gives the completed predecessor a live finalization +lease. The next run must wait without starting a provider turn or consuming an +attempt. After lease release, one durable resumption claim continues that same +run and preserves provider history through the third turn. The same-server case also starts preparation lease renewal during recovery and holds instruction collection open past a renewal tick after coordinator release. The executor must acquire renewal before preparation releases its reference, then diff --git a/maps/agent-access-map/agents.md b/maps/agent-access-map/agents.md index 75ef16b9af0..c9d591c1b71 100644 --- a/maps/agent-access-map/agents.md +++ b/maps/agent-access-map/agents.md @@ -19,6 +19,25 @@ approval activation, built-in provisioning and import preservation described below. The original baseline commit above still governs unrelated entries; this update does not claim operation-complete review or live agent verification. +Native Cursor Cloud uses a server-derived `provider_api` placement after its +complete native profile is validated. Create/update with `runner: paperclip` +stores `defaultEnvironmentId: null`; the caller cannot supply placement, commands, +working directories or arbitrary environment values. Only the bound +`CURSOR_API_KEY` is passed to the qualified API proxy. Test does not acquire a +Computer lease. Finish setup uses the same validated placement and account check; +it does not inherit a default Computer or its warm-task lifecycle. Incomplete +held drafts stay inactive until the complete profile passes activation validation. +Task/run authority, company boundaries, budgets and per-turn callback scope remain enforced. Local Computer execution stays denied under +managed-sandbox-only policy. Verify the actual saved agent and live run; a board +Test result does not prove agent task-tool access. + +On managed-only instances with native Runner enabled, a server-held Cursor setup +draft can resolve an automatic runner to the API controller before it has a +model. Unsaved automatic Test requests use the same advertised default and still +validate the full profile. Existing active Legacy agents and explicit Legacy +choices do not convert automatically. A client-supplied setup flag does not grant +draft status or bypass active model validation. + ## `hire` Capability ID: `agents/hire`. diff --git a/maps/agent-access-map/instance-operations.md b/maps/agent-access-map/instance-operations.md index db0c87834f0..cebe15a8df6 100644 --- a/maps/agent-access-map/instance-operations.md +++ b/maps/agent-access-map/instance-operations.md @@ -168,6 +168,14 @@ API: `present` — representative operation IDs: status remains an in-memory read. [Route coverage](../../server/src/__tests__/instance-settings-routes.test.ts) and [PostgreSQL coverage](../../server/src/services/heartbeat/upgrade-readiness.test.ts) establish this local boundary; hosted active-run continuity requires separate proof. +- `GET /api/instance/task-drain?idleSleepSafety=1` — the negotiated incoming-wake + extension requires a verified stack/action/owner-scoped Cloud control capability; + query parameters and tenant credentials cannot grant it. Its version-3 proof + permits only saved API keys and exactly bound brokered GitHub bot configuration. + Agent and board API requests authenticate under current tenant authority after + bounded host wake. Personal GitHub continuity remains blocked until the stacked + deadline proof is present. See [protocol scope](../../doc/idle-incoming-wake.md) + and [local auth/PostgreSQL evidence](../../server/src/__tests__/idle-incoming-wake.test.ts). - `GET /api/ready` and `HEAD /api/ready` — public deployment readiness only; 200 after recovery, 503 while starting, recovering, or draining. No SQL, session resolution, company identity, or control authority. diff --git a/maps/feature-map/agents.md b/maps/feature-map/agents.md index 206f4544abb..1fac593afb4 100644 --- a/maps/feature-map/agents.md +++ b/maps/feature-map/agents.md @@ -37,6 +37,22 @@ Save & exit keeps its configuration; Finish setup on the agent returns to the same record. Final submission requests approval when required, then follows the existing preparation and verification lifecycle. +For **Cursor Cloud**, choose its card in Cloud when native Runner is enabled and +the qualified controller is available. Its held setup draft uses the API +controller without a Computer, before model or credential selection. Local +operators can choose **Paperclip Runner** in Advanced. Bind a Cursor Cloud +API key, enter the exact model and select a repository/ref or a named Cursor +cloud environment. Cursor hosts the agent's environment; this native setup has +no Paperclip Computer selection. Save, reopen, and run Test. Test checks the +account catalog and callback authentication gate without a prompt; the effective-model +warning remains visible for this experimental integration. Finish setup repeats +that account check and reaches Ready without preparing or leasing a Computer, +even when the instance has a default plugin Computer. The Cursor lifecycle stays +per-turn. Assign a real task +and verify its output, question continuation, Plan approval and Stop separately. +Managed-sandbox-only policy stays enabled. It continues to deny local agent code. +Kubernetes-only execution is not supported by this API controller. + When the native Runner adapter is available, **Google Antigravity** appears in the ordinary adapter picker. Before runtime acceptance, its card says **Not available yet**. Selecting it opens Google account setup and model configuration without saving diff --git a/maps/feature-map/instance-operations.md b/maps/feature-map/instance-operations.md index 29387666067..6ff9a9b0633 100644 --- a/maps/feature-map/instance-operations.md +++ b/maps/feature-map/instance-operations.md @@ -154,3 +154,15 @@ Scope: cli Storybook: No authored story required for doctor/health command results. Production: Run doctor against an owned data directory; check readiness and nonzero failures. A listening port alone does not prove readiness; repair requires observing the repaired result. Sources: Source: [doctor.ts](../../cli/src/commands/doctor.ts). Tests: [doctor.test.ts](../../cli/src/__tests__/doctor.test.ts) + + +Incoming API wake is an optional Cloud deployment capability. On an owned qualified +stack, close the browser, retain valid saved credentials, let the stack sleep, +and send an ordinary API request. Check the persisted result and current +revocation/company authorization after wake, truthful retry failures, and return +to idle after successful API activity ends. Brokered GitHub bot wake uses the +existing durable connector queue and exact registration ownership. Direct +webhooks, persistent transports and personal GitHub credential continuity retain +their explicit dependencies. [Source protocol](../../doc/idle-incoming-wake.md) +and [local fixtures](../../server/src/__tests__/idle-incoming-wake.test.ts) cover +these boundaries; they do not establish deployed host/database suspension. diff --git a/server/src/__tests__/activity-service.test.ts b/server/src/__tests__/activity-service.test.ts index cf613e0b59e..d2ed798d5c7 100644 --- a/server/src/__tests__/activity-service.test.ts +++ b/server/src/__tests__/activity-service.test.ts @@ -325,6 +325,46 @@ describeEmbeddedPostgres("activity service", () => { expect(runs[0]).not.toHaveProperty("contextSnapshot"); }); + it.each(["requested", "acknowledged", "pending_native", "missing_native", "wrong_run", "wrong_company", + "malformed_execution", "null_execution", "malformed_native"] as const)( + "projects only bounded, owner-matched Stop evidence in issue history: %s", async proof => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Stop history", issuePrefix: `S${companyId.slice(0, 6)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Stopped runner", adapterType: "process" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Stop the current run", assigneeAgentId: agentId }); + const native = { schema: "paperclip.native-cancellation.v1", runId, companyId, + dispatchState: proof === "pending_native" ? "pending" : "acknowledged", + intentAuditId: "private-stop-proof", effects: ["private-stop-proof"], secret: "private-stop-proof" }; + const result: Record = { + executionCancellation: { state: proof === "requested" ? "requested" : "acknowledged", + proof: "private-stop-proof", secret: "private-stop-proof" }, + nativeCancellation: native, unrelated: "private-stop-proof", + }; + if (proof === "missing_native") delete result.nativeCancellation; + if (proof === "wrong_run") native.runId = randomUUID(); + if (proof === "wrong_company") native.companyId = randomUUID(); + if (proof === "malformed_native") result.nativeCancellation = { ...native, dispatchState: { secret: "private-stop-proof" } }; + if (proof === "malformed_execution") result.executionCancellation = { state: { secret: "private-stop-proof" } }; + if (proof === "null_execution") result.executionCancellation = null; + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, scopeKind: "issue", + nativeIssueId: issueId, runtimeMode: "native", status: "cancelled", finishedAt: new Date(), + contextSnapshot: { issueId }, livenessState: "advanced", resultJson: result }); + + const rows = await activityService(db).runsForIssue(companyId, issueId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ runId, runtimeMode: "native", status: "cancelled" }); + expect(rows[0]!.resultJson).toEqual({ + executionCancellation: { state: proof === "requested" ? "requested" + : proof === "malformed_execution" || proof === "null_execution" ? "unknown" : "acknowledged" }, + ...(["missing_native", "wrong_run", "wrong_company", "malformed_native"].includes(proof) ? {} : { + nativeCancellation: { schema: "paperclip.native-cancellation.v1", runId, companyId, + dispatchState: proof === "pending_native" ? "pending" : "acknowledged" }, + }), + }); + expect(JSON.stringify(rows)).not.toContain("private-stop-proof"); + }, + ); + it("backfills missing liveness for completed issue runs before returning the ledger", async () => { const companyId = randomUUID(); const agentId = randomUUID(); diff --git a/server/src/__tests__/adapter-model-refresh-routes.test.ts b/server/src/__tests__/adapter-model-refresh-routes.test.ts index 6d94b9b6ed9..f65f12b52f1 100644 --- a/server/src/__tests__/adapter-model-refresh-routes.test.ts +++ b/server/src/__tests__/adapter-model-refresh-routes.test.ts @@ -101,13 +101,15 @@ function registerModuleMocks() { })); } +registerModuleMocks(); +const [{ agentRoutes }, { errorHandler }] = await Promise.all([ + vi.importActual("../routes/agents.js"), + vi.importActual("../middleware/index.js"), +]); + const refreshableAdapterType = "refreshable_adapter_route_test"; async function createApp() { - const [{ agentRoutes }, { errorHandler }] = await Promise.all([ - vi.importActual("../routes/agents.js"), - vi.importActual("../middleware/index.js"), - ]); const app = express(); app.use(express.json()); app.use((req, _res, next) => { @@ -159,11 +161,6 @@ async function unregisterTestAdapter(type: string) { describe("adapter model refresh route", () => { beforeEach(async () => { - vi.resetModules(); - vi.doUnmock("../routes/agents.js"); - vi.doUnmock("../routes/authz.js"); - vi.doUnmock("../middleware/index.js"); - registerModuleMocks(); vi.clearAllMocks(); mockCompanySkillService.listRuntimeSkillEntries.mockResolvedValue([]); mockCompanySkillService.resolveRequestedSkillKeys.mockResolvedValue([]); diff --git a/server/src/__tests__/adapter-routes.test.ts b/server/src/__tests__/adapter-routes.test.ts index 7a29bc1d1a0..75dccc36eac 100644 --- a/server/src/__tests__/adapter-routes.test.ts +++ b/server/src/__tests__/adapter-routes.test.ts @@ -1,4 +1,5 @@ import express from "express"; +import { unprocessable, forbidden } from "../errors.js"; import request from "supertest"; import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { vi } from "vitest"; @@ -259,6 +260,30 @@ describe("adapter routes", () => { expect(resolveRunnerTarget).not.toHaveBeenCalled(); }); + it("keeps Cursor API discovery available without granting local support when the implicit managed Computer is unavailable", async () => { + const unavailable = unprocessable("Managed Computer unavailable", { code: "managed_sandbox_unavailable" }); + const app = createApp({}, { resolveRunnerTarget: async () => { throw unavailable; } }); + const response = await request(app).get("/api/adapters?companyId=company-1"); + expect(response.status).toBe(200); + expect(response.body.find((adapter: any) => adapter.type === "cursor_cloud")).toMatchObject({ supportedRunners: ["paperclip"], defaultRunner: "paperclip" }); + expect(response.body.find((adapter: any) => adapter.type === "codex_local").supportedRunners).toEqual([]); + const explicit = await request(app).get("/api/adapters?companyId=company-1&environmentId=selected"); + expect(explicit.status).toBe(422); + const foreign = await request(createApp({}, { resolveRunnerTarget: async () => { throw forbidden("Foreign Computer", { code: "environment_company_mismatch" }); } })).get("/api/adapters?companyId=company-1"); + expect(foreign.status).toBe(403); + }); + + it("does not offer Cursor execution under Kubernetes-only policy, including the unavailable managed Computer fallback", async () => { + for (const resolveRunnerTarget of [ + async () => ({ driver: "sandbox", executionMode: "kubernetes" as const }), + async () => { throw unprocessable("Managed Computer unavailable", { code: "managed_sandbox_unavailable", executionMode: "kubernetes" }); }, + ]) { + const response = await request(createApp({}, { resolveRunnerTarget })).get("/api/adapters?companyId=company-1"); + expect(response.status).toBe(200); + expect(response.body.find((adapter: any) => adapter.type === "cursor_cloud").supportedRunners).toEqual([]); + } + }); + it.each(["environmentId=selected", "companyId=", "companyId=one&companyId=two", "companyId=one&environmentId="])("rejects malformed discovery scope before resolution (%s)", async query => { const resolveRunnerTarget = vi.fn(); const response = await request(createApp({}, { resolveRunnerTarget })).get(`/api/adapters?${query}`); diff --git a/server/src/__tests__/agent-adapter-validation-routes.test.ts b/server/src/__tests__/agent-adapter-validation-routes.test.ts index 75c8f730ed7..1606070bb74 100644 --- a/server/src/__tests__/agent-adapter-validation-routes.test.ts +++ b/server/src/__tests__/agent-adapter-validation-routes.test.ts @@ -68,7 +68,7 @@ const mockApprovalService = vi.hoisted(() => ({ const mockInstanceSettingsService = vi.hoisted(() => ({ get: vi.fn(async () => ({ defaultEnvironmentId: null })), getGeneral: vi.fn(async () => ({ censorUsernameInLogs: false })), - getExperimental: vi.fn(async (): Promise<{ enableNativeRunner: boolean; enableOpenAiDot?: boolean }> => ({ enableNativeRunner: false })), + getExperimental: vi.fn(async (): Promise<{ enableNativeRunner: boolean; enableOpenAiDot?: boolean; enableManagedSandboxOnly?: boolean }> => ({ enableNativeRunner: false })), })); const mockManagedAgentProfileService = vi.hoisted(() => ({ @@ -268,6 +268,7 @@ describe("agent routes adapter validation", () => { mockAccessService.setPrincipalPermission.mockResolvedValue(undefined); mockLogActivity.mockResolvedValue(undefined); mockSecretService.syncEnvBindingsForTarget.mockResolvedValue(undefined); + mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: null }); mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: false }); mockManagedAgentProfileService.requireQualified.mockResolvedValue({ id: "managed-primary", @@ -959,6 +960,46 @@ describe("agent routes adapter validation", () => { expect(mockAgentService.create).not.toHaveBeenCalled(); }); + it("selects the native Cursor API controller from the ordinary held Cloud setup card without a Computer", async () => { + const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + mockInstanceSettingsService.get.mockResolvedValue(settings); + mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental); + const environments = await import("../services/environments.js"); + const findManaged = vi.fn(async () => null); + const getEnvironment = vi.fn(async () => null); + vi.spyOn(environments, "environmentService").mockReturnValue({ findManagedSandboxEnvironment: findManaged, + getById: getEnvironment, listBoundCompanyIds: vi.fn(async () => []) } as never); + const existing = { ...(await mockAgentService.getById()), adapterType: "process", adapterConfig: {}, + status: "paused", lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] }; + mockAgentService.getById.mockResolvedValue(existing); + const app = await createApp(); + const response = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({ + name: "Cursor setup", adapterType: "cursor_cloud", runner: "auto", adapterConfig: {}, metadata: { setupAdapterPending: false }, + })); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] }); + expect(mockAgentService.update).toHaveBeenCalledWith(existing.id, expect.objectContaining({ + adapterType: "paperclip_runner", defaultEnvironmentId: null, + adapterConfig: expect.objectContaining({ provider: "cursor_cloud", lifecycleMode: "per_turn" }), + }), expect.anything()); + expect(findManaged).not.toHaveBeenCalled(); + expect(getEnvironment).not.toHaveBeenCalled(); + mockAgentService.update.mockClear(); + mockAgentService.getById.mockResolvedValue({ ...existing, status: "idle", lifecycleState: "ready", pauseReason: null, lifecycleHolds: [] }); + const active = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({ + adapterType: "cursor_cloud", runner: "auto", adapterConfig: {}, setupOnly: true, + })); + expect(active.status).toBe(422); + expect(active.body.details?.code).toBe("managed_sandbox_unavailable"); + expect(mockAgentService.update).not.toHaveBeenCalled(); + const incompleteActive = await requestApp(app, base => request(base).patch(`/api/agents/${existing.id}`).send({ + adapterType: "paperclip_runner", runner: "paperclip", adapterConfig: { provider: "cursor_cloud" }, setupOnly: true, + })); + expect(incompleteActive.status).toBe(422); + expect(incompleteActive.body.details?.code).toBe("paperclip_runner_cursor_cloud_model_required"); + expect(mockAgentService.update).not.toHaveBeenCalled(); + }); + it.each(["auto", "legacy"] as const)("creates Pi through the public route with %s selection and the experimental flag off", async runner => { const selection = await import("../services/agent-runner-selection.js"); vi.spyOn(selection, "resolveNewAgentRunnerForCompany").mockImplementation(async (_db, _companyId, input) => @@ -1107,9 +1148,9 @@ describe("agent routes adapter validation", () => { ); }); - it("saves and reopens an explicit native Cursor Cloud setup without Codex defaults", async () => { - mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true }); - let existing = { ...(await mockAgentService.getById()), adapterType: "cursor_cloud", adapterConfig: {}, + it("saves and reopens native Cursor API setup under managed-only policy without a Computer", async () => { + mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableNativeRunner: true, enableManagedSandboxOnly: true }); + let existing = { ...(await mockAgentService.getById()), adapterType: "cursor_cloud", adapterConfig: {}, defaultEnvironmentId: "33333333-3333-4333-8333-333333333333", status: "paused", lifecycleState: "paused", pauseReason: "setup", lifecycleHolds: ["setup"] }; mockAgentService.getById.mockImplementation(async () => existing); mockAgentService.update.mockImplementation(async (_id, patch) => { existing = { ...existing, ...patch }; return existing; }); @@ -1123,6 +1164,7 @@ describe("agent routes adapter validation", () => { provider: "cursor_cloud", model: config.model, repoUrl: config.repoUrl, repoRef: "feature", env: config.env, lifecycleMode: "per_turn", } }); expect(saved.body.adapterConfig).not.toHaveProperty("codexPermissionMode"); + expect(saved.body.defaultEnvironmentId).toBeNull(); const reopened = await requestApp(app, baseUrl => request(baseUrl).get(`/api/agents/${existing.id}/configuration`)); expect(reopened.status, JSON.stringify(reopened.body)).toBe(200); expect(reopened.body.adapterConfig).toEqual(saved.body.adapterConfig); diff --git a/server/src/__tests__/agent-harness-verification-provider-api.test.ts b/server/src/__tests__/agent-harness-verification-provider-api.test.ts new file mode 100644 index 00000000000..b117ae7eaf2 --- /dev/null +++ b/server/src/__tests__/agent-harness-verification-provider-api.test.ts @@ -0,0 +1,164 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { eq, sql } from "drizzle-orm"; +import { agents, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + environments, environmentLeases, executionWorkspaces, heartbeatRuns, plugins, type Db } from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { requireServerAdapter } from "../adapters/index.js"; +import { agentHarnessVerificationService } from "../services/agent-harness-verification.js"; +import { createAgentLifecycle, configureAgentLifecycle } from "../services/agent-lifecycle.js"; +import { createLifecycleDriver } from "../services/agent-lifecycle-driver.js"; +import { instanceSettingsService } from "../services/instance-settings.js"; +import { environmentService } from "../services/environments.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; + +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_PROVIDER_API_LIFECYCLE_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("Cursor Cloud API controller activation", () => { + let database: Awaited>; + let db: Db; + let home: string; + let companyId: string; + const workers: ReturnType[] = []; + const config = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", + lifecycleMode: "per_turn", env: { CURSOR_API_KEY: "synthetic-cursor-key" } }; + const fetch = vi.fn(async (input: unknown, init?: RequestInit) => { + const url = String(input); + if (url.startsWith("https://callback.example.test/mcp/native-runs/") && init?.method === "POST") return new Response(null, { status: 401 }); + if (url === "https://api.cursor.com/v1/models" && init?.method === "GET") return new Response(JSON.stringify({ items: [{ id: config.model, displayName: "Caller model" }] }), { headers: { "Content-Type": "application/json" } }); + throw new Error(`Unexpected provider request: ${init?.method} ${url}`); + }); + + beforeAll(async () => { + home = await mkdtemp(join(tmpdir(), "paperclip-provider-api-lifecycle-")); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", undefined); + vi.stubEnv("PAPERCLIP_SETTING_DEFAULTS", undefined); + vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://callback.example.test"); + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); + database = externalDatabaseUrl ? { connectionString: externalDatabaseUrl, cleanup: async () => {} } + : await startEmbeddedPostgresTestDatabase("paperclip-provider-api-lifecycle-"); + db = createDb(database.connectionString); + await db.execute(sql`set client_min_messages = warning`); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + beforeEach(async () => { + for (const worker of workers) await worker.stop(); + await instanceSettingsService(db).update({ defaultEnvironmentId: null }); + await db.execute(sql`truncate table companies, plugins, environments restart identity cascade`); + await instanceSettingsService(db).updateGeneral({ executionMode: "any" }); + await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true, enableNativeRunner: true }); + [companyId] = await db.insert(companies).values({ name: "API controller activation", issuePrefix: "API" }).returning({ id: companies.id }).then(rows => rows.map(row => row.id)); + fetch.mockClear(); + vi.stubGlobal("fetch", fetch); + }); + afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); + afterAll(async () => { + for (const worker of workers) await worker.stop(); + await db?.$client.end(); + await database?.cleanup(); + vi.unstubAllEnvs(); + if (home) await rm(home, { recursive: true, force: true }); + }); + + const current = (id: string) => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0]!); + async function saved(adapterType = "paperclip_runner", adapterConfig = config) { + return (await db.insert(agents).values({ companyId, name: "Saved controller", adapterType, adapterConfig, + status: "paused", lifecycleState: "verifying" }).returning())[0]!; + } + async function noComputerWork() { + expect(await db.select().from(environmentLeases)).toHaveLength(0); + expect(await db.select().from(executionWorkspaces)).toHaveLength(0); + expect(await db.select().from(heartbeatRuns)).toHaveLength(0); + } + + it.each([false, true])("finishes the same held draft and reaches Ready with an experimental account WARN, inherited plugin Computer=%s", async defaultComputer => { + if (defaultComputer) { + const pluginId = randomUUID(); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "unneeded-computer", version: "1.0.0", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", requiresAgent: true, supportsTasks: true, warmTaskVersion: 1 }] } as never }); + const [computer] = await db.insert(environments).values({ name: "Inherited Computer", driver: "plugin", status: "active", + config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }).returning(); + await instanceSettingsService(db).update({ defaultEnvironmentId: computer.id }); + } + expect(await environmentService(db).findManagedSandboxEnvironment(companyId)).toBeNull(); + const lifecycle = createAgentLifecycle(db); + const draft = await lifecycle.requestHire(companyId, { name: "Cursor setup", adapterType: "paperclip_runner", + adapterConfig: { provider: "cursor_cloud" } }, { setupOnly: true, runnerResolved: true }); + expect(draft).toMatchObject({ status: "paused", lifecycleState: "paused", lifecycleHolds: ["setup"], defaultEnvironmentId: null }); + await lifecycle.updateConfiguration(draft.id, { adapterConfig: config, runtimeConfig: { heartbeat: { enabled: false } } }); + expect(await current(draft.id)).toMatchObject({ lifecycleState: "paused", lifecycleHolds: ["setup"] }); + const finished = await lifecycle.requestHire(companyId, { id: draft.id, name: draft.name, adapterType: "paperclip_runner", + adapterConfig: config, runtimeConfig: { heartbeat: { enabled: false } }, defaultEnvironmentId: null }, { setupAgentId: draft.id, runnerResolved: true }); + expect(finished.id).toBe(draft.id); + expect(finished.adapterConfig.lifecycleMode).toBe("per_turn"); + const probe = vi.spyOn(requireServerAdapter("paperclip_runner"), "testEnvironment"); + const driver = createLifecycleDriver(db, {} as never); + const worker = configureAgentLifecycle(db, driver); + workers.push(worker); + await worker.process(draft.id); + expect(await current(draft.id)).toMatchObject({ status: "idle", lifecycleState: "ready", lifecycleHolds: [], defaultEnvironmentId: null }); + expect(probe).toHaveBeenCalledOnce(); + expect(await probe.mock.results[0]!.value).toMatchObject({ status: "warn", checks: [ + { code: "cursor_cloud_account_ready", level: "info" }, + { code: "cursor_cloud_model_listed", level: "info" }, + { code: "cursor_cloud_model_identity_unverified", level: "warn" }, + ] }); + expect(fetch.mock.calls.map(([url, input]) => [String(url), input?.method])).toEqual([ + ["https://callback.example.test/mcp/native-runs/00000000-0000-4000-8000-000000000000", "POST"], + ["https://api.cursor.com/v1/models", "GET"], + ]); + await noComputerWork(); + }); + + it("keeps native Computer-backed setup preparation and warm-task defaults", async () => { + const pluginId = randomUUID(); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "required-computer", version: "1.0.0", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", requiresAgent: true, supportsTasks: true, warmTaskVersion: 1 }] } as never }); + const [computer] = await db.insert(environments).values({ name: "Required Computer", driver: "plugin", status: "active", + config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }).returning(); + await instanceSettingsService(db).update({ defaultEnvironmentId: computer.id }); + const draft = await createAgentLifecycle(db).requestHire(companyId, { name: "Codex setup", adapterType: "paperclip_runner", + adapterConfig: { provider: "codex", model: "gpt-5.6-luna", lifecycleMode: "per_turn" } }, { setupOnly: true, runnerResolved: true }); + expect(draft).toMatchObject({ status: "paused", lifecycleState: "preparing", lifecycleHolds: ["setup"], adapterConfig: { lifecycleMode: "warm" } }); + expect(fetch).not.toHaveBeenCalled(); + await noComputerWork(); + }); + + it("verifies a saved native controller without resolving an unavailable managed Computer", async () => { + const agent = await saved(); + await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).resolves.toBe("complete"); + expect(fetch).toHaveBeenCalledTimes(2); + await noComputerWork(); + }); + + it("rejects Kubernetes-only activation before any provider probe", async () => { + const agent = await saved(); + await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" }); + await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({ + status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" }, + }); + await expect(createAgentLifecycle(db).requestHire(companyId, { name: "Forbidden hire", adapterType: "paperclip_runner", adapterConfig: config }, { runnerResolved: true })) + .rejects.toMatchObject({ status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" } }); + expect(fetch).not.toHaveBeenCalled(); + await noComputerWork(); + }); + + it("does not grant API placement to an incomplete active profile", async () => { + const agent = await saved("paperclip_runner", { ...config, model: "" }); + await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({ code: "paperclip_runner_cursor_cloud_model_required" }); + expect(fetch).not.toHaveBeenCalled(); + await noComputerWork(); + }); + + it.each(["opencode_local", "cursor_cloud"])("preserves the managed Computer requirement for %s", async adapterType => { + const agent = await saved(adapterType); + await expect(agentHarnessVerificationService(db, {} as never).verify(agent)).rejects.toMatchObject({ details: { code: "managed_sandbox_unavailable" } }); + expect(fetch).not.toHaveBeenCalled(); + await noComputerWork(); + }); +}); diff --git a/server/src/__tests__/agent-test-environment-routes.test.ts b/server/src/__tests__/agent-test-environment-routes.test.ts index 7a9088665ed..9a93d6badb2 100644 --- a/server/src/__tests__/agent-test-environment-routes.test.ts +++ b/server/src/__tests__/agent-test-environment-routes.test.ts @@ -293,6 +293,177 @@ describe("agent test-environment route", () => { await unregisterTestAdapter("external_test"); }); + it("tests the saved Cursor API controller without allocating a Computer under managed-only policy", async () => { + const { registerServerAdapter, unregisterServerAdapter, getServerAdapter } = await import("../adapters/index.js"); + const previous = getServerAdapter("paperclip_runner"); + const probe = vi.fn(async () => ({ adapterType: "paperclip_runner", status: "warn" as const, testedAt: new Date().toISOString(), + checks: [{ code: "cursor_cloud_account_ready", level: "info" as const, message: "Synthetic account probe; no prompt." }] })); + registerServerAdapter({ ...externalAdapter, type: "paperclip_runner", testEnvironment: probe }); + try { + const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const adapterConfig = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", env: { CURSOR_API_KEY: "synthetic" } }; + mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "paperclip_runner", adapterConfig, defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" }); + mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableManagedSandboxOnly: true }); + mockInstanceSettingsService.get.mockResolvedValue({ defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" }); + const app = await createApp(); + const response = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({ agentId: id, adapterConfig }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ status: "warn", checks: [{ code: "cursor_cloud_account_ready" }] }); + expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null, config: adapterConfig })); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.realizeWorkspace).not.toHaveBeenCalled(); + expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled(); + mockInstanceSettingsService.getGeneral.mockResolvedValue({ censorUsernameInLogs: false, executionMode: "kubernetes" }); + const denied = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({ agentId: id, adapterConfig }); + expect(denied.status).toBe(422); + expect(denied.body.details?.code ?? denied.body.code).toBe("provider_api_kubernetes_execution_unsupported"); + expect(probe).toHaveBeenCalledOnce(); + } finally { unregisterServerAdapter("paperclip_runner"); if (previous) registerServerAdapter(previous); } + }); + + async function mockCursorSwitchProbes() { + const { getServerAdapter } = await import("../adapters/index.js"); + const legacy = vi.spyOn(getServerAdapter("cursor_cloud"), "testEnvironment").mockImplementation(async () => ({ + adapterType: "cursor_cloud", status: "pass", checks: [], testedAt: new Date().toISOString(), + })); + const native = vi.spyOn(getServerAdapter("paperclip_runner"), "testEnvironment").mockImplementation(async () => ({ + adapterType: "paperclip_runner", status: "warn", testedAt: new Date().toISOString(), + checks: [{ code: "cursor_cloud_account_ready", level: "info", message: "Synthetic no-prompt probe" }], + })); + return { legacy, native, restore: () => { legacy.mockRestore(); native.mockRestore(); } }; + } + + function savedCursorSwitchFixture() { + const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const environmentId = "22222222-2222-4222-8222-222222222222"; + const adapterConfig = { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", env: {} }; + mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "paperclip_runner", + adapterConfig, runtimeConfig: {}, defaultEnvironmentId: "11111111-1111-4111-8111-111111111111" }); + const envVars = { CURSOR_API_KEY: { type: "secret_ref", secretId: "computer-key" } }; + mockEnvironmentService.getById.mockResolvedValue({ id: environmentId, name: "Chosen Computer", + driver: "local", config: {}, envVars }); + mockSecretService.resolveEnvBindings.mockResolvedValue({ env: { CURSOR_API_KEY: "chosen-computer-key" }, + secretKeys: new Set(["CURSOR_API_KEY"]), manifest: [] }); + return { id, environmentId, adapterConfig, envVars }; + } + + it.each([false, true])("tests a saved native Cursor's explicit Legacy switch on the selected Computer (legacyPrompt=%s)", async legacyPrompt => { + const probes = await mockCursorSwitchProbes(); + const probe = probes.legacy; + try { + const f = savedCursorSwitchFixture(); + const config = { ...f.adapterConfig, ...(legacyPrompt ? { promptTemplate: "Legacy prompt" } : {}) }; + const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment") + .send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId, adapterConfig: config }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(probe).toHaveBeenCalledWith(expect.objectContaining({ environmentName: "Chosen Computer", executionTarget: null, + config: expect.objectContaining({ model: "caller-model", env: { CURSOR_API_KEY: "chosen-computer-key" }, + ...(legacyPrompt ? { promptTemplate: "Legacy prompt" } : {}) }) })); + expect(probe.mock.calls[0][0].config).not.toHaveProperty("provider"); + expect(mockSecretService.resolveEnvBindings).toHaveBeenCalledWith("company-1", f.envVars, + expect.objectContaining({ consumerType: "environment", consumerId: f.environmentId })); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(probes.native).not.toHaveBeenCalled(); + } finally { probes.restore(); } + }); + + it("fails closed on a foreign Computer before credentials for an explicit Legacy switch", async () => { + const probes = await mockCursorSwitchProbes(); + try { + const f = savedCursorSwitchFixture(); + mockEnvironmentService.listBoundCompanyIds.mockResolvedValue(["company-2"]); + const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment") + .send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId, + adapterConfig: { ...f.adapterConfig, promptTemplate: "Legacy prompt" } }); + expect(response.status).toBe(403); + expect(mockSecretService.resolveAdapterConfigForRuntime).not.toHaveBeenCalled(); + expect(mockSecretService.resolveEnvBindings).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(probes.legacy).not.toHaveBeenCalled(); + expect(probes.native).not.toHaveBeenCalled(); + } finally { probes.restore(); } + }); + + it("keeps managed-only policy for a Legacy switch rather than testing its API controller", async () => { + const probes = await mockCursorSwitchProbes(); + const probe = probes.legacy; + try { + const f = savedCursorSwitchFixture(); + mockInstanceSettingsService.getExperimental.mockResolvedValue({ enableManagedSandboxOnly: true }); + const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment") + .send({ agentId: f.id, runner: "legacy", environmentId: f.environmentId, adapterConfig: f.adapterConfig }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ status: "fail", checks: expect.arrayContaining([ + expect.objectContaining({ code: "managed_sandbox_unavailable", level: "error" }), + ]) }); + expect(probe).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(probes.native).not.toHaveBeenCalled(); + } finally { probes.restore(); } + }); + + it("keeps an explicit Paperclip Cursor test API-only despite a selected Computer", async () => { + const probes = await mockCursorSwitchProbes(); + const probe = probes.native; + try { + const f = savedCursorSwitchFixture(); + const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + mockInstanceSettingsService.get.mockResolvedValue(settings); + mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental); + const response = await request(await createApp()).post("/api/companies/company-1/adapters/paperclip_runner/test-environment") + .send({ agentId: f.id, runner: "paperclip", environmentId: f.environmentId, adapterConfig: f.adapterConfig }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null, + config: expect.objectContaining({ provider: "cursor_cloud" }) })); + expect(mockEnvironmentService.getById).not.toHaveBeenCalled(); + expect(mockSecretService.resolveEnvBindings).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(probes.legacy).not.toHaveBeenCalled(); + } finally { probes.restore(); } + }); + + it.each([ + { patch: { model: "" }, code: "paperclip_runner_cursor_cloud_model_required" }, + { patch: { command: "untrusted" }, code: "paperclip_runner_cursor_cloud_config_invalid" }, + ])("returns a useful validation error before Computer or credential work for invalid native Cursor input ($code)", async ({ patch, code }) => { + const app = await createApp(); + const response = await request(app).post("/api/companies/company-1/adapters/paperclip_runner/test-environment").send({ + adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", ...patch }, + }); + expect(response.status, JSON.stringify(response.body)).toBe(422); + expect(response.body.details?.code).toBe(code); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.realizeWorkspace).not.toHaveBeenCalled(); + expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled(); + }); + + it.each([undefined, "auto"] as const)("tests the prospective managed Cursor API default without converting saved Legacy agents (%s)", async runner => { + const { getServerAdapter } = await import("../adapters/index.js"); + const native = getServerAdapter("paperclip_runner"); + const probe = vi.spyOn(native, "testEnvironment").mockImplementation(async () => ({ adapterType: "paperclip_runner", status: "warn" as const, testedAt: new Date().toISOString(), + checks: [{ code: "cursor_cloud_account_ready", level: "info" as const, message: "Synthetic account probe; no prompt." }] })); + try { + const settings = { defaultEnvironmentId: null, experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + mockInstanceSettingsService.get.mockResolvedValue(settings); + mockInstanceSettingsService.getExperimental.mockResolvedValue(settings.experimental); + const adapterConfig = { model: "caller-model", repoUrl: "https://github.com/example/qualification", env: { CURSOR_API_KEY: "synthetic" } }; + const app = await createApp(); + const response = await request(app).post("/api/companies/company-1/adapters/cursor_cloud/test-environment").send({ runner, adapterConfig }); + expect(response.status, JSON.stringify(response.body)).toBe(200); + expect(response.body).toMatchObject({ adapterType: "paperclip_runner", checks: [{ code: "cursor_cloud_account_ready" }] }); + expect(probe).toHaveBeenCalledWith(expect.objectContaining({ executionTarget: null, environmentName: null, + config: expect.objectContaining({ provider: "cursor_cloud", model: "caller-model" }) })); + expect(mockEnvironmentService.findManagedSandboxEnvironment).not.toHaveBeenCalled(); + expect(mockEnvironmentRuntime.acquireRunLease).not.toHaveBeenCalled(); + const id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + mockAgentService.getById.mockResolvedValue({ id, companyId: "company-1", adapterType: "cursor_cloud", adapterConfig, defaultEnvironmentId: null }); + const legacy = await request(app).post("/api/companies/company-1/adapters/cursor_cloud/test-environment").send({ agentId: id, adapterConfig }); + expect(legacy.status).toBe(422); + expect(legacy.body.details?.code).toBe("managed_sandbox_unavailable"); + expect(probe).toHaveBeenCalledOnce(); + } finally { probe.mockRestore(); } + }); + it("accepts a managed Dot environment without allocating an idle sandbox", async () => { vi.stubEnv("PAPERCLIP_PUBLIC_URL", "https://paperclip.example"); mockInstanceSettingsService.getExperimental.mockResolvedValue({ diff --git a/server/src/__tests__/budgets-service.test.ts b/server/src/__tests__/budgets-service.test.ts index 1b2495cf7b6..0d1580064ef 100644 --- a/server/src/__tests__/budgets-service.test.ts +++ b/server/src/__tests__/budgets-service.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { DELIVERY_QUEUES, subscribeDeliveryWork } from "../services/delivery-work-notifications.js"; import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agents, @@ -11,10 +11,14 @@ import { companies, costEvents, createDb, + heartbeatRuns, + issues, + nativeRunFinalizations, projects, } from "@paperclipai/db"; import { budgetService, deliverBudgetEnforcement, hasPendingBudgetEnforcement } from "../services/budgets.ts"; import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, } from "./helpers/embedded-postgres.js"; @@ -40,13 +44,16 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => { await import("../services/agent-lifecycle.js"); if (!externalUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-budgets-service-"); db = createDb(externalUrl ?? tempDb!.connectionString); - }, 20_000); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterEach(async () => { await db.delete(budgetIncidents); await db.delete(approvals); await db.delete(budgetPolicies); await db.delete(costEvents); + await db.delete(nativeRunFinalizations); + await db.delete(heartbeatRuns); + await db.delete(issues); await db.delete(projects); await db.delete(agents); await db.delete(companies); @@ -118,6 +125,131 @@ describeEmbeddedPostgres("budgetService release gate enforcement", () => { return event!; } + it.each(["no policy", "healthy policy", "warning policy", "manual company pause", "manual project pause", "orphan budget pause", "recovering run"])( + "checks %s while another accounting writer holds the company lock", + async state => { + const { companyId, agentId, projectId } = await createBudgetFixture(); + if (state === "healthy policy" || state === "warning policy" || state === "recovering run") { + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "company", scopeId: companyId, + windowKind: "calendar_month_utc", amount: 100 }).returning(); + if (state !== "recovering run") await insertCostEvent({ companyId, agentId, costCents: state === "warning policy" ? 80 : 20 }); + if (state === "warning policy") { + const now = new Date(); + await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "company", scopeId: companyId, + metric: "billed_cents", windowKind: "calendar_month_utc", + windowStart: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1)), + windowEnd: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1)), + thresholdType: "soft", amountLimit: 100, amountObserved: 80 }); + } + } + if (state === "recovering run") { + const [issue] = await db.insert(issues).values({ companyId, projectId, title: "Recover accounting" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, scopeKind: "issue", issueId: issue.id, + nativeIssueId: issue.id, runtimeMode: "native", status: "failed", finishedAt: new Date(), costAccountingPending: true }).returning(); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: issue.id, runId: run.id, phase: "observed" }); + } + if (state === "manual company pause" || state === "orphan budget pause") { + await db.update(companies).set({ status: "paused", pauseReason: state === "orphan budget pause" ? "budget" : "manual" }) + .where(eq(companies.id, companyId)); + } + if (state === "manual project pause") { + await db.update(projects).set({ pausedAt: new Date(), pauseReason: "manual" }).where(eq(projects.id, projectId)); + } + + const reader = createDb(externalUrl ?? tempDb!.connectionString, { maxConnections: 1 }); + let acquired!: () => void, release!: () => void; + const lockAcquired = new Promise(resolve => { acquired = resolve; }); + const holdLock = new Promise(resolve => { release = resolve; }); + const writer = db.transaction(async tx => { + await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)).for("no key update"); + acquired(); + await holdLock; + }); + try { + await Promise.race([lockAcquired, writer]); + await reader.execute(sql`set lock_timeout = '500ms'`); + if (state === "recovering run") { + await expect(budgetService(reader).getInvocationBlock(companyId, agentId, { projectId })) + .rejects.toMatchObject({ cause: { code: "55P03" } }); + } else { + const block = await budgetService(reader).getInvocationBlock(companyId, agentId, { projectId }); + if (state === "manual company pause") { + expect(block).toMatchObject({ scopeType: "company", scopeId: companyId, + reason: "Company is paused and cannot start new work." }); + } else if (state === "manual project pause") { + expect(block).toMatchObject({ scopeType: "project", scopeId: projectId, + reason: "Project is paused and cannot start new work." }); + } else if (state === "orphan budget pause") { + expect(block).toMatchObject({ scopeType: "company", scopeId: companyId, + reason: "Budget pause requires a policy or an explicit operator resume." }); + } else expect(block).toBeNull(); + if (state === "warning policy") { + expect(await reader.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, companyId))) + .toMatchObject([{ thresholdType: "soft", status: "open", resolvedAt: null }]); + } + if (state === "no policy") { + await expect(budgetService(reader).evaluateAdmission(companyId, agentId, { projectId })) + .rejects.toMatchObject({ cause: { code: "55P03" } }); + } + } + } finally { + release(); + try { await writer; } + finally { await reader.$client.end({ timeout: 1 }); } + } + if (state === "recovering run") { + const cancelWorkForScope = vi.fn(); + expect(await budgetService(db, { cancelWorkForScope }).getInvocationBlock(companyId, agentId, { projectId })) + .toMatchObject({ scopeType: "company", scopeId: companyId, + reason: "New work must wait while a native run recovers its unfinished accounting." }); + expect(cancelWorkForScope).not.toHaveBeenCalled(); + expect((await db.select().from(companies).where(eq(companies.id, companyId)))[0]) + .toMatchObject({ status: "active", pauseReason: null }); + expect((await db.select().from(agents).where(eq(agents.id, agentId)))[0]) + .toMatchObject({ status: "active", pauseReason: null }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId))) + .toMatchObject([{ status: "failed", costAccountingPending: true }]); + expect(await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.companyId, companyId))) + .toMatchObject([{ phase: "observed", resultId: null }]); + } + }, + ); + + it.each([true, false])("releases an agent budget hold when its policy is healthy (active: %s)", async isActive => { + const { companyId, agentId } = await createBudgetFixture(); + await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId, + windowKind: "calendar_month_utc", amount: 100, isActive }); + await db.update(agents).set({ status: "paused", lifecycleState: "paused", pauseReason: "budget", pausedAt: new Date(), lifecycleHolds: ["budget"] }) + .where(eq(agents.id, agentId)); + + expect(await budgetService(db).getInvocationBlock(companyId, agentId)).toBeNull(); + expect((await db.select().from(agents).where(eq(agents.id, agentId)))[0]).toMatchObject({ + lifecycleState: "resuming", pauseReason: null, lifecycleHolds: [], + }); + }); + + it.each(["hard", "soft"])("resolves an obsolete %s incident even when the scope no longer has a budget hold", async thresholdType => { + const { companyId, agentId } = await createBudgetFixture(); + const [policy] = await db.insert(budgetPolicies).values({ companyId, scopeType: "agent", scopeId: agentId, + windowKind: "calendar_month_utc", amount: 100 }).returning(); + const [incident] = await db.insert(budgetIncidents).values({ companyId, policyId: policy.id, scopeType: "agent", scopeId: agentId, + metric: "billed_cents", windowKind: "calendar_month_utc", windowStart: new Date(0), windowEnd: new Date(1), + thresholdType, amountLimit: 100, amountObserved: 100 }).returning(); + + expect(await budgetService(db).getInvocationBlock(companyId, agentId)).toBeNull(); + expect((await db.select().from(budgetIncidents).where(eq(budgetIncidents.id, incident.id)))[0]) + .toMatchObject({ status: "resolved", resolvedAt: expect.any(Date) }); + }); + + it("rejects another company's agent or project before allowing a no-policy invocation", async () => { + const own = await createBudgetFixture(); + const other = await createBudgetFixture(); + const service = budgetService(db); + await expect(service.getInvocationBlock(own.companyId, other.agentId)).rejects.toThrow("Budget scope not found"); + await expect(service.getInvocationBlock(own.companyId, own.agentId, { projectId: other.projectId })) + .rejects.toThrow("Budget scope not found"); + }); + it("raises one soft incident per window before hard-stopping and preserving safe local audit details", async () => { const { companyId, agentId } = await createBudgetFixture(); const cancelWorkForScope = vi.fn().mockResolvedValue(undefined); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 25d571411f2..5364202d735 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -11,6 +11,7 @@ import { retainGitHubReviewReceipt } from "../services/chat-channels/github/task import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; import { recordRunPresentationDecision } from "../services/heartbeat/run-completion.js"; import { environmentService } from "../services/environments.js"; +import { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } from "../services/workspace-runtime.js"; import { createAdmissionTransactionScope, createWakeAdmissionWriter } from "../modules/wake-queue/adapters/postgres.js"; import { chatFailedRetryNoticeWork } from "../services/chat-failed-retry-notice-work.js"; import { issueTreeControlService } from "../services/issue-tree-control.js"; @@ -33,6 +34,10 @@ import * as githubTaskPreparation from "../services/chat-channels/github/task-pr import { budgetPolicies } from "@paperclipai/db"; import { budgetService } from "../services/budgets.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; +import { readIdleLocalWork } from "../services/idle-local-work.js"; +import { incomingWakeWorkQuery } from "../services/idle-incoming-wake.js"; +import { createChatRuntime } from "../services/chat-runtime.js"; +import { createChatSdkRuntime } from "../services/chat-sdk-runtime.js"; import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js"; import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js"; import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; @@ -809,6 +814,8 @@ class FakeChatSdkRuntime { return this.endpoints.get(endpointId) ?? null; } + hasUnsettledLifecycle() { return false; } + async replaceEndpoint(options: CreateChatSdkEndpointRuntimeOptions) { this.replaceCount += 1; this.configurations.set(options.endpointId, options); @@ -1128,8 +1135,20 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { // Do not expire or time out its connections on those virtual clock jumps. db = createDb(externalTestDatabaseUrl ?? tempDb!.connectionString, { idleTimeoutSeconds: 0, connectTimeoutSeconds: 0, maxLifetimeSeconds: 0, - ...(historyTestDatabaseUrl ? { maxConnections: 4 } : {}), + ...(externalTestDatabaseUrl ? { maxConnections: 4 } : {}), }); + // The composed local-probe inventory includes the workspace owner. Match + // server startup on this fresh fixture; never adopt or start a leftover. + const [runtimeState] = await db.execute<{ configured: boolean }>(sql`SELECT + EXISTS (SELECT 1 FROM workspace_runtime_services) + OR EXISTS (SELECT 1 FROM project_workspaces) + OR EXISTS (SELECT 1 FROM execution_workspaces) AS configured`); + expect(runtimeState.configured).toBe(false); + expect(await reconcilePersistedRuntimeServicesOnStartup(db)).toEqual({ + reconciled: 0, adopted: 0, stopped: 0, backfilled: 0, restarted: 0, + restartFailed: 0, exposureReservationDrift: [], + }); + expect(readWorkspaceRuntimeIdleWork()).toBe("none"); }, 90_000); afterAll(async () => { @@ -4185,10 +4204,14 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .toEqual({ next: null, active: 0 }); expect(f.responseWrites).toHaveLength(baseline); vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); - const reads = vi.spyOn(db, "select"), writes = vi.spyOn(db, "transaction"); + const readStacks: string[] = [], select = db.select; + const reads = vi.spyOn(db, "select").mockImplementation((...args) => { + if (readStacks.length < 5) readStacks.push(new Error("Unexpected idle database read").stack ?? "Stack unavailable"); + return Reflect.apply(select, db, args); + }), writes = vi.spyOn(db, "transaction"); try { await vi.advanceTimersByTimeAsync(120_000); - expect(reads).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); + expect(reads, readStacks.join("\n\n")).not.toHaveBeenCalled(); expect(writes).not.toHaveBeenCalled(); expect(f.responseWrites).toHaveLength(baseline); } finally { reads.mockRestore(); writes.mockRestore(); vi.useRealTimers(); } const settlePresentation = async (tx: Parameters[0]>[0]) => { @@ -29957,13 +29980,23 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }), ); await initializationEntered; + expect(runtime.hasUnsettledLifecycle()).toBe(false); // Fake registry tail is already settled. + expect(await registeredIdleProbeState()).toBe("present"); await service.configure(endpoint.id, { action: "pause" }, "owner-user"); + let shutdownSettled = false; + const shutdown = service.shutdown().then(() => { shutdownSettled = true; }); + await new Promise(resolve => setImmediate(resolve)); + expect(shutdownSettled).toBe(false); + expect(await registeredIdleProbeState()).toBe("present"); releaseInitialization(); await expect(initializingWebhook).rejects.toMatchObject({ status: 409, details: { code: "chat_endpoint_runtime_superseded" }, }); + await shutdown; + expect(shutdownSettled).toBe(true); + expect(await registeredIdleProbeState()).toBe("none"); expect(runtime.endpoints.has(endpoint.id)).toBe(false); await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "paused", @@ -49515,7 +49548,12 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { .then((rows) => rows.length); expect(deliveryCount).toBeGreaterThan(0); + const endpointBlocksSleep = async () => (await db.execute(sql` + SELECT EXISTS (${incomingWakeWorkQuery("chat_endpoints")} AND endpoint.id = ${endpoint.id}::uuid) AS blocked`))[0].blocked; + if (provider === "github") expect(await endpointBlocksSleep()).toBe(true); await service.configure(endpoint.id, { action: "remove" }, "owner-user"); + if (provider === "github") expect(await endpointBlocksSleep()).toBe(false); + else expect(await endpointBlocksSleep()).toBe(true); // No persistent/provider coverage expansion. await expect(service.get(endpoint.id)).resolves.toMatchObject({ status: "archived", @@ -49557,6 +49595,44 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }, ); + async function registeredIdleProbeState() { + let state: string | undefined; + await readIdleLocalWork({ diagnostic: (reason, value) => { if (reason === "local.registered_probes") state = value; } }); + return state; + } + + it.each([ + { name: "SDK", create: createChatSdkRuntime }, + { name: "shared", create: createChatRuntime }, + ])("retains the service idle probe after failed $name retirement and shutdown until owner retry", async ({ create }) => { + const fixture = await seedCompany(); + const f = await configuredGitHubEndpoint(fixture); + const registry = create(); + const first = await registry.replaceEndpoint(f.runtime.configurations.get(f.endpoint.id)!); + const shutdown = vi.spyOn(first, "shutdown") + .mockRejectedValueOnce(new Error("remove retirement uncertain")) + .mockRejectedValueOnce(new Error("shutdown retirement uncertain")) + .mockResolvedValue(undefined); + const service = chatChannelService(db, { runtime: registry, fetch: f.providerFetch, publicBaseUrl: "https://paperclip.example" }); + fixtureServices.add(service); + expect(await registeredIdleProbeState()).toBe("none"); + await service.configure(f.endpoint.id, { action: "remove" }, "owner-user"); + expect(registry.get(f.endpoint.id)).toBeNull(); + expect(registry.list()).toEqual([]); + expect(registry.hasUnsettledLifecycle()).toBe(true); + const select = vi.spyOn(db, "select"), transaction = vi.spyOn(db, "transaction"); + try { + expect(await registeredIdleProbeState()).toBe("present"); + expect(select).not.toHaveBeenCalled(); expect(transaction).not.toHaveBeenCalled(); + } finally { select.mockRestore(); transaction.mockRestore(); } + await expect(service.shutdown()).rejects.toThrow("shutdown retirement uncertain"); + expect(await registeredIdleProbeState()).toBe("present"); + await service.shutdown(); + expect(shutdown).toHaveBeenCalledTimes(3); + expect(registry.hasUnsettledLifecycle()).toBe(false); + expect(await registeredIdleProbeState()).toBe("none"); + }); + it("retries credential cleanup after archival made a remove partially complete", async () => { const fixture = await seedCompany(); const { endpoint, service } = await configuredSlackEndpoint(fixture); @@ -60587,7 +60663,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const commentsBefore = await db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)); + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(asc(issueComments.id)); const wakesBefore = wakeup.mock.calls.length; const reactionsBefore = providerRuntime.reactions.length; const postsBefore = providerRuntime.posts.length; @@ -60617,7 +60694,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { await db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)), + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(asc(issueComments.id)), ).toEqual(commentsBefore); expect(wakeup).toHaveBeenCalledTimes(wakesBefore); expect(cancelRun).not.toHaveBeenCalled(); @@ -64441,6 +64519,45 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } finally { await f.service.shutdown(); } }); + it("retries saved GitHub input after an endpoint admission lock collision", async () => { + const f = await successfulGitHubQueuedInputFixture(); + const heartbeat = heartbeatService(db); + const failures: unknown[] = []; + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, + onError: error => failures.push(error) }); + let unlock!: () => void, locked!: () => void; + const acquired = new Promise(resolve => { locked = resolve; }); + const released = new Promise(resolve => { unlock = resolve; }); + const competingAdmission = db.transaction(async tx => { + await tx.select({ id: chatEndpoints.id }).from(chatEndpoints) + .where(eq(chatEndpoints.id, f.endpoint.id)).for("no key update"); + locked(); + await released; + }); + try { + await acquired; + await heartbeat.registerDeferredReceiptWork(coordinator, () => true).ready; + const priorFailures = failures.length; + const retry = await requestTaskRetry(f.input); + await expect.poll(() => failures.length, { timeout: 5_000 }).toBeGreaterThan(priorFailures); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull(); + unlock(); await competingAdmission; + await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId, + { timeout: 40_000 }).toBeTruthy(); + await expect.poll(async () => (await readAgentWakeupRequestsById(retry.requestId!))[0].status, + { timeout: 40_000 }).toBe("coalesced"); + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)) + .filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1); + expect((await readAgentWakeupRequestsById(retry.requestId!))[0].runId).toBe(saved.runId); + expect((await readHeartbeatRunsById(f.runId))[0]).toMatchObject({ status: "succeeded" }); + } finally { + unlock(); await competingAdmission; + await coordinator.stop(); + await f.service.shutdown(); + } + }, 90_000); + it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)( "admits current GitHub task Retry without replaying obsolete input: %s", async kind => { const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input")); diff --git a/server/src/__tests__/chat-failed-run-retry-boundary.test.ts b/server/src/__tests__/chat-failed-run-retry-boundary.test.ts index ad62169924d..67cbd7f863a 100644 --- a/server/src/__tests__/chat-failed-run-retry-boundary.test.ts +++ b/server/src/__tests__/chat-failed-run-retry-boundary.test.ts @@ -14,7 +14,7 @@ function createService(db: Db) { const runtimeShutdown = vi.fn().mockResolvedValue(undefined); const service = chatChannelService(db, { heartbeat: { wakeup }, - runtime: { shutdown: runtimeShutdown } as unknown as ChatChannelServiceOptions["runtime"], + runtime: { shutdown: runtimeShutdown, hasUnsettledLifecycle: () => false } as unknown as ChatChannelServiceOptions["runtime"], }); services.add(service); return { service, wakeup, runtimeShutdown }; diff --git a/server/src/__tests__/deferred-execution-receipt-work.test.ts b/server/src/__tests__/deferred-execution-receipt-work.test.ts index e902c734429..7c028e3c2e8 100644 --- a/server/src/__tests__/deferred-execution-receipt-work.test.ts +++ b/server/src/__tests__/deferred-execution-receipt-work.test.ts @@ -72,14 +72,18 @@ describe("deferred receipt durable dispatch", () => { // during actual process execution; accelerated steps only settle idle work. vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: true }); heartbeat = heartbeatService(db); - coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, + coordinator = createCoordinator(); + }); + function createCoordinator() { + const worker = createDeliveryWorkCoordinator({ owner: db, canRun: () => !drain, canReconcile: () => true, onError: error => { errors.push(error); errorTimes.push(Date.now()); } }); - const register = coordinator.register.bind(coordinator); - coordinator.register = (queue, task) => register(queue, { ...task, run: signal => { + const register = worker.register.bind(worker); + worker.register = (queue, task) => register(queue, { ...task, run: signal => { const pass = Promise.resolve(task.run(signal)); passes.add(pass); void pass.then(() => passes.delete(pass), () => passes.delete(pass)); return pass; } }); - }); + return worker; + } afterEach(async () => { await coordinator.stop(); await heartbeat.drainActiveRunExecutions(); for (const stop of extraStops.splice(0)) await stop(); @@ -153,6 +157,78 @@ describe("deferred receipt durable dispatch", () => { source: "on_demand", status: "completed", reason: "task_retry", payload: { issueId: f.issue.id } }); await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; await quiet(); }); + it.each([false, true])("reconsiders every saved receipt during repeated company eligibility changes (expired wait: %s)", async expiredWait => { + const f = await fixture(); + const ids = Array.from({ length: 178 }, (_, index) => `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`); + await db.insert(agentWakeupRequests).values(ids.map(id => ({ id, companyId: f.companyId, agentId: f.agent.id, + source: "on_demand", status: "deferred_issue_execution", + updatedAt: new Date(Date.now() - 60_000), + payload: { issueId: f.issue.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] }, + ...(expiredWait ? { executionWait: { reason: "recovery" } } : {}) }, + }))); + const checked = new Set(); + await registerDeferredReceiptWork(db, coordinator, { + resume: async scope => { checked.add(scope.requestId!); }, + settleTaskChat: async () => {}, createNativeBatch: () => async () => {}, schedulingSuppressed: async () => false, + }, () => !drain).ready; + for (let pass = 0; pass < 12; pass++) { + await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId })); + await vi.advanceTimersByTimeAsync(31_000); + await Promise.allSettled([...passes]); + await immediate(); + } + expect(checked.size).toBe(ids.length); + await quiet(); + }); + it("dispatches task Retry past a repeatedly invalidated backlog into real process output", async () => { + const f = await fixture(); + // Hold executor capacity while queue time advances; reconstruct the saved + // run with fresh workers at PostgreSQL time before real process execution. + const [occupant] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agent.id, + status: "running", startedAt: new Date() }).returning(); + await db.update(agents).set({ runtimeConfig: { heartbeat: { enabled: true, intervalSec: 0, + wakeOnDemand: true, maxConcurrentRuns: 1 } } }).where(eq(agents.id, f.agent.id)); + const [closed] = await db.insert(issues).values({ companyId: f.companyId, title: "Closed task history", + status: "done", assigneeAgentId: f.agent.id }).returning(); + await db.insert(agentWakeupRequests).values(Array.from({ length: 178 }, (_, index) => ({ + id: `${(index + 1).toString(16).padStart(8, "0")}-0000-4000-8000-000000000000`, + companyId: f.companyId, agentId: f.agent.id, source: "on_demand", status: "deferred_issue_execution", + payload: { issueId: closed.id, _paperclipWakeContext: { wakeCommentIds: [randomUUID()] } }, + }))); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: f.issue.id, actorId: f.actorId, + observedRunId: f.source.id, idempotencyKey: randomUUID() }); + const read = async () => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0]; + for (let pass = 0; pass < 12 && !(await read()).runId; pass++) { + await db.transaction(tx => notifyDeferredExecutionEligibility(tx, { companyId: f.companyId })); + await vi.advanceTimersByTimeAsync(31_000); + await Promise.allSettled([...passes]); + await immediate(); + } + const receipt = await read(); + expect(receipt.runId).toBeTruthy(); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!)))[0].status).toBe("queued"); + await coordinator.stop(); + await heartbeat.drainActiveRunExecutions(); + vi.setSystemTime(vi.getRealSystemTime()); + heartbeat = heartbeatService(db); + coordinator = createCoordinator(); + await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; + await heartbeat.registerScheduledRetryCancellationWork(coordinator, () => !drain).ready; + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, occupant.id)); + await heartbeat.registerRunDispatchWork(coordinator, () => !drain).ready; + await waitForReal(async () => { + await heartbeat.drainActiveRunExecutions(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, receipt.runId!)); + expect(run.status, JSON.stringify({ error: run.error, code: run.errorCode })).toBe("succeeded"); + expect(run.stdoutExcerpt).toContain("receipt-proof"); + }, { timeout: 15_000 }); + expect(await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(sql`${heartbeatRuns.contextSnapshot}->>'taskRetryRequestId' = ${retry.requestId}`)).toHaveLength(1); + await finish(f.issue.id); + await quiet(); + }); it("dispatches an ordinary committed task Retry into saved process output", async () => { const f = await fixture(); await heartbeat.registerDeferredReceiptWork(coordinator, () => !drain).ready; diff --git a/server/src/__tests__/ephemeral-migration.test.ts b/server/src/__tests__/ephemeral-migration.test.ts index 69733b6eb8d..a3ef74bd1d2 100644 --- a/server/src/__tests__/ephemeral-migration.test.ts +++ b/server/src/__tests__/ephemeral-migration.test.ts @@ -10,8 +10,11 @@ import { randomBytes, randomUUID } from "node:crypto"; import { S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteObjectsCommand, ListObjectsV2Command, PutObjectCommand } from "@aws-sdk/client-s3"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; +import { activityLog, agents, agentFileRoots, agentInstructionWorkingCopies, companies, executionWorkspaces, executionWorkspaceSnapshots, issues, workspaceOperations, createDb, closeFileStorageDb, heartbeatRuns, providerTraceRecords } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "../services/remote-workspace-snapshot-storage.js"; +import { restoreRemoteWorkspaceArchive } from "../services/remote-workspace-snapshot-archive.js"; import { migrateEphemeralServerData } from "../maintenance/ephemeral-migration-data.js"; import { agentFileStore } from "../services/agent-file-store.js"; import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; @@ -35,6 +38,7 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll home = await fs.mkdtemp(path.join(os.tmpdir(), "ephemeral-migration-")); await s3.send(new CreateBucketCommand({ Bucket: bucket })); vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "false"); vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", process.env.PAPERCLIP_TEST_S3_ENDPOINT!); @@ -43,6 +47,10 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll }); afterEach(async () => { await db.delete(activityLog).where(eq(activityLog.companyId, companyId)); + await db.delete(workspaceOperations).where(eq(workspaceOperations.companyId, companyId)); + await db.delete(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.companyId, companyId)); + await db.delete(issues).where(eq(issues.companyId, companyId)); + await db.delete(executionWorkspaces).where(eq(executionWorkspaces.companyId, companyId)); await db.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, companyId)); await db.delete(agents).where(eq(agents.companyId, companyId)); await db.delete(companies).where(eq(companies.id, companyId)); @@ -90,6 +98,33 @@ describe.skipIf(!process.env.PAPERCLIP_TEST_S3_ENDPOINT)("existing-volume enroll await s3.send(new PutObjectCommand({ Bucket: bucket, Key: file.objectKey, Body: "corrupt" })); await expect(verifyAgentFileObjects(db, getStorageService(), AbortSignal.timeout(30_000))).rejects.toThrow(/agent_file_object_(size|digest)_mismatch/); }, 120_000); + it("migrates overlapping task folders through maintenance and restores both from S3 after volume loss", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "73".repeat(32)); + const parent = path.join(home, "project"), child = path.join(parent, ".paperclip/worktrees/task"); + await fs.mkdir(child, { recursive: true }); + const fixtures = [{ cwd: parent, file: "project.txt", bytes: "project work" }, { cwd: child, file: "task.txt", bytes: "nested task work" }]; + const ids: string[] = []; + for (const fixture of fixtures) { + await fs.writeFile(path.join(fixture.cwd, fixture.file), fixture.bytes); + const [issue] = await db.insert(issues).values({ companyId, title: fixture.file }).returning(); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId, sourceIssueId: issue!.id, + name: fixture.file, mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: fixture.cwd }).returning(); + ids.push(workspace!.id); + await db.update(issues).set({ executionWorkspaceId: workspace!.id }).where(eq(issues.id, issue!.id)); + } + expect(await migrate()).toMatchObject({ eligibility: { eligible: true }, counts: { workspaces: 2 } }); + await fs.rm(home, { recursive: true }); home = await fs.mkdtemp(path.join(os.tmpdir(), "workspace-successor-")); + vi.stubEnv("PAPERCLIP_HOME", home); + for (const [index, workspaceId] of ids.entries()) { + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const authority = readRemoteWorkspaceAuthority(workspace!.metadata)!; + const reference = await getWorkspaceSnapshot(db, { companyId, workspaceId, snapshotId: authority.snapshotId! }); + const archivePath = path.join(home, `${index}.pcws`), targetDir = path.join(home, `restored-${index}`); + await loadWorkspaceSnapshotArchive({ storage: getStorageService(), reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, fixtures[index]!.file), "utf8")).toBe(fixtures[index]!.bytes); + } + }, 120_000); const entry = fileURLToPath(new URL("../../dist-runtime/ephemeral-migration.mjs", import.meta.url)); it.skipIf(!existsSync(entry))("runs the actual bundled maintenance entry with plain Node and exits cleanly", async () => { await agent(); diff --git a/server/src/__tests__/external-object-refresh-work.test.ts b/server/src/__tests__/external-object-refresh-work.test.ts index b07e4c3cf8c..d1d9b7a0d01 100644 --- a/server/src/__tests__/external-object-refresh-work.test.ts +++ b/server/src/__tests__/external-object-refresh-work.test.ts @@ -16,6 +16,7 @@ import { instanceSettingsService } from "../services/instance-settings.js"; import { notifyExternalObjectRefresh } from "../services/external-object-refresh-notifications.js"; import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -92,7 +93,7 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { } throw new Error("refresh worker did not settle"); } - async function quiet() { await settle(() => coordinator!.nextWakeAt() === null); } + async function quiet() { await settle(() => coordinator!.nextWakeAt() === null && idleWorkSnapshot().active === 0); } function clock() { vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout", "setInterval", "clearInterval"], shouldAdvanceTime: true }); } async function change(companyId: string, patch: Partial) { await db.transaction(async tx => { @@ -601,7 +602,10 @@ const active: ExternalObjectResolveResult = { ok: true, snapshot: { await db.insert(externalObjectMentions).values(batch.map(objectId => ({ companyId: f.companyId, sourceIssueId: f.issueId, sourceKind: "description" as const, objectId }))); } - await start(svc); await settle(() => coordinator!.nextWakeAt() === future.getTime()); + await start(svc); + await vi.waitFor(() => { + expect(coordinator!.nextWakeAt()).toBe(future.getTime()); + }, { timeout: 10_000, interval: 5 }); const [incoming] = await db.insert(issues).values({ companyId: f.companyId, title: "Incoming", description: "https://example.com/new-object" }).returning(); const reads: Array<{ query: string; params: unknown[] }> = []; diff --git a/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts b/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts index 40ad08c67f4..4dc7bb99676 100644 --- a/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts +++ b/server/src/__tests__/heartbeat-undispatched-execution-wait.test.ts @@ -151,6 +151,16 @@ const support = await getEmbeddedPostgresTestSupport(); }); }); + it.each(["nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace", "nativeWorkspaceFinalizationOwner", "sessionCheckpoint"])( + "retains an admission-wait hold with a retained %s profile", async key => { + const f = await seed(true); + const [run] = await db.update(heartbeatRuns).set({ runnerProfileJson: { [key]: { retained: true } } }) + .where(eq(heartbeatRuns.id, f.runId)).returning(); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ recoveryActionId: f.actionId }); + }, + ); + it.each(["started", "runtime_selected", "process", "session", "controller", "stage", "receipt", "receipt_issue", "receipt_action", "lease", "coordinator", "launch", "provider", "workspace"])( "retains a hold with conflicting or incomplete %s evidence", async kind => { const f = await seed(true); diff --git a/server/src/__tests__/helpers/native-restart.ts b/server/src/__tests__/helpers/native-restart.ts index a05f5149f05..deccd592af1 100644 --- a/server/src/__tests__/helpers/native-restart.ts +++ b/server/src/__tests__/helpers/native-restart.ts @@ -16,6 +16,7 @@ import { prepareNativeHeartbeatRun } from "../../services/native-runtime/prepare import { createIsolatedTestHome } from "./isolated-test-home.js"; import { type NativeControllerIdentity } from "../../services/native-runtime/native-restart-recovery.js"; import { waitForPendingRunFailureReports } from "../../services/run-failure-report.js"; +import { registerLocalRuntimeApiServer } from "../../runtime-api.js"; // An explicitly supplied disposable database also permits qualification on hosts // where native Postgres cannot allocate semaphores. Never fall back silently. @@ -106,6 +107,7 @@ export function nativeRestartFixture() { : new Date(), }; server = createServer(); + registerLocalRuntimeApiServer(server); await new Promise((resolveListen) => server!.listen(0, "127.0.0.1", resolveListen), ); diff --git a/server/src/__tests__/helpers/remote-warm-controller.test-worker.ts b/server/src/__tests__/helpers/remote-warm-controller.test-worker.ts index 160be9a352e..16676581539 100644 --- a/server/src/__tests__/helpers/remote-warm-controller.test-worker.ts +++ b/server/src/__tests__/helpers/remote-warm-controller.test-worker.ts @@ -1,6 +1,8 @@ import { createDb } from "@paperclipai/db"; import { closeIdleWarmNativeSessionsForRestart, executePaperclipNativeSession } from "../../services/native-runtime/native-session-executor.js"; import { dockerCommandRunner } from "./docker-command-runner.js"; +import { createServer } from "node:http"; +import { registerLocalRuntimeApiServer } from "../../runtime-api.js"; // Separate TypeScript fixture import time from actual controller admission. process.send?.({ ready: true }); @@ -15,7 +17,10 @@ process.once("message", async (input: { }) => { const target = input.options.runnerExecutionTarget; if (target?.kind !== "remote") throw new Error("Expected remote worker target"); + const api = createServer((_request, response) => { response.writeHead(404).end(); }); + registerLocalRuntimeApiServer(api); try { + await new Promise(resolve => api.listen(0, "127.0.0.1", resolve)); const runner = dockerCommandRunner(input.container); let inspectionOverridden = false; const execute = runner.execute.bind(runner); @@ -55,6 +60,9 @@ process.once("message", async (input: { process.send?.({ result: { ...result, inspectionOverridden } }); } catch (error) { process.send?.({ error: error instanceof Error ? error.stack : String(error) }); + } finally { + api.closeAllConnections(); + await new Promise(resolve => api.close(() => resolve())); } }); setInterval(() => undefined, 60_000); diff --git a/server/src/__tests__/idle-incoming-wake.test.ts b/server/src/__tests__/idle-incoming-wake.test.ts new file mode 100644 index 00000000000..18628e4c35b --- /dev/null +++ b/server/src/__tests__/idle-incoming-wake.test.ts @@ -0,0 +1,700 @@ +import { matchesApiWakeAdmission } from "../services/idle-api-wake-admission.js"; +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; +import express from "express"; +import request from "supertest"; +import { eq, sql } from "drizzle-orm"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { + agents, + agentApiKeys, + authUsers, + boardApiKeys, + companies, + companyMemberships, + createDb, + toolApplications, + toolConnections, + connectionGrants, + chatEndpoints, + chatGitHubRegistrations, + chatSlackRegistrations, + chatEndpointLeases, + chatActions, + chatDeliveries, + chatPublications, + chatConversations, + issues, + type Db, +} from "@paperclipai/db"; +import { setCloudRuntimeConfigSnapshot } from "../services/cloud-runtime-config-snapshot.js"; +import { actorMiddleware } from "../middleware/auth.js"; +import { + idleAdmissionMiddleware, + trackIdleRequestHandlers, +} from "../middleware/idle-admission.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { companyRoutes } from "../routes/companies.js"; +import { agentService } from "../services/agents.js"; +import { boardAuthService } from "../services/board-auth.js"; +import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { + readIdleIncomingWakeProof, + incomingWakeFingerprint, + incomingWakeWorkQuery, +} from "../services/idle-incoming-wake.js"; +import { registerConnectionEventWake } from "../services/connection-event-wake.js"; +import { idleApiActivity } from "../services/idle-api-activity.js"; +import { startTaskDrain, stopTaskDrain } from "../services/task-admission.js"; +import { + verifyCloudControlAssertion, + CLOUD_CONTROL_AUDIENCE, + CLOUD_CONTROL_JWS_TYPE, + CLOUD_RUNTIME_IDENTITY_ISSUER, +} from "../services/cloud-runtime-identity.js"; +import type { + IdleIncomingWakeCapability, + VerifiedIdleCapabilities, +} from "../services/idle-incoming-wake-capability.js"; + +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "./helpers/embedded-postgres.js"; +const url = resolveExternalTestDatabaseUrl( + "PAPERCLIP_INCOMING_WAKE_TEST_DATABASE_URL", +); +const support = url + ? { supported: true } + : await getEmbeddedPostgresTestSupport(); +const ownerId = "00000000-0000-4000-8000-000000000001"; +const now = Date.now(); +const hold = () => ({ + ownerId, + draining: true, + startedAt: new Date(now - 1000), + expiresAt: new Date(now + 300_000), + activeRuns: 0, + pendingWakes: 0, +}); +const capability: IdleIncomingWakeCapability = { + version: 1, + ownerId, + generation: "a".repeat(64), + apiKeyProof: { version: 1 as const, challenge: "c".repeat(64) }, + routes: ["tenant-api-http-v2"], +}; +function verified(incomingWake = capability): VerifiedIdleCapabilities { + const pair = generateKeyPairSync("ed25519"), + kid = randomUUID(), + issued = Math.floor(Date.now() / 1000); + const encode = (v: unknown) => + Buffer.from(JSON.stringify(v)).toString("base64url"); + const header = encode({ alg: "EdDSA", typ: CLOUD_CONTROL_JWS_TYPE, kid }); + const payload = encode({ + v: 1, + iss: CLOUD_RUNTIME_IDENTITY_ISSUER, + aud: CLOUD_CONTROL_AUDIENCE, + sub: "fixture-stack", + action: "task-drain:read", + requestId: randomUUID(), + iat: issued, + exp: issued + 60, + idleCapabilities: { incomingWake }, + }); + const token = `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`), pair.privateKey).toString("base64url")}`; + return verifyCloudControlAssertion({ + compactJws: token, + expectedAction: "task-drain:read", + env: { + PAPERCLIP_CLOUD_STACK_ID: "fixture-stack", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ + keys: [ + { + ...pair.publicKey.export({ format: "jwk" }), + kid, + use: "sig", + alg: "EdDSA", + }, + ], + }), + }, + }).idleCapabilities as VerifiedIdleCapabilities; +} + +(support.supported ? describe : describe.skip)( + "scoped incoming wake PostgreSQL contract", + () => { + let db: Db; + let database: + Awaited> | undefined; + beforeAll(async () => { + if (!url) + database = await startEmbeddedPostgresTestDatabase( + "idle-incoming-wake-", + ); + db = createDb(url ?? database!.connectionString, { maxConnections: 4 }); + await db.execute(sql`SELECT 1 FROM agent_api_keys LIMIT 1`); // Coordinator migrates the fresh fixture; no embedded fallback. + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "fixture-only"); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack"); + setCloudRuntimeConfigSnapshot({ version: 1, revision: 0, config: null }); + }); + afterAll(async () => { + setCloudRuntimeConfigSnapshot(null); + stopTaskDrain(); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await db?.$client.end({ timeout: 5 }); + await database?.cleanup(); + }); + // The externally supplied database is dedicated to this suite. Preserve all + // rows during every eligibility scenario, then isolate the next scenario. + const resetFixture = async () => { + await db.execute( + sql`TRUNCATE companies, "user", chat_github_registrations CASCADE`, + ); + }; + beforeEach(resetFixture); + afterEach(async () => { + stopTaskDrain(); + vi.restoreAllMocks(); + await resetFixture(); + }); + + const read = (incomingWake?: VerifiedIdleCapabilities) => + readIdleSleepSafety( + db, + hold, + () => now, + ownerId, + async () => "none", + undefined, + undefined, + false, + false, + { incomingWake }, + ); + async function seed() { + const companyId = randomUUID(), + agentId = randomUUID(), + userId = randomUUID(), + foreignId = randomUUID(); + await db.insert(companies).values([ + { + id: companyId, + name: "Incoming fixture", + issuePrefix: `I${companyId.slice(0, 6)}`, + }, + { + id: foreignId, + name: "Foreign fixture", + issuePrefix: `F${foreignId.slice(0, 6)}`, + }, + ]); + await db + .insert(authUsers) + .values({ + id: userId, + name: "Fixture owner", + email: `${userId}@example.test`, + createdAt: new Date(), + updatedAt: new Date(), + }); + await db + .insert(companyMemberships) + .values({ + companyId, + principalType: "user", + principalId: userId, + membershipRole: "owner", + status: "active", + }); + await db + .insert(agents) + .values({ + id: agentId, + companyId, + name: "Fixture agent", + role: "engineer", + adapterType: "process", + status: "idle", + }); + const board = await boardAuthService(db).createNamedBoardApiKey({ + userId, + name: "incoming board", + expiresAt: null, + }); + const agent = await agentService(db).createApiKey( + agentId, + "incoming agent", + { kind: "standard" }, + { responsibleUserId: userId }, + ); + return { companyId, foreignId, userId, agentId, board, agent }; + } + it("retains real saved keys and exempts them only for the verified exact owner/profile", async () => { + const f = await seed(); + expect((await read()).backgroundWork).toBe("present"); + expect(await read(verified())).toMatchObject({ + version: 3, + backgroundWork: "none", + incomingWake: capability, + }); + const receipt = (await read(verified())).apiWakeAdmission!; + expect(receipt.keys).toHaveLength(2); + expect(matchesApiWakeAdmission(receipt, { stackId: "fixture-stack", token: f.board.token, now })).toBe(true); + expect(matchesApiWakeAdmission(receipt, { stackId: "fixture-stack", token: f.agent.token, now })).toBe(true); + const serialized = JSON.stringify(receipt); + expect(serialized).not.toContain(f.board.token); + expect(serialized).not.toContain(f.agent.token); + await db.update(boardApiKeys).set({ revokedAt: new Date() }).where(eq(boardApiKeys.id, f.board.id)); + const changedReceipt = (await read(verified())).apiWakeAdmission!; + expect(changedReceipt.generation).not.toBe(receipt.generation); + expect(matchesApiWakeAdmission(changedReceipt, { stackId: "fixture-stack", token: f.board.token, now })).toBe(false); + expect( + (await read(verified({ ...capability, ownerId: randomUUID() }))) + .backgroundWork, + ).toBe("unknown"); + expect( + await db + .select() + .from(agentApiKeys) + .where(eq(agentApiKeys.id, f.agent.id)), + ).toHaveLength(1); + expect( + await db + .select() + .from(boardApiKeys) + .where(eq(boardApiKeys.id, f.board.id)), + ).toHaveLength(1); + }); + it("delivers through actual tenant auth/routes, denies current revoked/expired/foreign authority and holds only successful API use", async () => { + const f = await seed(); + let monotonic = 1; + vi.spyOn(performance, "now").mockImplementation(() => monotonic); + let release: (() => void) | undefined, arrived: (() => void) | undefined; + const app = express(); + // Named Cloud wake boundary: preserve request until the simulated compute handoff finishes. + app.use(async (_req, _res, next) => { + if (release) { + await new Promise((resolve) => { + release = resolve; + arrived?.(); + }); + } + next(); + }); + app.use(idleAdmissionMiddleware); + app.use(express.json()); + app.use( + actorMiddleware(db, { + deploymentMode: "authenticated", + resolveSession: async () => null, + }), + ); + app.use("/api/companies", companyRoutes(db)); + app.use(errorHandler); + trackIdleRequestHandlers(app); + expect(idleApiActivity.active()).toBe(false); + expect( + ( + await request(app) + .get(`/api/companies/${f.companyId}`) + .set("Authorization", "Bearer unknown") + ).status, + ).toBe(401); + expect(idleApiActivity.active()).toBe(false); + expect( + ( + await request(app) + .get(`/api/companies/${f.foreignId}`) + .set("Authorization", `Bearer ${f.board.token}`) + ).status, + ).toBe(403); + expect(idleApiActivity.active()).toBe(false); + const own = await request(app) + .patch(`/api/companies/${f.companyId}`) + .set("Authorization", `Bearer ${f.board.token}`) + .send({ name: "Persisted after wake" }); + expect(own.status).toBe(200); + expect(idleApiActivity.active()).toBe(true); + expect( + ( + await db.select().from(companies).where(eq(companies.id, f.companyId)) + )[0]!.name, + ).toBe("Persisted after wake"); + monotonic += 300_000; + expect( + ( + await request(app) + .get(`/api/companies/${f.companyId}`) + .set("Authorization", `Bearer ${f.agent.token}`) + ).status, + ).toBe(200); + monotonic += 300_001; + expect(idleApiActivity.active()).toBe(true); + monotonic += 299_999; + expect(idleApiActivity.active()).toBe(false); + const arrival = new Promise((resolve) => { + arrived = resolve; + }); + release = () => {}; + const held = request(app) + .patch(`/api/companies/${f.companyId}`) + .set("Authorization", `Bearer ${f.board.token}`) + .send({ name: "Must not commit" }) + .then((r) => r); + await arrival; + await db + .update(boardApiKeys) + .set({ revokedAt: new Date() }) + .where(eq(boardApiKeys.id, f.board.id)); + release!(); + release = undefined; + expect((await held).status).toBe(401); + expect(idleApiActivity.active()).toBe(false); + const expired = await boardAuthService(db).createNamedBoardApiKey({ + userId: f.userId, + name: "expired", + expiresAt: new Date(0), + }); + expect( + ( + await request(app) + .get(`/api/companies/${f.companyId}`) + .set("Authorization", `Bearer ${expired.token}`) + ).status, + ).toBe(401); + expect(idleApiActivity.active()).toBe(false); + startTaskDrain({ purpose: "idle", ttlMs: 60_000 }); + expect( + ( + await request(app) + .patch(`/api/companies/${f.companyId}`) + .set("Authorization", `Bearer ${f.agent.token}`) + .send({ name: "Draining" }) + ).status, + ).toBe(503); + expect(idleApiActivity.active()).toBe(false); + stopTaskDrain(); + expect( + ( + await db.select().from(companies).where(eq(companies.id, f.companyId)) + )[0]!.name, + ).toBe("Persisted after wake"); + }); + // Fault injection below supplies only malformed/incomplete persisted states; + // the positive archived transition is covered by the real configure(remove) + // producer in chat-channels.integration.test.ts, without deleting history. + async function removedGitHubFixture() { + const f = await seed(), applicationId = randomUUID(), connectionId = randomUUID(), endpointId = randomUUID(); + await db.insert(toolApplications).values({ id: applicationId, companyId: f.companyId, name: "Removed GitHub", type: "chat" }); + await db.insert(toolConnections).values({ id: connectionId, companyId: f.companyId, applicationId, + name: "Removed GitHub", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel", + status: "archived", enabled: false, credentialSecretRefs: [] }); + await db.insert(chatEndpoints).values({ id: endpointId, companyId: f.companyId, connectionId, + assignedAgentId: f.agentId, provider: "github", publicId: randomUUID(), status: "archived", + archivedAt: new Date(), setup: { step: "complete", runtimeGeneration: 1 } }); + const blocked = async () => (await db.execute(sql`SELECT EXISTS ( + ${incomingWakeWorkQuery("chat_endpoints")} AND endpoint.id = ${endpointId}::uuid) AS blocked`))[0].blocked; + return { ...f, applicationId, connectionId, endpointId, blocked }; + } + it.each(["stamp", "active", "paused", "revoked", "persistent_provider", "enabled_connection", "active_connection", + "non_sdk_connection", "credentials", "malformed_credentials", "malformed_setup", "missing_generation", "fractional_generation", + "lease", "expired_lease", "pending_registration", "unconsumed_registration", "wrong_registry"])( + "keeps incomplete removed GitHub proof blocked: %s", async fault => { + const f = await removedGitHubFixture(); + if (fault === "stamp") await db.update(chatEndpoints).set({ archivedAt: null }).where(eq(chatEndpoints.id, f.endpointId)); + if (["active", "paused", "revoked"].includes(fault)) await db.update(chatEndpoints) + .set({ status: fault as "active" | "paused" | "revoked" }).where(eq(chatEndpoints.id, f.endpointId)); + if (fault === "persistent_provider") await db.update(chatEndpoints).set({ provider: "discord" }).where(eq(chatEndpoints.id, f.endpointId)); + if (fault === "enabled_connection") await db.update(toolConnections).set({ enabled: true }).where(eq(toolConnections.id, f.connectionId)); + if (fault === "active_connection") await db.update(toolConnections).set({ status: "active" }).where(eq(toolConnections.id, f.connectionId)); + if (fault === "non_sdk_connection") await db.update(toolConnections).set({ transport: "voice" }).where(eq(toolConnections.id, f.connectionId)); + if (fault === "credentials" || fault === "malformed_credentials") await db.execute(sql`UPDATE tool_connections + SET credential_secret_refs = ${fault === "credentials" ? '[{"secretId":"retained"}]' : '{}'}::jsonb WHERE id = ${f.connectionId}::uuid`); + if (fault === "malformed_setup") await db.execute(sql`UPDATE chat_endpoints SET setup = '"unknown"'::jsonb WHERE id = ${f.endpointId}::uuid`); + if (fault === "missing_generation" || fault === "fractional_generation") await db.update(chatEndpoints) + .set({ setup: fault === "missing_generation" ? { step: "complete" } : { step: "complete", runtimeGeneration: 1.5 } }) + .where(eq(chatEndpoints.id, f.endpointId)); + if (fault === "lease" || fault === "expired_lease") await db.insert(chatEndpointLeases).values({ companyId: f.companyId, + endpointId: f.endpointId, leaseKey: "credentials", token: randomUUID(), expiresAt: new Date(fault === "lease" ? Date.now() + 60_000 : 0) }); + if (fault === "pending_registration" || fault === "unconsumed_registration") await db.insert(chatGitHubRegistrations).values({ + companyId: f.companyId, endpointId: f.endpointId, userId: f.userId, stateHash: randomUUID(), trustedOrigin: "https://tenant.example.test", + status: fault === "pending_registration" ? "pending" : "completed", expiresAt: new Date(Date.now() + 60_000) }); + if (fault === "wrong_registry") await db.insert(chatSlackRegistrations).values({ endpointId: f.endpointId, companyId: f.companyId, + requestId: randomUUID(), status: "uncertain", manifest: {}, manifestHash: "a".repeat(64), callbackUri: "https://tenant.example.test", + createdByUserId: f.userId, secretIds: { clientSecret: randomUUID() } }); + expect(await f.blocked()).toBe(true); + expect(await db.select({ id: chatEndpoints.id }).from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId))).toHaveLength(1); + }); + it("retains tenant boundaries and rejects unfamiliar archived state at the schema boundary", async () => { + const f = await removedGitHubFixture(); + await expect(db.execute(sql`UPDATE chat_endpoints SET connection_id = ${randomUUID()}::uuid WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow(); + const foreignApplication = randomUUID(), foreignConnection = randomUUID(); + await db.insert(toolApplications).values({ id: foreignApplication, companyId: f.foreignId, name: "Foreign", type: "chat" }); + await db.insert(toolConnections).values({ id: foreignConnection, companyId: f.foreignId, applicationId: foreignApplication, + name: "Foreign", uid: randomUUID(), transport: "chat_sdk", connectionPurpose: "channel", status: "archived", enabled: false }); + await expect(db.execute(sql`UPDATE chat_endpoints SET connection_id = ${foreignConnection}::uuid WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow(); + await expect(db.execute(sql`UPDATE chat_endpoints SET status = 'unknown' WHERE id = ${f.endpointId}::uuid`)).rejects.toThrow(); + expect((await db.select().from(chatEndpoints).where(eq(chatEndpoints.id, f.endpointId)))[0].connectionId).toBe(f.connectionId); + }); + it("does not let removed GitHub endpoint history discharge pending delivery, publication or action obligations", async () => { + const f = await removedGitHubFixture(); + await db.insert(chatGitHubRegistrations).values({ companyId: f.companyId, endpointId: f.endpointId, userId: f.userId, + stateHash: randomUUID(), trustedOrigin: "https://tenant.example.test", status: "completed", consumedAt: new Date(), expiresAt: new Date(0) }); + expect(await f.blocked()).toBe(false); + const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Retained chat debt", status: "done" }).returning(); + const [conversation] = await db.insert(chatConversations).values({ companyId: f.companyId, endpointId: f.endpointId, + issueId: issue.id, externalConversationId: "github:retained", externalLabel: "retained", state: "endpoint_removed" }).returning(); + await db.insert(chatDeliveries).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id, + providerEventId: "retained", deduplicationKey: "retained", eventKind: "message", normalizedEvent: {}, state: "retry", nextAttemptAt: new Date() }); + await db.insert(chatPublications).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id, + issueId: issue.id, idempotencyKey: "retained", payload: { text: "Retained" }, state: "retry", nextAttemptAt: new Date() }); + await db.insert(chatActions).values({ companyId: f.companyId, endpointId: f.endpointId, conversationId: conversation.id, + kind: "provider_effect", providerActionId: "retained", payload: {}, status: "received" }); + const report = await readIdleSleepSafety(db, hold, () => now, ownerId, async () => "none", undefined, undefined, + false, false, { incomingWake: verified(), collectDiagnostics: true }); + expect(report.backgroundWork).toBe("present"); + expect(report.diagnostics?.checks).toEqual(expect.arrayContaining([ + { reason: "chat_endpoints", state: "none" }, { reason: "chat_deliveries", state: "present" }, + { reason: "chat_publications", state: "present" }, { reason: "chat_actions", state: "present" }, + ])); + expect(await db.select().from(chatDeliveries)).toHaveLength(1); + expect(await db.select().from(chatPublications)).toHaveLength(1); + expect(await db.select().from(chatActions)).toHaveLength(1); + }); + + it("requires current broker instance and exact owner binding, preserving continuity and unsupported transport blockers", async () => { + const f = await seed(); + const applicationId = randomUUID(), + connectionId = randomUUID(), + grantId = randomUUID(); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1"); + const botConnectionId = randomUUID(), + endpointId = randomUUID(), + registration = "b".repeat(43); + const botApplication = randomUUID(); + await db + .insert(toolApplications) + .values({ + id: botApplication, + companyId: f.companyId, + name: "GitHub bot", + type: "chat", + }); + await db + .insert(toolConnections) + .values({ + id: botConnectionId, + companyId: f.companyId, + applicationId: botApplication, + name: "GitHub bot", + uid: randomUUID(), + transport: "chat_sdk", + connectionPurpose: "channel", + status: "active", + enabled: true, + config: { provider: "github" }, + }); + await db + .insert(chatEndpoints) + .values({ + id: endpointId, + companyId: f.companyId, + connectionId: botConnectionId, + assignedAgentId: f.agentId, + provider: "github", + publicId: "p".repeat(43), + status: "active", + activatedAt: new Date(), + providerAccountId: "fixture-app", + botExternalId: "fixture-bot", + setup: { + step: "complete", + github: { + stage: "test", + registrationStatus: "completed", + cloudRegistrationId: registration, + }, + }, + }); + await db + .insert(chatGitHubRegistrations) + .values({ + companyId: f.companyId, + endpointId, + userId: f.userId, + stateHash: randomUUID(), + trustedOrigin: "https://tenant.example.test", + status: "completed", + expiresAt: new Date(0), + handoff: { + cloudId: registration, + redemptionId: randomUUID(), + returnState: randomUUID(), + }, + }); + const botTuple = [ + `github-app:${registration}`, + f.companyId, + f.userId, + "github.bot", + `gateway:${registration}`, + registration, + registration, + ]; + const stop = registerConnectionEventWake(db, () => {}); + const botCap: IdleIncomingWakeCapability = { + ...capability, + routes: ["github-broker-events-v1", "tenant-api-http-v2"], + githubBroker: { + instanceId: "fixture-instance", + bindingsFingerprint: incomingWakeFingerprint([botTuple]), + }, + }; + expect((await read(verified())).backgroundWork).toBe("present"); + expect(await read(verified(botCap))).toMatchObject({ + version: 3, + backgroundWork: "none", + }); + const botProof = await readIdleIncomingWakeProof(db, { + capabilities: verified(botCap), + ownerId, + receiverOwner: db, + }); + expect(botProof?.endpointIds).toEqual([endpointId]); + expect(botProof?.connectionIds).toEqual([botConnectionId]); + expect( + await db + .select() + .from(chatEndpoints) + .where(eq(chatEndpoints.id, endpointId)), + ).toHaveLength(1); + + await db + .insert(toolApplications) + .values({ + id: applicationId, + companyId: f.companyId, + name: "GitHub", + type: "mcp_http", + }); + await db + .insert(toolConnections) + .values({ + id: connectionId, + companyId: f.companyId, + applicationId, + name: "GitHub", + uid: randomUUID(), + transport: "mcp_remote", + connectionPurpose: "tool", + status: "active", + enabled: true, + config: { + sourceTemplateKey: "github", + oauth: { connectorProfile: "github.code" }, + }, + }); + const github = { + userId: "42", + login: "fixture", + installationCount: 1, + repositoryCount: 1, + repositorySelection: "all" as const, + installationIds: ["123"], + installationOwnerLogins: ["fixture"], + lastAccessRefreshAt: new Date().toISOString(), + }; + await db + .insert(connectionGrants) + .values({ + id: grantId, + companyId: f.companyId, + connectionId, + kind: "user", + subjectUserId: f.userId, + providerTenant: { github }, + status: "active", + }); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1"); + const brokerCap: IdleIncomingWakeCapability = { + ...capability, + routes: ["github-broker-events-v1", "tenant-api-http-v2"], + githubBroker: { + instanceId: "fixture-instance", + bindingsFingerprint: incomingWakeFingerprint([ + botTuple, + [ + `${grantId}_123`, + f.companyId, + f.userId, + "github.code", + "123", + connectionId, + grantId, + ], + ]), + }, + }; + try { + const proof = await readIdleIncomingWakeProof(db, { + capabilities: verified(brokerCap), + ownerId, + receiverOwner: db, + }); + expect(proof?.requiredContinuityConnectionIds).toEqual([connectionId]); + expect(proof?.connectionIds).toEqual([botConnectionId]); + expect((await read(verified(brokerCap))).backgroundWork).toBe( + "present", + ); // transport alone never discharges real credential deadline + await db + .update(connectionGrants) + .set({ + providerTenant: { + github: { + ...github, + bindingSync: { id: randomUUID(), removeInstallationIds: [] }, + }, + }, + }) + .where(eq(connectionGrants.id, grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe( + "unknown", + ); + await db + .update(connectionGrants) + .set({ status: "revoked" }) + .where(eq(connectionGrants.id, grantId)); + expect((await read(verified(brokerCap))).backgroundWork).toBe( + "unknown", + ); + await db + .update(toolConnections) + .set({ transport: "local_stdio" }) + .where(eq(toolConnections.id, connectionId)); + expect((await read(verified())).backgroundWork).toBe("present"); + } finally { + stop(); + } + }); + }, +); diff --git a/server/src/__tests__/inbox-mine-routes.test.ts b/server/src/__tests__/inbox-mine-routes.test.ts index 527e661c42f..5a029b4ada8 100644 --- a/server/src/__tests__/inbox-mine-routes.test.ts +++ b/server/src/__tests__/inbox-mine-routes.test.ts @@ -285,11 +285,14 @@ describeEmbeddedPostgres("Mine inbox snapshot routes", () => { const responses = await Promise.all(requests); expect(responses.every(response => response.status === 200)).toBe(true); if (coalesced) expect((await coalesced).headers["x-paperclip-request-cache"]).toBe("coalesced"); - await request(app).get(snapshotPath).expect(200); - const computesBeforeFreshRead = started.mock.calls.length; - const fresh = await request(app).get(snapshotPath).expect(200); + // The aborted snapshot can outlive the seven ordinary reads. A coalesced + // response waits for that compute to settle before we require a fresh one. + const settled = await request(app).get(snapshotPath).expect(200); + const fresh = settled.headers["x-paperclip-request-cache"] === "coalesced" + ? await request(app).get(snapshotPath).expect(200) + : settled; expect(fresh.headers["x-paperclip-request-cache"]).toBe("miss"); - expect(started).toHaveBeenCalledTimes(computesBeforeFreshRead + 1); + expect(started).toHaveBeenCalledTimes(9); } finally { release(); await aborted; diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index 08123e23a55..eb9fb0cb38e 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -1,3 +1,4 @@ +import { generateKeyPairSync, randomUUID, sign } from "node:crypto"; import express from "express"; import request from "supertest"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -81,21 +82,24 @@ function localAdminActor() { describe("instance settings routes", () => { const routeModules = hoistModuleGraph(registerModuleMocks, async () => { - const [{ errorHandler }, { instanceSettingsRoutes }] = await Promise.all([ + const [{ errorHandler }, { instanceSettingsRoutes }, { cloudControlMiddleware }, controlIdentity] = await Promise.all([ vi.importActual("../middleware/index.js"), vi.importActual("../routes/instance-settings.js"), + vi.importActual("../middleware/cloud-control.js"), + vi.importActual("../services/cloud-runtime-identity.js"), ]); - return { errorHandler, instanceSettingsRoutes }; + return { errorHandler, instanceSettingsRoutes, cloudControlMiddleware, controlIdentity }; }); - function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise) { - const { errorHandler, instanceSettingsRoutes } = routeModules.value; + function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise, verifyControl = false) { + const { errorHandler, instanceSettingsRoutes, cloudControlMiddleware } = routeModules.value; const app = express(); app.use(express.json()); app.use((req, _res, next) => { req.actor = actor; next(); }); + if (verifyControl) app.use(cloudControlMiddleware()); app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup)); app.use(errorHandler); return app; @@ -1101,7 +1105,7 @@ describe("instance settings routes", () => { await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200); expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 }); await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200); - expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false); + expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined }); }); it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => { @@ -1128,7 +1132,7 @@ describe("instance settings routes", () => { const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1"); expect(res.status).toBe(200); expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report }); - expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined }); expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus); }); @@ -1142,7 +1146,7 @@ describe("instance settings routes", () => { .get("/api/instance/task-drain?idleSleepSafety=1&idleSleepDiagnostics=1&ownerId=fixture-owner").expect(200); expect(response.body.idleSleepSafety.diagnostics.checks).toEqual([{ reason: "agent_api_keys", state: "present" }]); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, undefined, false, false, { collectDiagnostics: true }); + "fixture-owner", expect.any(Function), undefined, undefined, false, false, { incomingWake: undefined, deadlineCapability: undefined, collectDiagnostics: true }); mockReadIdleSleepSafety.mockClear(); for (const actor of [nonAdminActor, { type: "agent", agentId: "agent-1", companyId: "company-1", source: "api_key" }]) { await request(createApp(actor)) @@ -1164,7 +1168,93 @@ describe("instance settings routes", () => { await request(createApp({ ...adminActor, source: "cloud_control" })) .get(`/api/instance/task-drain?idleSleepSafety=1&scheduledWake=${version}&ownerId=fixture-owner`).expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, undefined, true, version === "2"); + "fixture-owner", expect.any(Function), undefined, undefined, true, version === "2", { incomingWake: undefined, deadlineCapability: undefined }); + }); + + describe("signed incoming capability handoff", () => { + const createControlApp = (actor: any) => createApp(actor, undefined, undefined, true); + const keyPair = generateKeyPairSync("ed25519"); + const publicJwk = { ...keyPair.publicKey.export({ format: "jwk" }), kid: "idle-route-test", alg: "EdDSA", use: "sig" }; + const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895"; + const catalog = { + incomingWake: { version: 1, ownerId, generation: "a".repeat(64), apiKeyProof: { version: 1, challenge: "c".repeat(64) }, routes: ["tenant-api-http-v2"] }, + deadlineCatalog: "domain_deadlines_v1", + }; + const path = `/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`; + function assertion(claims: Record = {}) { + const identity = routeModules.value.controlIdentity; + const encode = (value: unknown) => Buffer.from(JSON.stringify(value)).toString("base64url"); + const iat = Math.floor(Date.now() / 1000); + const header = encode({ alg: "EdDSA", typ: identity.CLOUD_CONTROL_JWS_TYPE, kid: publicJwk.kid }); + const payload = encode({ v: 1, iss: identity.CLOUD_RUNTIME_IDENTITY_ISSUER, aud: identity.CLOUD_CONTROL_AUDIENCE, + sub: "stack-idle-route-test", action: "task-drain:read", requestId: randomUUID(), iat, exp: iat + 60, + idleCapabilities: catalog, ...claims }); + return `${header}.${payload}.${sign(null, Buffer.from(`${header}.${payload}`, "ascii"), keyPair.privateKey).toString("base64url")}`; + } + beforeEach(() => { + vi.stubEnv("PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS", JSON.stringify({ keys: [publicJwk] })); + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "stack-idle-route-test"); + routeModules.value.controlIdentity.resetCloudControlReplayFenceForTests(); + mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); + mockReadIdleSleepSafety.mockResolvedValue({ version: 3, backgroundWork: "none" }); + }); + afterEach(() => vi.unstubAllEnvs()); + + it("passes only the immutable signed catalog through real middleware and the production route", async () => { + const token = assertion(); + const response = await request(createControlApp({ type: "none" })) + .get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify({ ...catalog.incomingWake, generation: "b".repeat(64) }))}`) + .set("x-paperclip-cloud-control", token) + .send({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, ownerId: randomUUID() } } }).expect(200); + expect(response.body.idleSleepSafety).toEqual({ version: 3, backgroundWork: "none" }); + expect(mockReadIdleSleepSafety).toHaveBeenCalledExactlyOnceWith(mockDb, expect.any(Function), Date.now, + ownerId, expect.any(Function), undefined, undefined, false, false, + { incomingWake: catalog, deadlineCapability: catalog }); + const options = mockReadIdleSleepSafety.mock.calls[0][9]; + expect(options.incomingWake).toBe(options.deadlineCapability); + expect(Object.isFrozen(options.incomingWake)).toBe(true); + expect(Object.isFrozen(options.incomingWake.incomingWake)).toBe(true); + expect(Object.isFrozen(options.incomingWake.incomingWake.routes)).toBe(true); + await request(createControlApp({ type: "none" })).get(path).set("x-paperclip-cloud-control", token).expect(401); + expect(mockReadIdleSleepSafety).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["wrong stack", () => ({ sub: "other-stack" })], + ["wrong action", () => ({ action: "task-drain:start" })], + ["expired", () => ({ iat: Math.floor(Date.now() / 1000) - 600, exp: Math.floor(Date.now() / 1000) - 300 })], + ["unsupported catalog", () => ({ idleCapabilities: { incomingWake: { ...catalog.incomingWake, routes: ["unverified-path"] } } })], + ])("rejects %s before safety inspection", async (_name, claims) => { + await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", assertion(claims())).expect(401); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); + + it("rejects a tampered catalog signature before safety inspection", async () => { + const [header, payload, signature] = assertion().split("."); + const changed = JSON.parse(Buffer.from(payload!, "base64url").toString()); + changed.idleCapabilities.incomingWake.generation = "b".repeat(64); + const tampered = `${header}.${Buffer.from(JSON.stringify(changed)).toString("base64url")}.${signature}`; + await request(createControlApp(adminActor)).get(path).set("x-paperclip-cloud-control", tampered).expect(401); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); + + it("leaves legacy signed reads without a catalog conservative", async () => { + await request(createControlApp({ type: "none" })).get(path) + .set("x-paperclip-cloud-control", assertion({ idleCapabilities: undefined })).expect(200); + expect(mockReadIdleSleepSafety.mock.calls[0][9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined }); + }); + + it("cannot install capabilities from tenant, actor, query or body values", async () => { + for (const source of ["session", "cloud_tenant", "local_implicit", "cloud_control"]) { + const actor = { ...adminActor, source, idleCapabilities: catalog, incomingWake: catalog }; + await request(createControlApp(actor)).get(`${path}&incomingWake=${encodeURIComponent(JSON.stringify(catalog))}`) + .send({ idleCapabilities: catalog }).expect(200); + expect(mockReadIdleSleepSafety.mock.lastCall?.[9]).toEqual({ incomingWake: undefined, deadlineCapability: undefined }); + } + mockReadIdleSleepSafety.mockClear(); + await request(createControlApp(nonAdminActor)).get(path).send({ idleCapabilities: catalog }).expect(403); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + }); }); it("uses the live worker manager for inspection and releases only the owned hold", async () => { @@ -1199,7 +1289,7 @@ describe("instance settings routes", () => { expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, backup, false, false); + "fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined }); mockReadIdleSleepSafety.mockClear(); await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403); expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); @@ -1221,7 +1311,7 @@ describe("instance settings routes", () => { const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup); await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, - "fixture-owner", expect.any(Function), undefined, backup, false, false); + "fixture-owner", expect.any(Function), undefined, backup, false, false, { incomingWake: undefined, deadlineCapability: undefined }); } finally { delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; } diff --git a/server/src/__tests__/legacy-continuation-authority.test.ts b/server/src/__tests__/legacy-continuation-authority.test.ts index 9d8bbd35dc9..2581f087528 100644 --- a/server/src/__tests__/legacy-continuation-authority.test.ts +++ b/server/src/__tests__/legacy-continuation-authority.test.ts @@ -1,10 +1,13 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, agentWakeupRequests, companies, createDb, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db"; +import { agents, agentWakeupRequests, companies, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db"; import { heartbeatService } from "../services/heartbeat.js"; import { recoveryService } from "../services/recovery/service.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createPostgresRunDispatchAdapter } from "../modules/run-dispatch/adapters/postgres.js"; +import { legacyExecutionNeedsReconciliationWithEvidence, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; +import { getExecutionBlocker } from "../services/execution-blocker.js"; describe("legacy continuation persisted authority", () => { let temporary: Awaited>; @@ -33,6 +36,103 @@ describe("legacy continuation persisted authority", () => { } return { companyId, agentId, issueId, runId, createRecovery, runs, actions, finish }; } + it("does not freeze a never-started dispatch cancellation or retain its historical hold", async () => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ status: "queued", resultJson: null }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(issues).set({ assigneeAgentId: null }).where(eq(issues.id, f.issueId)); + expect(await createPostgresRunDispatchAdapter(db).cancelStaleQueuedRun({ companyId: f.companyId, + runId: f.runId, expectedStatus: "queued", now: new Date() })).toMatchObject({ outcome: "cancelled", errorCode: "issue_assignee_changed" }); + const run = (await f.runs())[0]; + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(false); + await terminalizeLegacyExecution({ db, run, status: "cancelled", fromStatuses: ["cancelled"], reconcileIfNeeded: true }); + expect(await f.actions()).toHaveLength(0); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board", + fingerprint: `legacy-execution:${run.id}`, status: "resolved", outcome: "blocked", nextAction: "Reconcile", + evidence: { runId: run.id, automaticRecovery: { replay: "blocked" } } }); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + for (const errorCode of ["issue_not_found", "issue_not_in_progress", "issue_waiting_for_response", + "issue_dependencies_blocked", "issue_blocked", + "issue_execution_lock_changed", "issue_terminal_status", "issue_review_participant_changed"]) { + const changed = { ...run, errorCode }; + await db.update(heartbeatRuns).set(changed).where(eq(heartbeatRuns.id, run.id)); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, changed)).toBe(false); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + } + const reviewWait = { ...run, errorCode: "issue_continuation_waiting_on_review", resultJson: { + stopReason: "issue_continuation_waiting_on_review", timeoutSource: "stale_queued_run_gate", + } }; + await db.update(issues).set({ assigneeAgentId: f.agentId }).where(eq(issues.id, f.issueId)); + await db.update(heartbeatRuns).set(reviewWait).where(eq(heartbeatRuns.id, run.id)); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, reviewWait)).toBe(false); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ canRetry: true }); + for (const patch of [{ finishedAt: null }, { startedAt: new Date() }, { runtimeModeResolvedAt: new Date() }, + { processPid: 123456789 }, { processGroupId: 123456789 }, { processStartedAt: new Date() }, + { sessionIdAfter: "provider-session" }, { controllerBootId: randomUUID() }, + { controllerLeaseExpiresAt: new Date() }, { executionStage: "adapter_running" }, { errorCode: "cancelled" }, + { resultJson: { workspaceRestoreFailure: "restore_unsafe_archive" } }, + { resultJson: { workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1" } } }, + ...["nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace", "nativeWorkspaceFinalizationOwner", "sessionCheckpoint"] + .map(key => ({ runnerProfileJson: { [key]: { retained: true } } }))]) { + const changed = { ...run, ...patch }; + await db.update(heartbeatRuns).set(changed).where(eq(heartbeatRuns.id, run.id)); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, changed)).toBe(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + } + await db.update(heartbeatRuns).set(run).where(eq(heartbeatRuns.id, run.id)); + await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: run.id, agentId: f.agentId, + seq: 2, eventType: "adapter.invoke" }); + await db.update(heartbeatRuns).set({ nextEventSeq: 3 }).where(eq(heartbeatRuns.id, run.id)); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + }); + it("keeps historical holds for missing, foreign, or conflicting sources and workspace recovery", async () => { + const f = await fixture(), foreign = await fixture(); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, foreign.issueId)); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + errorCode: "issue_dependencies_blocked", resultJson: null }).where(eq(heartbeatRuns.id, f.runId)); + const evidence = { runId: f.runId, automaticRecovery: { replay: "blocked" } }; + const [hold] = await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board", + fingerprint: `legacy-execution:${f.runId}`, status: "resolved", outcome: "blocked", nextAction: "Reconcile", evidence }).returning(); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + for (const changed of [{ ...evidence, runId: "malformed" }, { ...evidence, runId: randomUUID() }, + { ...evidence, runId: foreign.runId }, { automaticRecovery: evidence.automaticRecovery }, + { ...evidence, workspaceRestoreFailure: "restore_unsafe_archive" }, + { ...evidence, workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1" } }]) { + await db.update(issueRecoveryActions).set({ evidence: changed }).where(eq(issueRecoveryActions.id, hold.id)); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + } + await db.update(issueRecoveryActions).set({ evidence: { sourceRunId: f.runId, + automaticRecovery: evidence.automaticRecovery } }).where(eq(issueRecoveryActions.id, hold.id)); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + const [other] = await db.insert(issues).values({ companyId: f.companyId, title: "Another task" }).returning(); + await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: other.id } }).where(eq(heartbeatRuns.id, f.runId)); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + await db.update(heartbeatRuns).set({ issueId: f.issueId, + contextSnapshot: { issueId: f.issueId } }).where(eq(heartbeatRuns.id, f.runId)); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + expect(await f.actions()).toHaveLength(1); + }); + it.each(["released environment", "provider event"])("retains a hold with a recorded %s", async retained => { + const f = await fixture(); + const [run] = await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date(), + errorCode: "issue_dependencies_blocked", resultJson: null }).where(eq(heartbeatRuns.id, f.runId)).returning(); + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", ownerType: "board", + fingerprint: `legacy-execution:${f.runId}`, status: "active", nextAction: "Reconcile", evidence: { runId: f.runId } }); + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(false); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toBeNull(); + if (retained === "released environment") await db.insert(environmentLeases).values({ companyId: f.companyId, + heartbeatRunId: f.runId, issueId: f.issueId, provider: "local", status: "released", releasedAt: new Date(), cleanupStatus: "success" }); + else { + await db.insert(heartbeatRunEvents).values({ companyId: f.companyId, runId: f.runId, agentId: f.agentId, + seq: 1, eventType: "provider.diagnostic", sourceEventId: "provider-event-1" }); + await db.update(heartbeatRuns).set({ nextEventSeq: 2 }).where(eq(heartbeatRuns.id, f.runId)); + } + expect(await legacyExecutionNeedsReconciliationWithEvidence(db, run)).toBe(true); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).not.toBeNull(); + }); it.each([1, 2, 3, 4, 5])("deduplicates concurrent replay with stale prose-derived liveness (%s)", async () => { const f = await fixture(); const recovery = f.createRecovery(); diff --git a/server/src/__tests__/live-events-toast.test.ts b/server/src/__tests__/live-events-toast.test.ts index e82af433d73..ffa96702e50 100644 --- a/server/src/__tests__/live-events-toast.test.ts +++ b/server/src/__tests__/live-events-toast.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it, vi } from "vitest"; import { setupLiveEventsWebSocketServer } from "../realtime/live-events-ws.js"; import { publishLiveEvent } from "../services/live-events.js"; import { taskToastEligible } from "../realtime/task-toast-eligibility.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; vi.mock("../realtime/task-toast-eligibility.js", () => ({ taskToastEligible: vi.fn(async () => false) })); @@ -29,7 +30,7 @@ function database(onPredicate: (kind: "where" | "join", condition: SQL) => void innerJoin: (_table: unknown, condition: SQL) => { onPredicate("join", condition); return query; }, where: (condition: SQL) => { onPredicate("where", condition); return query; }, limit: async () => { - if (table === "issues") return [{ id: "task" }]; + if (table === "issues") return [{ id: "task", identifier: "PAP-1234" }]; if (table === "company_memberships") return fields.name ? [{ name: "Alex Example", image: "/alex.png" }] : [{ id: "membership" }]; if (table === "agents") return [{ name: "Coder", appearance: { schemaVersion: 1, characterVersion: "cap-v1", paletteId: "bubblegum-sky" } }]; return []; @@ -39,6 +40,102 @@ function database(onPredicate: (kind: "where" | "join", condition: SQL) => void } }; } describe("authenticated activity notifications", () => { + it("retains live delivery idle work until the identifier read settles after the last socket closes", async () => { + const baseline = idleWorkSnapshot().active; + let release!: () => void, entered!: () => void; + const pending = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const db = database(), select = db.select; + db.select = fields => { + const query = select(fields); + if (fields.identifier) { + const limit = query.limit; + query.limit = async () => { entered(); await pending; return limit(); }; + } + return query; + }; + const server = createServer(); + const wss = setupLiveEventsWebSocketServer(server, db as never, { deploymentMode: "local_trusted" }); + const closed = new Promise(resolve => wss.on("connection", socket => socket.on("close", resolve))); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as { port: number }).port}/api/companies/company/events/ws`); + try { + await once(socket, "open"); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline)); + // System activity has no actor lookup: this gate is the new identifier SQL. + publishLiveEvent({ companyId: "company", type: "activity.logged", payload: { + action: "issue.updated", entityType: "issue", entityId: "visible", actorType: "system", + } }); + await started; + socket.terminate(); + await closed; + expect(wss.clients.size).toBe(0); + expect(idleWorkSnapshot().active).toBe(baseline + 1); + release(); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline)); + } finally { + release(); + socket.terminate(); + await new Promise(resolve => server.close(() => resolve())); + await vi.waitFor(() => expect(idleWorkSnapshot().active).toBe(baseline)); + } + }); + + it.each(["local_trusted", "authenticated"] as const)("keeps authorized notifications when optional identifier reads fail (%s)", async deploymentMode => { + const db = database(); + const select = db.select; + const identifierRead = vi.fn(); + db.select = fields => { + if (fields.identifier) { + identifierRead(); + if (identifierRead.mock.calls.length <= 3) throw new Error("Identifier lookup unavailable"); + } + return select(fields); + }; + const server = createServer(); + setupLiveEventsWebSocketServer(server, db as never, { deploymentMode, + resolveCloudActor: async () => ({ userId: "viewer", companyIds: ["company"] }) }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const socket = new WebSocket(`ws://127.0.0.1:${(server.address() as { port: number }).port}/api/companies/company/events/ws`); + try { + await once(socket, "open"); + const messages: Array<{ payload: Record }> = []; + const delivered = new Promise(resolve => socket.on("message", data => { + const event = JSON.parse(data.toString()); + messages.push(event); + if (event.payload.entityId === "after-failure") resolve(); + })); + const activity = (entityId: string, actorType: string) => publishLiveEvent({ + companyId: "company", type: "activity.logged", payload: { + action: "issue.updated", entityType: "issue", entityId, actorType, + actorId: actorType === "user" ? "viewer" : actorType === "agent" ? "agent" : null, + issueIdentifier: "SPOOF-1", details: { secret: "private task details" }, + }, + }); + if (deploymentMode === "authenticated") activity("hidden", "system"); + for (const actorType of ["system", "agent", "user"]) { + activity("visible", actorType); + } + activity("after-failure", "system"); + await delivered; + expect(messages).toHaveLength(4); + expect(identifierRead).toHaveBeenCalledTimes(4); + for (const [index, actorType] of ["system", "agent", "user"].entries()) { + const event = messages[index]; + expect(event.payload).toMatchObject({ entityId: "visible", actorType, issueIdentifier: null }); + if (actorType === "agent") expect(event.payload.actorName).toBe("Coder"); + if (actorType === "user") expect(event.payload.actorName).toBe("Alex Example"); + if (deploymentMode === "authenticated") expect(event.payload).not.toHaveProperty("details"); + } + expect(messages[3].payload.issueIdentifier).toBe("PAP-1234"); + } finally { + socket.terminate(); + await new Promise(resolve => server.close(() => resolve())); + } + }); + it("preserves only an authorized handoff task identity for empty case-panel refresh", async () => { const server = createServer(); setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode: "authenticated", @@ -88,6 +185,7 @@ describe("authenticated activity notifications", () => { } }); const [data] = await delivery; expect(JSON.parse(data.toString()).toastEligible).toBe(true); + expect(JSON.parse(data.toString()).payload.issueIdentifier).toBe("PAP-1234"); expect(taskToastEligible).toHaveBeenLastCalledWith(expect.anything(), expect.anything(), "local-board"); } finally { socket.terminate(); @@ -123,6 +221,8 @@ describe("authenticated activity notifications", () => { expect(b.toastEligible).toBe(false); expect(a.payload.entityId).toBe(entityId); expect(b.payload.entityId).toBe(entityId); + expect(a.payload.issueIdentifier).toBe("PAP-1234"); + expect(b.payload.issueIdentifier).toBe("PAP-1234"); expect(original).not.toHaveProperty("toastEligible"); } } finally { @@ -132,9 +232,9 @@ describe("authenticated activity notifications", () => { } }); - it("keeps actor identity after authorization and excludes hidden tasks and private details", async () => { + it.each(["authenticated", "local_trusted"] as const)("keeps task and actor identity for %s without exposing hidden tasks or copying private details", async (deploymentMode) => { const server = createServer(); - setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode: "authenticated", + setupLiveEventsWebSocketServer(server, database() as never, { deploymentMode, resolveCloudActor: async () => ({ userId: "viewer", companyIds: ["company"] }) }); server.listen(0, "127.0.0.1"); await once(server, "listening"); @@ -145,19 +245,21 @@ describe("authenticated activity notifications", () => { socket.on("message", (data) => messages.push(JSON.parse(data.toString()))); const activity = (entityId: string, actorType = "user") => publishLiveEvent({ companyId: "company", type: "activity.logged", payload: { action: "issue.created", entityType: "issue", entityId, actorType, actorId: actorType === "user" ? "viewer" : "agent", + issueIdentifier: "SPOOF-1", details: { bodySnippet: "private comment", secret: "not for the socket" }, } }); - activity("hidden"); + if (deploymentMode === "authenticated") activity("hidden"); const first = once(socket, "message"); activity("visible"); await first; expect(messages).toHaveLength(1); - expect(messages[0].payload).toEqual({ action: "issue.created", entityType: "issue", entityId: "visible", actorType: "user", actorId: "viewer", actorName: "Alex Example", actorImage: "/alex.png" }); + expect(messages[0].payload).toMatchObject({ action: "issue.created", entityType: "issue", entityId: "visible", issueIdentifier: "PAP-1234", actorType: "user", actorId: "viewer", actorName: "Alex Example", actorImage: "/alex.png" }); + if (deploymentMode === "authenticated") expect(messages[0].payload).not.toHaveProperty("details"); const second = once(socket, "message"); activity("visible", "agent"); await second; - expect(messages[1].payload).toMatchObject({ actorName: "Coder", actorType: "agent", actorId: "agent", actorAppearance: { paletteId: "bubblegum-sky" } }); - expect(messages[1].payload).not.toHaveProperty("details"); + expect(messages[1].payload).toMatchObject({ issueIdentifier: "PAP-1234", actorName: "Coder", actorType: "agent", actorId: "agent", actorAppearance: { paletteId: "bubblegum-sky" } }); + if (deploymentMode === "authenticated") expect(messages[1].payload).not.toHaveProperty("details"); } finally { socket.terminate(); await new Promise((resolve) => server.close(() => resolve())); @@ -167,6 +269,20 @@ describe("authenticated activity notifications", () => { describe("activity profile query boundaries", () => { + it("reads the task identifier within its company even when there is no actor", async () => { + const predicates: { sql: string; params: unknown[] }[] = []; + const db = database((_kind, condition) => { + const { sql, params } = new PgDialect().sqlToQuery(condition); + predicates.push({ sql, params }); + }); + expect(await activityToastIdentity(db as never, "company-A", { + action: "issue.updated", entityType: "issue", entityId: "task-A", actorType: "system", + })).toMatchObject({ issueIdentifier: "PAP-1234" }); + expect(predicates).toEqual([ + { sql: '("issues"."company_id" = $1 and "issues"."id" = $2)', params: ["company-A", "task-A"] }, + ]); + }); + it("requires the exact company, human actor, and active membership before reading a person profile", async () => { const predicates: { kind: string; sql: string; params: unknown[] }[] = []; const db = database((kind, condition) => { diff --git a/server/src/__tests__/live-recovery-work.test.ts b/server/src/__tests__/live-recovery-work.test.ts index ae79cab0a1a..25f3eeb9984 100644 --- a/server/src/__tests__/live-recovery-work.test.ts +++ b/server/src/__tests__/live-recovery-work.test.ts @@ -32,8 +32,14 @@ import { terminalizeLegacyExecution } from "../services/legacy-execution-recover import { issueService } from "../services/issues.js"; import { nativeRetryCancellationEligible, pendingFailedNativeRetryCancellationCondition } from "../services/native-runtime/native-cancellation-request.js"; import { requestNativeQuestionRunCancellation } from "../services/native-runtime/native-question-bridge.js"; -import { readNativeLocalProcessStop } from "../services/native-local-process-stop.js"; +import { readNativeLocalProcessStop, releaseSettledNativeRunProcess, settleNativeRunProcess } from "../services/native-local-process-stop.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js"; +import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; +import * as nativeStopEvidence from "../services/acknowledged-native-stop.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { subscribeCompanyLiveEvents } from "../services/live-events.js"; +import type { LiveEvent } from "@paperclipai/shared"; import { logger } from "../middleware/logger.js"; vi.setConfig({ testTimeout: 90_000 }); @@ -295,6 +301,115 @@ describe("residual recovery production matrix", () => { }, { timeout: 45_000 }); await competing.stop(); extraStops.pop(); await quiet(); }); + it.each(["confirmed", "already_retired", "missing_terminal", "unproven_terminal", "ack_failure", "late_terminal"] as const)( + "settles an API-only Stop after deferred controller retirement without inventing proof: %s", async mode => { + const f = await fixture(), runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); + const statusEvents: LiveEvent[] = []; + const unsubscribe = subscribeCompanyLiveEvents(f.companyId, event => { + if (event.type === "heartbeat.run.status" && event.payload.runId === runId) statusEvents.push(event); + }); + extraStops.push(async () => unsubscribe()); + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" }); + await once(child, "spawn"); + const stopChild = async () => { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = once(child, "exit"); child.kill("SIGTERM"); await exited; + }; + extraStops.push(stopChild); + const recordedAt = new Date().toISOString(); + const nativeExecutionInput = buildNativeExecutionInput({ companyId: f.companyId, agentId: f.agent.id, runId, + provider: "cursor_cloud", model: "explicit-model", cursorCloud: { schema: "paperclip.cursor-cloud.config.v1", + repos: [{ url: "https://github.com/example/qualification" }] }, + issue: { id: f.issue.id, identifier: "API-1", title: f.issue.title, description: null, workMode: "standard" }, + taskPrompt: "Stop the foreground command", normalizedSessionId: sessionId, + workspace: { id: randomUUID(), cwd: "/private/controller", repoUrl: null, repoRef: null, branchName: null }, + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Stop the foreground command", criteria: [{ id: "stop", requirement: "Stop the foreground command" }] } }, + runtimeContext: nativeRuntimeContextFixture(), + }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agent.id, + runtimeMode: "native", driverKind: "cursor_cloud_api", nativeIssueId: f.issue.id, nativeSessionId: sessionId, + runnerInstanceId: runnerId, status: "cancelled", finishedAt: new Date(), + runnerProfileJson: { backend: "cursor_cloud_api", nativeExecutionInput, sessionCheckpoint: { providerSessionId: "cloud-agent" }, + executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } }, + resultJson: { executionCancellation: { state: "requested", requestedAt: recordedAt }, + cancelledByActorType: "user", cancelledByUserId: f.actorId, nativeCancellation: { + schema: "paperclip.native-cancellation.v1", companyId: f.companyId, runId, issueId: f.issue.id, + scope: "run", reasonCode: "cancellation_run_only", dispatchState: "acknowledged", dispatched: true, + effects: ["release_run_resources"], recordedAt, acknowledgedAt: recordedAt, + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID() } } }); + await persistHeartbeatRunProcessMetadata(db, runId, { pid: child.pid!, processGroupId: child.pid!, startedAt: recordedAt, targetKind: "local" }); + await db.insert(nativeRunFinalizations).values({ companyId: f.companyId, issueId: f.issue.id, runId, phase: "committed" }); + const appendTerminal = async (target: typeof db = db) => { + const terminal = { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "turn.interrupted", + sourceKind: "runner", sourceInstanceId: runnerId, sourceEventId: `${runnerId}:${runId}:1`, sourceSeq: 1, + runId, turnId: "stopped-turn", normalizedSessionId: sessionId, emittedAt: new Date().toISOString(), priority: 1, + payload: { status: "interrupted", error: null, ...(mode === "unproven_terminal" ? { providerTerminalObserved: false } : {}) } }; + await appendHeartbeatRunEvent(target, { companyId: f.companyId, agentId: f.agent.id, runId, + eventType: terminal.eventType, payload: { prpEvent: terminal }, nativeSource: { + sourceInstanceId: runnerId, sourceEventId: terminal.sourceEventId, sourceSeq: 1, + protocolSchemaVersion: 1, canonicalPayload: terminal } }); + }; + if (mode !== "missing_terminal" && mode !== "late_terminal") await appendTerminal(); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, runId))).toEqual([]); + expect(await settleNativeRunProcess(db, runId)).toBe("pending"); + expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" }); + expect(statusEvents).toEqual([]); + if (mode === "already_retired") { await stopChild(); expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); } + await start(); + if (mode !== "already_retired") { + await vi.waitFor(() => expect(deadlines.get(DELIVERY_QUEUES.settledNativeProcesses)?.()).toBeGreaterThan(Date.now())); + expect((await getRun(runId)).processPid).toBe(child.pid); + await stopChild(); + } + const failure = new Error("synthetic Stop proof read failure after controller retirement"); + if (mode === "ack_failure") vi.spyOn(nativeStopEvidence, "acknowledgedNativeApiStopExecutionHasStopped").mockRejectedValueOnce(failure); + await advance(31_000); + if (mode === "ack_failure") { + expect((await getRun(runId)).processPid).toBeNull(); + expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" }); + expect(errors).toHaveLength(1); + expect(errors[0]).toBeInstanceOf(AggregateError); + expect((errors[0] as AggregateError).errors).toEqual([failure]); errors.length = 0; + expect(statusEvents).toEqual([]); + await advance(31_000); + } + const stopped = await getRun(runId); + expect(stopped).toMatchObject({ processPid: null, processGroupId: null, processStartedAt: null }); + expect(await readNativeLocalProcessStop(db, f.companyId, runId)).toMatchObject({ processPid: child.pid, processGroupId: child.pid }); + expect(stopped.resultJson?.executionCancellation).toMatchObject(["missing_terminal", "unproven_terminal", "late_terminal"].includes(mode) + ? { state: "requested" } : { state: "acknowledged", proof: "provider_api_stop_receipt" }); + expect(statusEvents).toHaveLength(["missing_terminal", "unproven_terminal", "late_terminal"].includes(mode) ? 0 : 1); + await quiet(); + if (mode === "late_terminal") { + const rollback = new Error("rollback the uncommitted terminal receipt"); + await expect(db.transaction(async tx => { await appendTerminal(tx as unknown as typeof db); throw rollback; })).rejects.toBe(rollback); + await advance(1_000); + expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" }); + expect(statusEvents).toEqual([]); + await quiet(); + let commit!: () => void, entered!: () => void; + const gate = new Promise(resolve => { commit = resolve; }); + const recorded = new Promise(resolve => { entered = resolve; }); + const transaction = db.transaction(async tx => { await appendTerminal(tx as unknown as typeof db); entered(); await gate; }); + await recorded; + try { + await vi.advanceTimersByTimeAsync(500); await immediate(); + expect((await getRun(runId)).resultJson?.executionCancellation).toMatchObject({ state: "requested" }); + expect(statusEvents).toEqual([]); + expect(await db.select().from(heartbeatRunEvents).where(and(eq(heartbeatRunEvents.runId, runId), + eq(heartbeatRunEvents.eventType, "turn.interrupted")))).toEqual([]); + } finally { commit(); await transaction; } + await vi.waitFor(async () => expect((await getRun(runId)).resultJson?.executionCancellation) + .toMatchObject({ state: "acknowledged", proof: "provider_api_stop_receipt" })); + await quiet(); + expect(statusEvents).toHaveLength(1); + expect(statusEvents[0]).toMatchObject({ companyId: f.companyId, type: "heartbeat.run.status", + payload: { runId, agentId: f.agent.id, status: "cancelled" } }); + expect(statusEvents[0]!.payload).not.toHaveProperty("resultJson"); + } + }, + ); it("releases a settled native process only after matching host and birth evidence arrives", async () => { const f = await fixture(), startedAt = new Date(); const child = spawn(process.execPath, ["-e", "console.log('identity-proof')"], { stdio: "ignore" }); diff --git a/server/src/__tests__/native-run-projection-work.test.ts b/server/src/__tests__/native-run-projection-work.test.ts index 26f0a9d239a..f9718354a1c 100644 --- a/server/src/__tests__/native-run-projection-work.test.ts +++ b/server/src/__tests__/native-run-projection-work.test.ts @@ -1,3 +1,4 @@ +import * as wakeSettlementNotifications from "../services/external-conversation-settlement-notifications.js"; import { heartbeatService } from "../services/heartbeat.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "../services/heartbeat/run-lifecycle.js"; @@ -8,14 +9,14 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { agentWakeupRequests, agents, companies, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, workspaceOperations, type Db } from "@paperclipai/db"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { nativeRunProjectionWork } from "../services/native-runtime/native-run-projection-work.js"; import { projectCommittedRun } from "../services/native-runtime/native-run-projection.js"; import { notifyNativeFinalizationWork } from "../services/native-finalization-notifications.js"; -import { finalizeNativeRun } from "../services/native-runtime/native-run-finalizer.js"; +import { finalizeNativeRun, recordNativeFinalizationFailure } from "../services/native-runtime/native-run-finalizer.js"; import * as statusCommitter from "../services/native-runtime/status-decision-committer.js"; import { PaperclipControlPlanePort } from "../services/native-runtime/paperclip-control-plane-port.js"; import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js"; @@ -120,6 +121,154 @@ describe("committed native run projection lifecycle", () => { async function accepted(status: "succeeded" | "failed" | "cancelled" = "succeeded") { const binding = await seed(); await complete(binding, "control_plane", db, status); await barrier(binding); return binding; } + async function claimedWake(binding: Awaited>) { + const [wake] = await db.insert(agentWakeupRequests).values({ companyId, agentId, source: "issue", + status: "claimed", runId: binding.runId, claimedAt: new Date(), payload: { issueId: binding.issueId } }).returning(); + await db.update(heartbeatRuns).set({ wakeupRequestId: wake.id }).where(eq(heartbeatRuns.id, binding.runId)); + return wake; + } + const readWake = async (id: string) => (await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, id)))[0]; + + it.each(["succeeded", "failed", "cancelled"] as const)("settles the exact wake atomically with a fresh %s projection", async status => { + const binding = await accepted(status), wake = await claimedWake(binding); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const run = await read(binding.runId), settled = await readWake(wake.id); + expect(run.status).toBe(status); + expect(settled).toMatchObject({ status: status === "succeeded" ? "completed" : status, runId: binding.runId, + companyId, agentId, finishedAt: run.finishedAt, error: run.error }); + expect(settled.finishedAt).not.toBeNull(); + await projectCommittedRun({ db, run, coordinator: await state(binding.runId) }); + expect(await readWake(wake.id)).toEqual(settled); + }); + + it.each(["provider", "workspace"] as const)("settles the exact wake when %s finalization retries exhaust", async scope => { + const binding = await accepted(), wake = await claimedWake(binding); + if (scope === "workspace") await db.update(workspaceOperations).set({ status: "failed" }) + .where(eq(workspaceOperations.heartbeatRunId, binding.runId)); + if (scope === "provider") await db.update(nativeRunResults).set({ + resultJson: sql`jsonb_set(${nativeRunResults.resultJson}, '{terminal,runTerminalState}', '"unknown"'::jsonb)`, + }).where(eq(nativeRunResults.runId, binding.runId)); + for (let attempt = 0; attempt < 3; attempt++) { + await db.update(nativeRunFinalizations).set({ nextAttemptAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, binding.runId)); + if (scope === "provider") await reconcileNativeFinalizations(db, [binding.runId]); + else await recordNativeFinalizationFailure({ db, runId: binding.runId, projectRunStatus: true, + failureScope: "workspace", error: new Error("native_workspace_finalize_failed") }); + if (attempt < 2) { + expect((await read(binding.runId)).status).toBe("running"); + expect(await readWake(wake.id)).toMatchObject({ status: "claimed", finishedAt: null }); + } + } + const run = await read(binding.runId); + expect((await state(binding.runId)).phase).toBe("terminal_failure"); + expect(run).toMatchObject({ status: scope === "provider" ? "succeeded" : "failed", nativePhase: "terminal_failure" }); + expect(run.finishedAt).not.toBeNull(); + expect(await readWake(wake.id)).toMatchObject({ status: scope === "provider" ? "completed" : "failed", + runId: binding.runId, finishedAt: run.finishedAt }); + }); + + it("keeps the source wake claimed when exhausted failure does not own terminal projection", async () => { + const binding = await accepted(), wake = await claimedWake(binding); + await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId)); + for (let attempt = 0; attempt < 3; attempt++) await recordNativeFinalizationFailure({ db, runId: binding.runId, + projectRunStatus: false, failureScope: "workspace", error: new Error("native_workspace_finalize_failed") }); + expect((await state(binding.runId)).phase).toBe("terminal_failure"); + expect((await read(binding.runId)).status).toBe("running"); + expect(await readWake(wake.id)).toEqual(wake); + }); + + it("rolls back exhausted projection and its claim if wake notification registration fails", async () => { + const binding = await accepted(), wake = await claimedWake(binding); + await db.update(workspaceOperations).set({ status: "failed" }).where(eq(workspaceOperations.heartbeatRunId, binding.runId)); + const fail = () => recordNativeFinalizationFailure({ db, runId: binding.runId, + projectRunStatus: true, failureScope: "workspace" as const, error: new Error("native_workspace_finalize_failed") }); + await fail(); await fail(); + const beforeRun = await read(binding.runId), beforeCoordinator = await state(binding.runId); + const notify = wakeSettlementNotifications.notifyExternalConversationSettlement; + vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement").mockImplementation(async (tx, scope) => { + const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); + expect(current.status).toBe("claimed"); + await notify(tx, scope); + throw new Error("exhaustion wake notification failure"); + }); + await expect(fail()).rejects.toThrow("exhaustion wake notification failure"); + expect(await read(binding.runId)).toEqual(beforeRun); + expect(await state(binding.runId)).toEqual(beforeCoordinator); + expect(await readWake(wake.id)).toEqual(wake); + }); + + it.each(["succeeded", "failed", "cancelled"] as const)("discovers and repairs an old %s run's claimed wake once", async status => { + const binding = await accepted(status); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + const run = await read(binding.runId), wake = await claimedWake(binding); + const work = nativeRunProjectionWork(db); + const [entry] = await work.load({ companyId, runId: binding.runId }); + expect(entry).toMatchObject({ id: binding.runId }); + await work.check(entry); + expect(await readWake(wake.id)).toMatchObject({ status: status === "succeeded" ? "completed" : status, + finishedAt: run.finishedAt }); + expect((await read(binding.runId)).finishedAt).toEqual(run.finishedAt); + expect(await work.load({ companyId, runId: binding.runId })).toEqual([]); + }); + + it.each(["unbound", "newer-run", "queued", "cancelled", "foreign-company", "foreign-agent", "live-owner"] as const)( + "does not settle a %s wake claim", async fault => { + const binding = await accepted(), wake = await claimedWake(binding); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); + if (fault === "unbound" || fault === "newer-run") await db.update(agentWakeupRequests) + .set({ runId: fault === "unbound" ? null : randomUUID() }).where(eq(agentWakeupRequests.id, wake.id)); + if (fault === "queued" || fault === "cancelled") await db.update(agentWakeupRequests) + .set({ status: fault }).where(eq(agentWakeupRequests.id, wake.id)); + if (fault === "foreign-company") { + const otherCompany = randomUUID(); + await db.insert(companies).values({ id: otherCompany, name: "Other wake owner", issuePrefix: `OW${randomUUID().slice(0, 6)}` }); + await db.update(agentWakeupRequests).set({ companyId: otherCompany }).where(eq(agentWakeupRequests.id, wake.id)); + } + if (fault === "foreign-agent") { + const otherAgent = randomUUID(); + await db.insert(agents).values({ id: otherAgent, companyId, name: "Other wake owner", adapterType: "paperclip_runner" }); + await db.update(agentWakeupRequests).set({ agentId: otherAgent }).where(eq(agentWakeupRequests.id, wake.id)); + } + if (fault === "live-owner") await db.update(heartbeatRuns).set({ status: "running", nativePhase: "terminal_failure", + errorCode: "native_execution_ownership_unverified" }).where(eq(heartbeatRuns.id, binding.runId)); + const before = await readWake(wake.id); + await projectCommittedRun({ db, run: await read(binding.runId), coordinator: await state(binding.runId) }); + expect(await readWake(wake.id)).toEqual(before); + if (fault === "live-owner") expect((await read(binding.runId)).status).toBe("running"); + }, + ); + + it("registers wake settlement intent before mutation and rolls back notification failure", async () => { + const binding = await accepted(), wake = await claimedWake(binding); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); + const before = await read(binding.runId); + const notify = wakeSettlementNotifications.notifyExternalConversationSettlement; + const notification = vi.spyOn(wakeSettlementNotifications, "notifyExternalConversationSettlement") + .mockImplementation(async (tx, scope) => { + const [current] = await tx.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)); + expect(current.status).toBe("claimed"); + await notify(tx, scope); + throw new Error("wake intent registration failure"); + }); + await expect(projectCommittedRun({ db, run: before, coordinator: await state(binding.runId) })) + .rejects.toThrow("wake intent registration failure"); + expect(notification).toHaveBeenCalledOnce(); + expect(await read(binding.runId)).toEqual(before); + expect(await readWake(wake.id)).toEqual(wake); + }); + + it("rolls back terminal projection and exact wake settlement together", async () => { + const binding = await accepted(), wake = await claimedWake(binding); + await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); + const before = await read(binding.runId); + await expect(db.transaction(async tx => { + await projectCommittedRun({ db: tx as unknown as Db, run: before, coordinator: await state(binding.runId) }); + throw new Error("projection rollback fixture"); + })).rejects.toThrow("projection rollback fixture"); + expect(await read(binding.runId)).toEqual(before); + expect(await readWake(wake.id)).toEqual(wake); + }); + it("reconstructs a committed decision with an unfinished run projection and stays quiet after repair", async () => { const binding = await accepted(); await finalizeNativeRun({ db, runId: binding.runId, workspaceFinalizeStatus: "succeeded" }); diff --git a/server/src/maintenance/ephemeral-migration-data.ts b/server/src/maintenance/ephemeral-migration-data.ts index 17c713b0f12..eda8e547d6e 100644 --- a/server/src/maintenance/ephemeral-migration-data.ts +++ b/server/src/maintenance/ephemeral-migration-data.ts @@ -8,6 +8,7 @@ import { skillInstallationMaintenance } from "../services/skill-installation-mai import { skillPackageMaintenance } from "../services/skill-package-maintenance.js"; import { hasUnsettledLocalInstructionCopies } from "../services/agent-instruction-working-copies.js"; import { providerTraceStore } from "../services/provider-trace-store.js"; +import { migrateWorkspaceTrees } from "../services/remote-workspace-migration-batch.js"; /** Runs only in the maintenance process after Cloud has confirmed all legacy * deployments stopped. Each owner retains its own migration/publication rules; @@ -23,10 +24,15 @@ export async function migrateEphemeralServerData(db: Db, input: { if (await hasUnsettledLocalInstructionCopies(db)) throw new Error("local_instruction_copies_require_settlement"); const before = await readEphemeralServerDataEligibility(db); const unsupported = before.blockers.filter(blocker => ![ - "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", + "agent_files_require_migration", "run_logs_require_migration", "provider_traces_require_migration", "skills_require_migration", "workspaces_require_migration", ].includes(blocker)); if (unsupported.length) throw new Error(unsupported[0]); + phase("workspaces"); + const storage = getStorageService(); + if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); + counts.workspaces = await migrateWorkspaceTrees(db, { storage, signal, phase }); + phase("agent_files"); let after: string | undefined; do { @@ -77,8 +83,6 @@ export async function migrateEphemeralServerData(db: Db, input: { } phase("verify_agent_objects"); - const storage = getStorageService(); - if (storage.provider !== "s3") throw new Error("shared_object_storage_required"); counts.verifiedAgentObjects = await verifyAgentFileObjects(db, storage, signal); phase("verify_skill_versions"); after = undefined; diff --git a/server/src/middleware/auth.ts b/server/src/middleware/auth.ts index f9bd7446947..bf8a05f22d0 100644 --- a/server/src/middleware/auth.ts +++ b/server/src/middleware/auth.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "../services/idle-api-activity.js"; import { createHash, timingSafeEqual } from "node:crypto"; import type { Request, RequestHandler } from "express"; import { and, eq, isNull, sql } from "drizzle-orm"; @@ -43,7 +44,7 @@ import { ensureHumanRoleDefaultGrants } from "../services/principal-access-compa import { forbidden, unauthorized, unprocessable } from "../errors.js"; export { isCloudManagedInstance } from "../services/cloud-instance.js"; -import { cloudTenantPrimaryCompanyId } from "../services/cloud-instance.js"; +import { cloudTenantPrimaryCompanyId, isCloudManagedInstance } from "../services/cloud-instance.js"; function hashToken(token: string) { return createHash("sha256").update(token).digest("hex"); @@ -329,6 +330,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "board_key", }; + if (isCloudManagedInstance() && (access.isInstanceAdmin || access.companyIds.length)) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); return; } @@ -521,7 +525,9 @@ export function actorMiddleware(db: Db, opts: ActorMiddlewareOptions): RequestHa runId: runIdHeader || undefined, source: "agent_key", }; - + if (isCloudManagedInstance() && req.actor.onBehalfOfMemberships?.length) { + _res.once("finish", () => { if (_res.statusCode >= 200 && _res.statusCode < 400) idleApiActivity.record(); }); + } next(); }; return async (req, res, next) => { diff --git a/server/src/middleware/cloud-control.ts b/server/src/middleware/cloud-control.ts index b726aae3092..6165af6b0b6 100644 --- a/server/src/middleware/cloud-control.ts +++ b/server/src/middleware/cloud-control.ts @@ -1,3 +1,5 @@ +import type { Request } from "express"; +import type { VerifiedIdleCapabilities } from "../services/idle-incoming-wake-capability.js"; import type { RequestHandler } from "express"; import { logger } from "./logger.js"; import { @@ -6,6 +8,12 @@ import { type CloudControlAction, } from "../services/cloud-runtime-identity.js"; +const idleCapabilities = new WeakMap(); +/** Only this middleware's verified assertion can install request capabilities. */ +export function verifiedIdleCapabilities(req: Request): VerifiedIdleCapabilities | undefined { + return idleCapabilities.get(req); +} + /** Endpoint → method → the one action a control assertion must name to take it. */ const ACTIONS_BY_ENDPOINT: Record> = { "/api/instance/connector-events/wake": { POST: "connector-events:wake" }, @@ -50,7 +58,8 @@ export function cloudControlMiddleware(): RequestHandler { return; } try { - verifyCloudControlAssertion({ compactJws: assertion, expectedAction }); + const claims = verifyCloudControlAssertion({ compactJws: assertion, expectedAction }); + if (claims.idleCapabilities) idleCapabilities.set(req, claims.idleCapabilities as VerifiedIdleCapabilities); } catch (error) { logger.warn({ err: error }, "Rejected Cloud control assertion"); res.status(401).json({ error: "invalid_cloud_control_assertion" }); diff --git a/server/src/modules/README.md b/server/src/modules/README.md index 6c3bb456c49..1da9c9b7c67 100644 --- a/server/src/modules/README.md +++ b/server/src/modules/README.md @@ -56,3 +56,9 @@ Ordinary configuration updates use the root lifecycle command. The [agent lifecycle module](../../../doc/AGENT-LIFECYCLE.md) owns agent creation, pause, resume, and termination. Use its commands for these changes. + +The execution admission module owns project task-execution policy and the +durable admission records that charge project capacity. Today it exposes only +internal policy commands and the deletion guards that project, agent, and +company deletion call. Execution claims still belong to `services/heartbeat/queue.ts` +and native review dispatch. See [execution admission accounting](../../../doc/DATABASE.md#execution-admission-accounting). diff --git a/server/src/modules/agent-lifecycle/adapters/configuration.ts b/server/src/modules/agent-lifecycle/adapters/configuration.ts index 80e90dbd71d..7acebf46d54 100644 --- a/server/src/modules/agent-lifecycle/adapters/configuration.ts +++ b/server/src/modules/agent-lifecycle/adapters/configuration.ts @@ -28,7 +28,9 @@ export async function updateAgentConfiguration(tx: Db, effects: LifecycleEffects const prepareSetup = changed && current.lifecycleHolds.length === 1 && current.lifecycleHolds[0] === "setup" && ["paused", "preparing", "verifying"].includes(current.lifecycleState) && await effects.setupRequiresPreparation(tx, current.companyId, - Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") ? patch.defaultEnvironmentId : current.defaultEnvironmentId); + Object.prototype.hasOwnProperty.call(patch, "defaultEnvironmentId") ? patch.defaultEnvironmentId : current.defaultEnvironmentId, + { adapterType: patch.adapterType ?? current.adapterType, + adapterConfig: Object.prototype.hasOwnProperty.call(patch, "adapterConfig") ? patch.adapterConfig : current.adapterConfig }); const invalidate = changed && (["preparing", "verifying", "resuming"].includes(current.lifecycleState) || prepareSetup); if (invalidate) await effects.notifyWork(tx); const [updated] = await tx.update(agents).set({ ...patch, updatedAt: new Date(), ...(invalidate ? { diff --git a/server/src/modules/agent-lifecycle/adapters/effects.ts b/server/src/modules/agent-lifecycle/adapters/effects.ts index 57de2cc5468..e1787124063 100644 --- a/server/src/modules/agent-lifecycle/adapters/effects.ts +++ b/server/src/modules/agent-lifecycle/adapters/effects.ts @@ -6,7 +6,8 @@ import type { ActivityPublication } from "../../../types/activity-publication.js /** Services supply integrations; lifecycle owns the transaction and state writes. */ export interface LifecycleEffects { - setupRequiresPreparation(db: Db, companyId: string, environmentId?: string | null): Promise; + setupRequiresPreparation(db: Db, companyId: string, environmentId: string | null | undefined, + profile: Pick): Promise; notifyWork(db: Db): Promise; beforeStatusChange(db: Db, companyId: string, agentId: string, before: string, after: string): Promise; reportFailure(context: LifecycleFailure, error: unknown): void; diff --git a/server/src/modules/agent-lifecycle/adapters/postgres.ts b/server/src/modules/agent-lifecycle/adapters/postgres.ts index d7f8e77c553..2ce6580ef41 100644 --- a/server/src/modules/agent-lifecycle/adapters/postgres.ts +++ b/server/src/modules/agent-lifecycle/adapters/postgres.ts @@ -64,7 +64,7 @@ async function changeLockedAgent(tx: Db, effects: LifecycleEffects, agent: Lifec if (command === "complete" && next === "ready" && holds.length) next = "pausing"; if (command === "complete" && next === "paused" && !holds.length) next = "resuming"; if (["complete", "reconcile"].includes(command) && next === "paused" && holds.length === 1 && holds[0] === "setup" - && await effects.setupRequiresPreparation(tx, agent.companyId, agent.defaultEnvironmentId)) next = "preparing"; + && await effects.setupRequiresPreparation(tx, agent.companyId, agent.defaultEnvironmentId, agent)) next = "preparing"; if (next === agent.lifecycleState) { if (JSON.stringify(holds) !== JSON.stringify(agent.lifecycleHolds) || agent.pauseReason !== (holds[0] ?? null)) { const [updated] = await tx.update(agents).set({ lifecycleHolds: holds, pauseReason: holds[0] ?? null }).where(eq(agents.id, agent.id)).returning(); diff --git a/server/src/modules/agent-lifecycle/adapters/records.ts b/server/src/modules/agent-lifecycle/adapters/records.ts index 96892987b01..64e10144c01 100644 --- a/server/src/modules/agent-lifecycle/adapters/records.ts +++ b/server/src/modules/agent-lifecycle/adapters/records.ts @@ -36,7 +36,7 @@ export function agentRecords(db: Db, effects: LifecycleEffects) { const [company] = await tx.select({ status: companies.status }).from(companies).where(eq(companies.id, companyId)); const companyHolds = company?.status === "paused" ? ["company_paused"] : company?.status === "archived" ? ["company_archived"] : []; const holds = [...(status === "paused" ? [pauseReason ?? "manual"] : []), ...companyHolds]; - const prepareSetup = options?.setupOnly && !companyHolds.length && await effects.setupRequiresPreparation(txDb, companyId, prepared.defaultEnvironmentId); + const prepareSetup = options?.setupOnly && !companyHolds.length && await effects.setupRequiresPreparation(txDb, companyId, prepared.defaultEnvironmentId, prepared); const lifecycleState = status === "pending_approval" ? "pending_approval" : status === "terminated" ? "terminated" : prepareSetup ? "preparing" : status === "paused" ? "paused" : companyHolds.length ? "pausing" : "preparing"; if (lifecycleState === "preparing" || lifecycleState === "pausing") await effects.notifyWork(tx); diff --git a/server/src/modules/execution-admission/adapters/company-deletion.ts b/server/src/modules/execution-admission/adapters/company-deletion.ts new file mode 100644 index 00000000000..5e421170738 --- /dev/null +++ b/server/src/modules/execution-admission/adapters/company-deletion.ts @@ -0,0 +1,11 @@ +import { executionAdmissions } from "@paperclipai/db"; +import { eq } from "drizzle-orm"; +import type { CompanyDeletionParticipant } from "../../../lib/company-deletion.js"; + +/** Removes every admission, settled or not, before the company's runs and projects are deleted. */ +export const executionAdmissionCompanyDeletion: CompanyDeletionParticipant = { + async deleteCompanyData(tx, companyId) { + if (!("nestedIndex" in tx)) throw new Error("Company deletion requires a database transaction"); + await tx.delete(executionAdmissions).where(eq(executionAdmissions.companyId, companyId)); + }, +}; diff --git a/server/src/modules/execution-admission/adapters/postgres.test.ts b/server/src/modules/execution-admission/adapters/postgres.test.ts new file mode 100644 index 00000000000..ecdfef3cf07 --- /dev/null +++ b/server/src/modules/execution-admission/adapters/postgres.test.ts @@ -0,0 +1,239 @@ +import { randomUUID } from "node:crypto"; +import { + agents, + companies, + createDb, + executionAdmissionProjectCharges as charges, + executionAdmissions as admissions, + heartbeatRuns, + projectTaskExecutionPolicies as policies, + projects, +} from "@paperclipai/db"; +import { asc, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; +import { withAccountingTransaction } from "../../../services/accounting-transaction.js"; +import { deleteAgentDependencies } from "../../../services/agent-deletion.js"; +import { deleteCompany } from "../../../services/company-deletion.js"; +import { projectService } from "../../../services/projects.js"; +import { prepareAdmissionRecord, type AdmissionRecord } from "../domain/admission.js"; +import { createExecutionAdmission } from "../index.js"; +import { countActiveProjectCharges, recordAdmission, settleAdmission } from "./postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +type TestDb = ReturnType; + +function record(primaryProjectId: string | null, referencedProjectIds: string[], contextRevision = "issue:1") { + const result = prepareAdmissionRecord({ primaryProjectId, referencedProjectIds, contextRevision }); + if (result.kind !== "ok") throw new Error(result.kind); + return result.record; +} + +describe.skipIf(!support.supported)("execution admission storage", () => { + let database: Awaited>; + let db: TestDb; + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("execution-admission-"); + db = createDb(database.connectionString); + }, 60_000); + + afterAll(async () => { + await db?.$client.end(); + await database?.cleanup(); + }); + + async function fixture() { + const [company] = await db.insert(companies).values({ name: "Admission fixture", issuePrefix: `A${randomUUID().slice(0, 6)}` }).returning(); + const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Admission agent", adapterType: "process" }).returning(); + const projectRows = await db.insert(projects).values(["Design", "Docs", "Unlimited"].map(name => ({ companyId: company.id, name }))) + .returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id }).returning(); + return { companyId: company.id, agentId: agent.id, projectIds: projectRows.map(row => row.id).sort(), runId: run.id }; + } + + const inTransaction = (work: (tx: TestDb) => Promise) => db.transaction(tx => work(tx as unknown as TestDb)); + const admit = (input: { companyId: string; runId: string; admissionId?: string; record: AdmissionRecord }) => + inTransaction(tx => recordAdmission(tx, { ...input, admissionId: input.admissionId ?? randomUUID() })); + const settle = (companyId: string, admissionId: string, settlementId = randomUUID()) => + inTransaction(tx => settleAdmission(tx, { companyId, admissionId, settlementId })); + const chargedProjects = async (admissionId: string) => (await db.select({ projectId: charges.projectId }).from(charges) + .where(eq(charges.admissionId, admissionId)).orderBy(asc(charges.projectId))).map(row => row.projectId); + + it("stores absent, explicit unlimited and positive maximums with partial updates", async () => { + const { companyId, projectIds: [projectId] } = await fixture(); + const policy = createExecutionAdmission(db); + const scope = { companyId, projectId }; + expect(await policy.readTaskExecutionPolicy(scope)).toEqual({ kind: "found", policy: { state: "absent", maxConcurrentTasks: null } }); + expect(await policy.updateTaskExecutionPolicy(scope, {})).toMatchObject({ kind: "unchanged", policy: { state: "absent" } }); + expect(await db.select().from(policies)).not.toContainEqual(expect.objectContaining({ projectId })); + + expect(await policy.updateTaskExecutionPolicy(scope, { maxConcurrentTasks: 2 })) + .toEqual({ kind: "updated", policy: { state: "configured", maxConcurrentTasks: 2 } }); + expect(await policy.updateTaskExecutionPolicy(scope, undefined)).toMatchObject({ kind: "unchanged", policy: { maxConcurrentTasks: 2 } }); + expect(await policy.updateTaskExecutionPolicy(scope, { maxConcurrentTasks: 0 })).toMatchObject({ kind: "invalid" }); + expect(await policy.updateTaskExecutionPolicy(scope, null)) + .toEqual({ kind: "updated", policy: { state: "configured", maxConcurrentTasks: null } }); + expect(await policy.readTaskExecutionPolicy(scope)).toEqual({ kind: "found", policy: { state: "configured", maxConcurrentTasks: null } }); + + const foreign = { companyId: (await fixture()).companyId, projectId }; + expect(await policy.readTaskExecutionPolicy(foreign)).toEqual({ kind: "project_not_found" }); + expect(await policy.updateTaskExecutionPolicy(foreign, { maxConcurrentTasks: 1 })).toEqual({ kind: "project_not_found" }); + expect(await policy.readTaskExecutionPolicy(scope)).toMatchObject({ policy: { maxConcurrentTasks: null } }); + }); + + it("enforces policy constraints and company scope in the database", async () => { + const { companyId, projectIds: [projectId] } = await fixture(); + const other = await fixture(); + await expect(db.insert(policies).values({ companyId, projectId, maxConcurrentTasks: 0 })) + .rejects.toMatchObject({ cause: { constraint_name: "project_task_execution_policies_maximum_check" } }); + await expect(db.insert(policies).values({ companyId: other.companyId, projectId, maxConcurrentTasks: 1 })) + .rejects.toMatchObject({ cause: { constraint_name: "project_task_execution_policies_company_project_fk" } }); + }); + + it("charges every distinct project, including unlimited and primaryless sets", async () => { + const { companyId, runId, projectIds: [a, b, c] } = await fixture(); + await createExecutionAdmission(db).updateTaskExecutionPolicy({ companyId, projectId: c }, null); + const admissionId = randomUUID(); + const admitted = record(a, [c, b, a]); + expect(await admit({ companyId, runId, admissionId, record: admitted })).toEqual({ kind: "recorded", attempt: 1 }); + expect(await chargedProjects(admissionId)).toEqual([a, b, c]); + const [epoch] = await db.select().from(admissions).where(eq(admissions.id, admissionId)); + expect(epoch).toMatchObject({ companyId, runId, attempt: 1, primaryProjectId: a, contextRevision: "issue:1", settlementId: null }); + + const other = await fixture(); + const primaryless = randomUUID(); + expect(await admit({ ...other, admissionId: primaryless, record: record(null, [other.projectIds[1]]) })) + .toMatchObject({ kind: "recorded" }); + expect(await chargedProjects(primaryless)).toEqual([other.projectIds[1]]); + + const empty = await fixture(); + const emptyId = randomUUID(); + expect(await admit({ ...empty, admissionId: emptyId, record: record(null, []) })).toMatchObject({ kind: "recorded" }); + expect(await chargedProjects(emptyId)).toEqual([]); + expect([...(await countActiveProjectCharges(db, empty.companyId, empty.projectIds)).values()]).toEqual([0, 0, 0]); + }); + + it("replays one admission identity and refuses a second live epoch for the run", async () => { + const { companyId, runId, projectIds: [a, b] } = await fixture(); + const admissionId = randomUUID(); + const admitted = record(a, [b]); + await admit({ companyId, runId, admissionId, record: admitted }); + expect(await admit({ companyId, runId, admissionId, record: admitted })).toEqual({ kind: "replayed", attempt: 1 }); + expect(await admit({ companyId, runId, admissionId, record: record(a, []) })).toEqual({ kind: "admission_conflict" }); + expect(await admit({ companyId, runId, admissionId, record: record(a, [b], "issue:2") })).toEqual({ kind: "admission_conflict" }); + expect(await admit({ companyId, runId, record: admitted })).toEqual({ kind: "run_already_admitted", admissionId }); + expect(await db.select({ id: admissions.id }).from(admissions).where(eq(admissions.runId, runId))).toHaveLength(1); + await expect(db.insert(admissions).values({ id: randomUUID(), companyId, runId, attempt: 2, contextRevision: "x" })) + .rejects.toMatchObject({ cause: { constraint_name: "execution_admissions_live_run_uq" } }); + }); + + it("settles the whole set once and starts a new attempt for the same run", async () => { + const { companyId, runId, projectIds: [a, b] } = await fixture(); + const first = randomUUID(); + await admit({ companyId, runId, admissionId: first, record: record(a, [b]) }); + const settlementId = randomUUID(); + expect(await settle(companyId, first, settlementId)).toEqual({ kind: "settled", releasedProjectIds: [a, b] }); + expect(await settle(companyId, first, settlementId)).toEqual({ kind: "already_settled" }); + expect(await settle(companyId, first)).toEqual({ kind: "settled_by_other" }); + expect(await settle(randomUUID(), first)).toEqual({ kind: "not_found" }); + expect(await chargedProjects(first)).toEqual([]); + expect(await admit({ companyId, runId, admissionId: first, record: record(a, [b]) })).toEqual({ kind: "admission_settled" }); + + const second = randomUUID(); + expect(await admit({ companyId, runId, admissionId: second, record: record(a, [], "issue:2") })).toEqual({ kind: "recorded", attempt: 2 }); + expect(await db.select({ attempt: admissions.attempt, settled: sql`${admissions.settledAt} is not null` }) + .from(admissions).where(eq(admissions.runId, runId)).orderBy(asc(admissions.attempt))) + .toEqual([{ attempt: 1, settled: true }, { attempt: 2, settled: false }]); + await expect(db.update(admissions).set({ settlementId: randomUUID() }).where(eq(admissions.id, second))) + .rejects.toMatchObject({ cause: { constraint_name: "execution_admissions_settlement_check" } }); + }); + + it("keeps company isolation and rolls back the whole record", async () => { + const { companyId, runId, projectIds: [a] } = await fixture(); + const other = await fixture(); + expect(await admit({ companyId, runId, record: record(a, [other.projectIds[0]]) })) + .toEqual({ kind: "projects_unavailable", projectIds: [other.projectIds[0]] }); + expect(await admit({ companyId, runId: other.runId, record: record(a, []) })).toEqual({ kind: "run_not_found" }); + const admissionId = randomUUID(); + await admit({ companyId, runId, admissionId, record: record(a, []) }); + expect(await admit({ companyId: other.companyId, runId: other.runId, admissionId, record: record(null, []) })) + .toEqual({ kind: "admission_conflict" }); + await expect(db.insert(charges).values({ admissionId, companyId, projectId: other.projectIds[0] })) + .rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_company_project_fk" } }); + await expect(db.insert(charges).values({ admissionId, companyId, projectId: a })) + .rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_admission_id_project_id_pk" } }); + + const rolledBack = randomUUID(); + await expect(inTransaction(async tx => { + await recordAdmission(tx, { companyId: other.companyId, runId: other.runId, admissionId: rolledBack, record: record(null, other.projectIds) }); + throw new Error("claim lost"); + })).rejects.toThrow("claim lost"); + expect(await db.select().from(admissions).where(eq(admissions.id, rolledBack))).toEqual([]); + expect(await chargedProjects(rolledBack)).toEqual([]); + await expect(recordAdmission(db, { companyId, runId, admissionId: randomUUID(), record: record(a, []) })) + .rejects.toThrow("requires a database transaction"); + }); + + it("counts active charges per project through the project index", async () => { + const first = await fixture(); + const [a, b, c] = first.projectIds; + const [secondRun] = await db.insert(heartbeatRuns).values({ companyId: first.companyId, agentId: first.agentId }).returning(); + const held = randomUUID(); + await admit({ ...first, admissionId: held, record: record(a, [b]) }); + await admit({ companyId: first.companyId, runId: secondRun.id, record: record(null, [a]) }); + expect(Object.fromEntries(await countActiveProjectCharges(db, first.companyId, [a, b, c]))).toEqual({ [a]: 2, [b]: 1, [c]: 0 }); + await settle(first.companyId, held); + expect(Object.fromEntries(await countActiveProjectCharges(db, first.companyId, [a, b, c]))).toEqual({ [a]: 1, [b]: 0, [c]: 0 }); + expect(Object.fromEntries(await countActiveProjectCharges(db, randomUUID(), [a]))).toEqual({ [a]: 0 }); + + const plan = await inTransaction(async tx => { + await tx.execute(sql`set local enable_seqscan = off`); + return tx.execute(sql`explain select project_id, count(*) from execution_admission_project_charges + where company_id = ${first.companyId} and project_id in (${a}, ${b}) group by project_id`); + }); + expect(JSON.stringify(plan)).toContain("execution_admission_project_charges_project_idx"); + }); + + it("preserves unsettled charges through project archive and delete", async () => { + const { companyId, runId, projectIds: [a, b] } = await fixture(); + const admissionId = randomUUID(); + await admit({ companyId, runId, admissionId, record: record(a, [b]) }); + const projectsService = projectService(db); + await db.update(projects).set({ archivedAt: new Date() }).where(eq(projects.id, b)); + expect(await chargedProjects(admissionId)).toEqual([a, b]); + await expect(projectsService.remove(a)).rejects.toMatchObject({ status: 409, details: { code: "project_execution_admission_pending" } }); + await expect(db.delete(projects).where(eq(projects.id, b))) + .rejects.toMatchObject({ cause: { constraint_name: "execution_admission_project_charges_company_project_fk" } }); + expect(await chargedProjects(admissionId)).toEqual([a, b]); + + await createExecutionAdmission(db).updateTaskExecutionPolicy({ companyId, projectId: a }, { maxConcurrentTasks: 1 }); + await settle(companyId, admissionId); + expect(await projectsService.remove(a)).toMatchObject({ id: a }); + expect(await db.select().from(policies).where(eq(policies.projectId, a))).toEqual([]); + expect(await db.select({ primaryProjectId: admissions.primaryProjectId }).from(admissions).where(eq(admissions.id, admissionId))) + .toEqual([{ primaryProjectId: a }]); + }); + + it("guards agent deletion while unsettled and removes admissions with the company", async () => { + const { companyId, agentId, runId, projectIds: [a] } = await fixture(); + const admissionId = randomUUID(); + await admit({ companyId, runId, admissionId, record: record(a, []) }); + await expect(withAccountingTransaction(db, companyId, tx => deleteAgentDependencies(tx, companyId, agentId))) + .rejects.toMatchObject({ status: 409, details: { code: "agent_execution_admission_pending" } }); + expect(await chargedProjects(admissionId)).toEqual([a]); + await settle(companyId, admissionId); + await withAccountingTransaction(db, companyId, tx => deleteAgentDependencies(tx, companyId, agentId)); + expect(await db.select().from(admissions).where(eq(admissions.runId, runId))).toEqual([]); + + const deleted = await fixture(); + const retained = await fixture(); + const retainedAdmission = randomUUID(); + await admit({ ...retained, admissionId: retainedAdmission, record: record(retained.projectIds[0], []) }); + await admit({ ...deleted, record: record(deleted.projectIds[0], deleted.projectIds) }); + await db.update(agents).set({ lifecycleState: "terminated", status: "terminated" }).where(eq(agents.id, deleted.agentId)); + expect(await deleteCompany(db, deleted.companyId)).toMatchObject({ id: deleted.companyId }); + expect(await db.select().from(admissions).where(eq(admissions.companyId, deleted.companyId))).toEqual([]); + expect(await chargedProjects(retainedAdmission)).toEqual([retained.projectIds[0]]); + }, 60_000); +}); diff --git a/server/src/modules/execution-admission/adapters/postgres.ts b/server/src/modules/execution-admission/adapters/postgres.ts new file mode 100644 index 00000000000..937eef72e14 --- /dev/null +++ b/server/src/modules/execution-admission/adapters/postgres.ts @@ -0,0 +1,165 @@ +import { + executionAdmissionProjectCharges as charges, + executionAdmissions as admissions, + heartbeatRuns, + projectTaskExecutionPolicies as policies, + projects, + type Db, +} from "@paperclipai/db"; +import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; +import type { TaskExecutionPolicyStore } from "../application/policy.js"; +import { decideSettlement, sameAdmissionRecord, type AdmissionRecord } from "../domain/admission.js"; +import { ABSENT_TASK_EXECUTION_POLICY, type TaskExecutionPolicy } from "../domain/policy.js"; + +function assertTransaction(tx: Db) { + if (!("nestedIndex" in tx)) throw new Error("Execution admission storage requires a database transaction"); +} + +/** + * Global lock order step 3: locks the whole project set in ascending ID order + * in one statement. Returns the IDs that are not projects of the company. + */ +export async function lockAdmissionProjects(tx: Db, companyId: string, projectIds: readonly string[]) { + assertTransaction(tx); + if (projectIds.length === 0) return []; + const rows = await tx.select({ id: projects.id }).from(projects) + .where(and(eq(projects.companyId, companyId), inArray(projects.id, [...projectIds]))) + .orderBy(asc(projects.id)).for("no key update"); + const found = new Set(rows.map(row => row.id)); + return projectIds.filter(id => !found.has(id)); +} + +function toPolicy(row: { maxConcurrentTasks: number | null } | undefined): TaskExecutionPolicy { + return row ? { state: "configured", maxConcurrentTasks: row.maxConcurrentTasks } : ABSENT_TASK_EXECUTION_POLICY; +} + +export function createPostgresTaskExecutionPolicyStore(db: Db): TaskExecutionPolicyStore { + return { + async read({ companyId, projectId }) { + const [row] = await db.select({ configured: policies.projectId, maxConcurrentTasks: policies.maxConcurrentTasks }) + .from(projects).leftJoin(policies, eq(policies.projectId, projects.id)) + .where(and(eq(projects.companyId, companyId), eq(projects.id, projectId))); + if (!row) return null; + return row.configured ? toPolicy(row) : ABSENT_TASK_EXECUTION_POLICY; + }, + + update: ({ companyId, projectId }, change) => db.transaction(async transaction => { + const tx = transaction as unknown as Db; + if ((await lockAdmissionProjects(tx, companyId, [projectId])).length > 0) return null; + const [row] = await tx.select({ maxConcurrentTasks: policies.maxConcurrentTasks }).from(policies) + .where(eq(policies.projectId, projectId)); + const current = toPolicy(row); + const next = change(current); + if (next === current) return current; + if (next.state === "absent") throw new Error("A stored task execution policy cannot become absent"); + await tx.insert(policies).values({ projectId, companyId, maxConcurrentTasks: next.maxConcurrentTasks }) + .onConflictDoUpdate({ target: policies.projectId, set: { maxConcurrentTasks: next.maxConcurrentTasks, updatedAt: sql`now()` } }); + return next; + }), + }; +} + +export type RecordAdmissionInput = { companyId: string; runId: string; admissionId: string; record: AdmissionRecord }; + +export type RecordAdmissionOutcome = + | { kind: "recorded" | "replayed"; attempt: number } + | { kind: "projects_unavailable"; projectIds: string[] } + | { kind: "run_not_found" } + | { kind: "run_already_admitted"; admissionId: string } + | { kind: "admission_settled" } + | { kind: "admission_conflict" }; + +/** + * Storage transition only: persists one admission attempt and its charges in + * the caller's transaction. It neither checks capacity nor claims work; the + * composed admission command must do both under the same locks. Takes lock + * steps 3 (projects) and 6 (run); composed callers take them earlier. + */ +export async function recordAdmission(tx: Db, input: RecordAdmissionInput): Promise { + assertTransaction(tx); + const { companyId, runId, admissionId, record } = input; + const missing = await lockAdmissionProjects(tx, companyId, record.projectIds); + if (missing.length > 0) return { kind: "projects_unavailable", projectIds: missing }; + const [run] = await tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId))).for("no key update"); + if (!run) return { kind: "run_not_found" }; + + const [existing] = await tx.select().from(admissions).where(eq(admissions.id, admissionId)); + if (existing) { + if (existing.companyId !== companyId || existing.runId !== runId) return { kind: "admission_conflict" }; + if (existing.settledAt) return { kind: "admission_settled" }; + const charged = await tx.select({ projectId: charges.projectId }).from(charges) + .where(and(eq(charges.companyId, companyId), eq(charges.admissionId, admissionId))).orderBy(asc(charges.projectId)); + const stored = { primaryProjectId: existing.primaryProjectId, contextRevision: existing.contextRevision, + projectIds: charged.map(row => row.projectId) }; + return sameAdmissionRecord(stored, record) ? { kind: "replayed", attempt: existing.attempt } : { kind: "admission_conflict" }; + } + const [live] = await tx.select({ id: admissions.id }).from(admissions) + .where(and(eq(admissions.runId, runId), isNull(admissions.settledAt))); + if (live) return { kind: "run_already_admitted", admissionId: live.id }; + + const [{ attempt }] = await tx.select({ attempt: sql`coalesce(max(${admissions.attempt}), 0)::int + 1` }) + .from(admissions).where(eq(admissions.runId, runId)); + await tx.insert(admissions).values({ id: admissionId, companyId, runId, attempt, + primaryProjectId: record.primaryProjectId, contextRevision: record.contextRevision }); + if (record.projectIds.length > 0) { + await tx.insert(charges).values(record.projectIds.map(projectId => ({ admissionId, companyId, projectId }))); + } + return { kind: "recorded", attempt }; +} + +export type SettleAdmissionOutcome = + | { kind: "settled"; releasedProjectIds: string[] } + | { kind: "already_settled" | "settled_by_other" | "not_found" }; + +/** + * Storage transition only: releases the admission's whole charge set once. + * Callers must first verify authoritative lifecycle facts. Takes no project lock. + */ +export async function settleAdmission(tx: Db, input: { companyId: string; admissionId: string; settlementId: string }): +Promise { + assertTransaction(tx); + const { companyId, admissionId, settlementId } = input; + const [row] = await tx.select({ settlementId: admissions.settlementId }).from(admissions) + .where(and(eq(admissions.companyId, companyId), eq(admissions.id, admissionId))).for("no key update"); + if (!row) return { kind: "not_found" }; + const decision = decideSettlement(row.settlementId, settlementId); + if (decision !== "settle") return { kind: decision }; + const released = await tx.delete(charges) + .where(and(eq(charges.companyId, companyId), eq(charges.admissionId, admissionId))) + .returning({ projectId: charges.projectId }); + await tx.update(admissions).set({ settlementId, settledAt: sql`now()` }).where(eq(admissions.id, admissionId)); + return { kind: "settled", releasedProjectIds: released.map(charge => charge.projectId).sort() }; +} + +/** Active charge counts per project. Every stored charge is unsettled. */ +export async function countActiveProjectCharges(db: Db, companyId: string, projectIds: readonly string[]) { + const counts = new Map(projectIds.map(id => [id, 0])); + if (projectIds.length === 0) return counts; + const rows = await db.select({ projectId: charges.projectId, active: sql`count(*)::int` }).from(charges) + .where(and(eq(charges.companyId, companyId), inArray(charges.projectId, [...projectIds]))) + .groupBy(charges.projectId); + for (const row of rows) counts.set(row.projectId, row.active); + return counts; +} + +/** Project deletion holds the project row lock, so no charge can be added concurrently. */ +export async function hasUnsettledProjectCharges(db: Db, scope: { companyId: string; projectId: string }) { + const [row] = await db.select({ admissionId: charges.admissionId }).from(charges) + .where(and(eq(charges.companyId, scope.companyId), eq(charges.projectId, scope.projectId))).limit(1); + return Boolean(row); +} + +/** Removes settled history of an agent's runs before they are deleted; refuses while any admission is unsettled. */ +export async function deleteAgentAdmissionHistory(tx: Db, scope: { companyId: string; agentId: string }) { + assertTransaction(tx); + const agentRuns = tx.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, scope.companyId), eq(heartbeatRuns.agentId, scope.agentId))); + const [live] = await tx.select({ id: admissions.id }).from(admissions) + .where(and(eq(admissions.companyId, scope.companyId), inArray(admissions.runId, agentRuns), isNull(admissions.settledAt))) + .limit(1); + if (live) return { kind: "unsettled" as const }; + await tx.delete(admissions).where(and(eq(admissions.companyId, scope.companyId), inArray(admissions.runId, agentRuns))); + return { kind: "deleted" as const }; +} + diff --git a/server/src/modules/execution-admission/application/policy.ts b/server/src/modules/execution-admission/application/policy.ts new file mode 100644 index 00000000000..3498b31102a --- /dev/null +++ b/server/src/modules/execution-admission/application/policy.ts @@ -0,0 +1,50 @@ +import { + applyTaskExecutionPolicyPatch, + parseTaskExecutionPolicyPatch, + type TaskExecutionPolicy, +} from "../domain/policy.js"; + +export type ProjectScope = { companyId: string; projectId: string }; + +export interface TaskExecutionPolicyStore { + /** Null when the project is not in the company. */ + read(scope: ProjectScope): Promise; + /** + * Locks the project in admission lock order and stores `change(current)`. + * Returning `current` itself writes nothing. Null when the project is not in the company. + */ + update(scope: ProjectScope, change: (current: TaskExecutionPolicy) => TaskExecutionPolicy): Promise; +} + +export type PolicyReadResult = { kind: "found"; policy: TaskExecutionPolicy } | { kind: "project_not_found" }; + +export type PolicyUpdateResult = + | { kind: "updated" | "unchanged"; policy: TaskExecutionPolicy } + | { kind: "invalid"; field: "policy" | "maxConcurrentTasks"; reason: string } + | { kind: "project_not_found" }; + +/** + * Internal policy commands. No route, tool or import calls `update` until + * admission enforces the maximum; a cap written earlier would mislead users. + */ +export function createTaskExecutionPolicyCommands(store: TaskExecutionPolicyStore) { + return { + async readTaskExecutionPolicy(scope: ProjectScope): Promise { + const policy = await store.read(scope); + return policy ? { kind: "found", policy } : { kind: "project_not_found" }; + }, + + async updateTaskExecutionPolicy(scope: ProjectScope, input: unknown): Promise { + const parsed = parseTaskExecutionPolicyPatch(input); + if (parsed.kind === "invalid") return parsed; + let changed = false; + const policy = await store.update(scope, current => { + const next = applyTaskExecutionPolicyPatch(current, parsed.patch); + changed = next !== current; + return next; + }); + if (!policy) return { kind: "project_not_found" }; + return { kind: changed ? "updated" : "unchanged", policy }; + }, + }; +} diff --git a/server/src/modules/execution-admission/company-deletion.ts b/server/src/modules/execution-admission/company-deletion.ts new file mode 100644 index 00000000000..cfaf4097f2a --- /dev/null +++ b/server/src/modules/execution-admission/company-deletion.ts @@ -0,0 +1 @@ +export { executionAdmissionCompanyDeletion } from "./adapters/company-deletion.js"; diff --git a/server/src/modules/execution-admission/domain/admission.ts b/server/src/modules/execution-admission/domain/admission.ts new file mode 100644 index 00000000000..4177e9c4c06 --- /dev/null +++ b/server/src/modules/execution-admission/domain/admission.ts @@ -0,0 +1,53 @@ +/** Referenced projects are rejected beyond this bound instead of admitting a subset. */ +export const MAX_ADMISSION_PROJECTS = 100; +const MAX_CONTEXT_REVISION_LENGTH = 256; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +export type AdmissionRecordInput = { + primaryProjectId: string | null; + referencedProjectIds: readonly string[]; + contextRevision: string; +}; + +/** + * `projectIds` is the complete distinct set in canonical lock order. Lowercase + * UUID text order matches PostgreSQL uuid order. An empty set is valid and + * charges no project. + */ +export type AdmissionRecord = { + primaryProjectId: string | null; + projectIds: string[]; + contextRevision: string; +}; + +export type AdmissionRecordResult = + | { kind: "ok"; record: AdmissionRecord } + | { kind: "invalid_project_id" } + | { kind: "too_many_projects"; count: number; limit: number } + | { kind: "invalid_context_revision" }; + +export function prepareAdmissionRecord(input: AdmissionRecordInput): AdmissionRecordResult { + const contextRevision = input.contextRevision.trim(); + if (!contextRevision || contextRevision.length > MAX_CONTEXT_REVISION_LENGTH) return { kind: "invalid_context_revision" }; + const primaryProjectId = input.primaryProjectId?.toLowerCase() ?? null; + const candidates = [...input.referencedProjectIds, ...(primaryProjectId === null ? [] : [primaryProjectId])].map(id => id.toLowerCase()); + if (candidates.some(id => !UUID.test(id))) return { kind: "invalid_project_id" }; + const projectIds = [...new Set(candidates)].sort(); + if (projectIds.length > MAX_ADMISSION_PROJECTS) { + return { kind: "too_many_projects", count: projectIds.length, limit: MAX_ADMISSION_PROJECTS }; + } + return { kind: "ok", record: { primaryProjectId, projectIds, contextRevision } }; +} + +export function sameAdmissionRecord(left: AdmissionRecord, right: AdmissionRecord) { + return left.primaryProjectId === right.primaryProjectId && left.contextRevision === right.contextRevision + && left.projectIds.length === right.projectIds.length && left.projectIds.every((id, index) => id === right.projectIds[index]); +} + +export type SettlementDecision = "settle" | "already_settled" | "settled_by_other"; + +/** A settlement identity can be replayed; a second, different settlement cannot take over. */ +export function decideSettlement(existingSettlementId: string | null, settlementId: string): SettlementDecision { + if (existingSettlementId === null) return "settle"; + return existingSettlementId === settlementId ? "already_settled" : "settled_by_other"; +} diff --git a/server/src/modules/execution-admission/domain/policy.test.ts b/server/src/modules/execution-admission/domain/policy.test.ts new file mode 100644 index 00000000000..0c5ce8ebd6d --- /dev/null +++ b/server/src/modules/execution-admission/domain/policy.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; +import { decideSettlement, MAX_ADMISSION_PROJECTS, prepareAdmissionRecord } from "./admission.js"; +import { ABSENT_TASK_EXECUTION_POLICY, applyTaskExecutionPolicyPatch, parseTaskExecutionPolicyPatch, type TaskExecutionPolicy } from "./policy.js"; + +const configured = (maxConcurrentTasks: number | null): TaskExecutionPolicy => ({ state: "configured", maxConcurrentTasks }); +const apply = (current: TaskExecutionPolicy, input: unknown) => { + const parsed = parseTaskExecutionPolicyPatch(input); + if (parsed.kind !== "ok") throw new Error(parsed.reason); + return applyTaskExecutionPolicyPatch(current, parsed.patch); +}; + +describe("task execution policy", () => { + it("keeps never-configured distinct from explicit unlimited", () => { + expect(apply(ABSENT_TASK_EXECUTION_POLICY, null)).toEqual(configured(null)); + expect(apply(ABSENT_TASK_EXECUTION_POLICY, { maxConcurrentTasks: null })).toEqual(configured(null)); + expect(apply(ABSENT_TASK_EXECUTION_POLICY, { maxConcurrentTasks: 3 })).toEqual(configured(3)); + }); + + it("preserves the current value for omitted policies and fields", () => { + for (const current of [ABSENT_TASK_EXECUTION_POLICY, configured(null), configured(2)]) { + expect(apply(current, undefined)).toBe(current); + expect(apply(current, {})).toBe(current); + expect(apply(current, { maxConcurrentTasks: undefined })).toBe(current); + } + const two = configured(2); + expect(apply(two, { maxConcurrentTasks: 2 })).toBe(two); + const unlimited = configured(null); + expect(apply(unlimited, null)).toBe(unlimited); + }); + + it("clears or replaces a configured maximum", () => { + expect(apply(configured(2), null)).toEqual(configured(null)); + expect(apply(configured(2), { maxConcurrentTasks: null })).toEqual(configured(null)); + expect(apply(configured(null), { maxConcurrentTasks: 1 })).toEqual(configured(1)); + }); + + it.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, 2_147_483_648, "2", true, {}])( + "rejects maximum %s", value => { + expect(parseTaskExecutionPolicyPatch({ maxConcurrentTasks: value })).toMatchObject({ kind: "invalid", field: "maxConcurrentTasks" }); + }, + ); + + it("rejects malformed policies and unknown fields", () => { + for (const input of [1, "unlimited", [], { maxConcurrentTasks: 1, perTaskOverride: 3 }]) { + expect(parseTaskExecutionPolicyPatch(input)).toMatchObject({ kind: "invalid", field: "policy" }); + } + expect(parseTaskExecutionPolicyPatch({ maxConcurrentTasks: 2_147_483_647 })).toEqual({ kind: "ok", patch: { maxConcurrentTasks: 2_147_483_647 } }); + }); +}); + +describe("admission record", () => { + const a = "0b9d8e52-6f2e-4bb0-9a4a-1f0a7f2e0a01"; + const b = "7c1f3d9a-2b4e-4d8f-8a6b-3e5c7d9f1b02"; + const c = "f3a2b1c0-9d8e-4f7a-8b6c-5d4e3f2a1b03"; + + it("unions the primary with referenced projects in canonical order", () => { + expect(prepareAdmissionRecord({ primaryProjectId: c, referencedProjectIds: [b, a, b.toUpperCase(), c], contextRevision: " r1 " })) + .toEqual({ kind: "ok", record: { primaryProjectId: c, projectIds: [a, b, c], contextRevision: "r1" } }); + }); + + it("keeps a primaryless set and an empty set without a project bucket", () => { + expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [b], contextRevision: "r" })) + .toEqual({ kind: "ok", record: { primaryProjectId: null, projectIds: [b], contextRevision: "r" } }); + expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [], contextRevision: "r" })) + .toEqual({ kind: "ok", record: { primaryProjectId: null, projectIds: [], contextRevision: "r" } }); + }); + + it("rejects invalid references, oversized sets and missing context identity instead of narrowing", () => { + expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: [a, "Design"], contextRevision: "r" })) + .toEqual({ kind: "invalid_project_id" }); + expect(prepareAdmissionRecord({ primaryProjectId: "", referencedProjectIds: [a], contextRevision: "r" })) + .toEqual({ kind: "invalid_project_id" }); + const many = Array.from({ length: MAX_ADMISSION_PROJECTS + 1 }, (_, index) => + `00000000-0000-4000-8000-${index.toString(16).padStart(12, "0")}`); + expect(prepareAdmissionRecord({ primaryProjectId: null, referencedProjectIds: many, contextRevision: "r" })) + .toEqual({ kind: "too_many_projects", count: MAX_ADMISSION_PROJECTS + 1, limit: MAX_ADMISSION_PROJECTS }); + expect(prepareAdmissionRecord({ primaryProjectId: a, referencedProjectIds: [], contextRevision: " " })) + .toEqual({ kind: "invalid_context_revision" }); + expect(prepareAdmissionRecord({ primaryProjectId: a, referencedProjectIds: [], contextRevision: "x".repeat(257) })) + .toEqual({ kind: "invalid_context_revision" }); + }); + + it("settles once and replays only the same settlement", () => { + expect(decideSettlement(null, a)).toBe("settle"); + expect(decideSettlement(a, a)).toBe("already_settled"); + expect(decideSettlement(a, b)).toBe("settled_by_other"); + }); +}); diff --git a/server/src/modules/execution-admission/domain/policy.ts b/server/src/modules/execution-admission/domain/policy.ts new file mode 100644 index 00000000000..e171011d3b2 --- /dev/null +++ b/server/src/modules/execution-admission/domain/policy.ts @@ -0,0 +1,47 @@ +/** + * A project's task-execution policy. `absent` was never configured, so a later + * legacy migration may still choose a value. `configured` with a null maximum + * is an explicit "no limit" that such a migration must preserve. + */ +export type TaskExecutionPolicy = + | { state: "absent"; maxConcurrentTasks: null } + | { state: "configured"; maxConcurrentTasks: number | null }; + +/** Null clears the cap explicitly. An omitted field preserves its value. */ +export type TaskExecutionPolicyPatch = { maxConcurrentTasks?: number | null } | null; + +export type PolicyPatchResult = + | { kind: "ok"; patch: TaskExecutionPolicyPatch } + | { kind: "invalid"; field: "policy" | "maxConcurrentTasks"; reason: string }; + +export const ABSENT_TASK_EXECUTION_POLICY: TaskExecutionPolicy = { state: "absent", maxConcurrentTasks: null }; + +// The column is a PostgreSQL integer. +const MAX_STORED_MAXIMUM = 2_147_483_647; + +/** Undefined (an omitted policy) preserves everything; null clears the cap. */ +export function parseTaskExecutionPolicyPatch(input: unknown): PolicyPatchResult { + if (input === undefined) return { kind: "ok", patch: {} }; + if (input === null) return { kind: "ok", patch: null }; + if (typeof input !== "object" || Array.isArray(input)) { + return { kind: "invalid", field: "policy", reason: "Policy must be an object or null" }; + } + const unknownKey = Object.keys(input).find(key => key !== "maxConcurrentTasks"); + if (unknownKey) return { kind: "invalid", field: "policy", reason: `Unknown policy field ${unknownKey}` }; + if (!("maxConcurrentTasks" in input)) return { kind: "ok", patch: {} }; + const value = (input as { maxConcurrentTasks: unknown }).maxConcurrentTasks; + if (value === undefined) return { kind: "ok", patch: {} }; + if (value === null) return { kind: "ok", patch: { maxConcurrentTasks: null } }; + if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > MAX_STORED_MAXIMUM) { + return { kind: "invalid", field: "maxConcurrentTasks", reason: "Maximum concurrent tasks must be a positive whole number" }; + } + return { kind: "ok", patch: { maxConcurrentTasks: value } }; +} + +/** Applies a validated patch. Returns the current policy when nothing changes. */ +export function applyTaskExecutionPolicyPatch(current: TaskExecutionPolicy, patch: TaskExecutionPolicyPatch): TaskExecutionPolicy { + if (patch !== null && patch.maxConcurrentTasks === undefined) return current; + const maxConcurrentTasks = patch === null ? null : patch.maxConcurrentTasks ?? null; + if (current.state === "configured" && current.maxConcurrentTasks === maxConcurrentTasks) return current; + return { state: "configured", maxConcurrentTasks }; +} diff --git a/server/src/modules/execution-admission/index.ts b/server/src/modules/execution-admission/index.ts new file mode 100644 index 00000000000..85878c38a05 --- /dev/null +++ b/server/src/modules/execution-admission/index.ts @@ -0,0 +1,17 @@ +import type { Db } from "@paperclipai/db"; +import { createPostgresTaskExecutionPolicyStore } from "./adapters/postgres.js"; +import { createTaskExecutionPolicyCommands } from "./application/policy.js"; + +export type { TaskExecutionPolicy, TaskExecutionPolicyPatch } from "./domain/policy.js"; +export type { PolicyReadResult, PolicyUpdateResult, ProjectScope } from "./application/policy.js"; +// Lifecycle guards run inside the caller's deletion transaction. +export { deleteAgentAdmissionHistory, hasUnsettledProjectCharges } from "./adapters/postgres.js"; + +/** + * Execution admission owns project task-execution policy and the durable + * admission epochs that charge project capacity. Admission and settlement + * commands are not composed yet; nothing enforces the maximum. + */ +export function createExecutionAdmission(db: Db) { + return createTaskExecutionPolicyCommands(createPostgresTaskExecutionPolicyStore(db)); +} diff --git a/server/src/realtime/activity-toast-identity.ts b/server/src/realtime/activity-toast-identity.ts index 414f46e199f..b845c0630c5 100644 --- a/server/src/realtime/activity-toast-identity.ts +++ b/server/src/realtime/activity-toast-identity.ts @@ -1,5 +1,6 @@ import { and, eq } from "drizzle-orm"; -import { agents, authUsers, companyMemberships, type Db } from "@paperclipai/db"; +import { agents, authUsers, companyMemberships, issues, type Db } from "@paperclipai/db"; +import { logger } from "../middleware/logger.js"; const TOAST_ACTIONS = new Set(["issue.created", "issue.updated", "issue.comment_added"]); @@ -11,8 +12,26 @@ export async function activityToastIdentity( ) { const actorType = typeof payload.actorType === "string" ? payload.actorType : null; const actorId = typeof payload.actorId === "string" ? payload.actorId : null; - const identity = { actorType, actorId }; - if (!actorId || !TOAST_ACTIONS.has(String(payload.action))) return identity; + const actorIdentity = { actorType, actorId }; + if (!TOAST_ACTIONS.has(String(payload.action))) return actorIdentity; + + const issueId = payload.entityType === "issue" && typeof payload.entityId === "string" + ? payload.entityId : null; + let issueIdentifier: string | null = null; + if (issueId) { + try { + const [issue] = await db + .select({ identifier: issues.identifier }) + .from(issues) + .where(and(eq(issues.companyId, companyId), eq(issues.id, issueId))) + .limit(1); + issueIdentifier = issue?.identifier ?? null; + } catch (err) { + logger.warn({ err, companyId, issueId }, "task toast identifier lookup failed"); + } + } + const identity = { ...actorIdentity, ...(issueId ? { issueIdentifier } : {}) }; + if (!actorId) return identity; if (actorType === "agent") { const [agent] = await db diff --git a/server/src/realtime/live-event-delivery.test.ts b/server/src/realtime/live-event-delivery.test.ts index 02e8c536c83..2f8b4c4eac4 100644 --- a/server/src/realtime/live-event-delivery.test.ts +++ b/server/src/realtime/live-event-delivery.test.ts @@ -1,6 +1,7 @@ import { afterEach, expect, it, vi } from "vitest"; import type { LiveEvent } from "@paperclipai/shared"; import { createLiveEventDelivery, LIVE_DELIVERY_MAX_BYTES, LIVE_DELIVERY_MAX_ITEMS } from "./live-event-delivery.js"; +import { idleWorkSnapshot } from "../services/task-admission.js"; const event = (seq = 1, runId = "run-1"): LiveEvent => ({ id: seq, companyId: "company", type: "heartbeat.run.log", createdAt: "2026-10-24T00:00:00Z", payload: { runId, agentId: "agent", seq, chunk: "output" } }); @@ -116,7 +117,7 @@ it.each([false, true])("preserves the overflow deadline across resubscription un await vi.advanceTimersByTimeAsync(1_000); if (settled) expect(f.socket.close).not.toHaveBeenCalled(); else expect(f.socket.close).toHaveBeenCalledWith(1013, "persistent event backlog"); - } finally { f.delivery.dispose(); } + } finally { f.delivery.dispose(); release(event()); await settle(); } expect(vi.getTimerCount()).toBe(0); }); it("recovers failed delivery reads through resync rather than silently claiming healthy streaming", async () => { @@ -146,6 +147,7 @@ it("rejects unbounded interest sets and stale revision commands", () => { it.each([false, true].flatMap(negotiated => ["authorize", "decorate"].map(stage => ({ negotiated, stage }))))( "closes a stalled $stage read below the queue bound (negotiated=$negotiated)", async ({ negotiated, stage }) => { vi.useFakeTimers(); + const baseline = idleWorkSnapshot().active; let release!: (value: LiveEvent) => void; const pending = new Promise(resolve => { release = resolve; }); const f = fixture(stage === "authorize" ? vi.fn(() => pending) : undefined); @@ -168,11 +170,13 @@ it.each([false, true].flatMap(negotiated => ["authorize", "decorate"].map(stage expect(f.socket.close).toHaveBeenCalledWith(1013, "persistent event backlog"); expect(f.delivery.metrics().retainedItems).toBe(1); expect(vi.getTimerCount()).toBe(0); + expect(idleWorkSnapshot().active).toBe(baseline + 1); release(source); await settle(); + expect(idleWorkSnapshot().active).toBe(baseline); expect(f.delivery.metrics()).toMatchObject({ retainedItems: 0, retainedBytes: 0 }); expect(f.frames.filter(frame => frame.type.startsWith("heartbeat."))).toEqual([]); expect(f.authorize).toHaveBeenCalledOnce(); - } finally { f.delivery.dispose(); other.delivery.dispose(); } + } finally { f.delivery.dispose(); other.delivery.dispose(); release(event()); await settle(); } }); it.each(["resolve", "dispose"])("clears a pending-read deadline on %s without idle work", async ending => { diff --git a/server/src/realtime/live-event-delivery.ts b/server/src/realtime/live-event-delivery.ts index 3d7f4371779..588eb654a35 100644 --- a/server/src/realtime/live-event-delivery.ts +++ b/server/src/realtime/live-event-delivery.ts @@ -1,4 +1,5 @@ import type { LiveEvent } from "@paperclipai/shared"; +import { beginIdleTrackedWork } from "../services/task-admission.js"; export const LIVE_DELIVERY_MAX_ITEMS = 1_000; export const LIVE_DELIVERY_MAX_BYTES = 1024 * 1024; @@ -90,6 +91,7 @@ export function createLiveEventDelivery(input: { async function drain() { if (draining) return; draining = true; + const finishIdleWork = beginIdleTrackedWork(); try { while (queued.length && isOpen()) { const item = queued.shift()!; @@ -121,7 +123,10 @@ export function createLiveEventDelivery(input: { retainedBytes -= item.bytes; retainedItems -= 1; } } - } finally { draining = false; resync(); } + } finally { + try { draining = false; resync(); } + finally { finishIdleWork(); } + } } return { receive(raw: string) { diff --git a/server/src/routes/adapters.ts b/server/src/routes/adapters.ts index 7a02043644b..94e0ab10c97 100644 --- a/server/src/routes/adapters.ts +++ b/server/src/routes/adapters.ts @@ -48,7 +48,8 @@ import type { } from "@paperclipai/adapter-utils"; import { loadExternalAdapterPackage, getUiParserSource, getOrExtractUiParserSource, reloadExternalAdapter } from "../adapters/plugin-loader.js"; import { logger } from "../middleware/logger.js"; -import { badRequest, forbidden } from "../errors.js"; +import { badRequest, forbidden, HttpError } from "../errors.js"; +import { isCloudManagedInstance } from "../services/cloud-instance.js"; import { assertExternalAdapterCodeAllowed } from "../services/adapter-code-policy.js"; import { getHiddenSettings } from "../services/settings-visibility.js"; import { assertBoardOrgAccess, assertCompanyAccess, assertInstanceAdmin, getActorInfo } from "./authz.js"; @@ -281,7 +282,17 @@ export function adapterRoutes(options: { if (typeof companyId === "string") { assertCompanyAccess(_req, companyId); if (!options.resolveRunnerTarget) throw badRequest("Scoped adapter discovery is unavailable."); - target = await options.resolveRunnerTarget(companyId, typeof environmentId === "string" ? environmentId : undefined); + try { + target = await options.resolveRunnerTarget(companyId, typeof environmentId === "string" ? environmentId : undefined); + } catch (error) { + // API controllers do not need a Computer. Keep their catalog visible + // when only the implicit managed Computer is unavailable. This target + // grants no local execution support to any workspace harness. + if (environmentId !== undefined || !(error instanceof HttpError) || error.status !== 422 + || (error.details as { code?: unknown } | undefined)?.code !== "managed_sandbox_unavailable") throw error; + target = { driver: "unavailable", ...((error.details as { executionMode?: unknown }).executionMode === "kubernetes" + ? { executionMode: "kubernetes" as const } : {}) }; + } } await waitForExternalAdapters(); diff --git a/server/src/routes/agents.ts b/server/src/routes/agents.ts index 08508aafaae..dfda077ea7c 100644 --- a/server/src/routes/agents.ts +++ b/server/src/routes/agents.ts @@ -1,3 +1,4 @@ +import { assertProviderApiPlacementPolicy, managedCursorApiDefaultEnabled, resolveProviderApiPlacement } from "../services/native-runtime/provider-api-placement.js"; import { readTaskPanelActivations } from "../services/plugin-task-panels.js"; import { enqueueNativeRuntimeResponse } from "../services/native-runtime/native-runtime-response-delivery.js"; import { latestFailedRunsByAgent } from "@paperclipai/shared"; @@ -1961,6 +1962,9 @@ export function agentRoutes( return input.nextAdapterConfig; } if (["openai_dot", "muse"].includes(String(input.nextAdapterConfig.provider))) return input.nextAdapterConfig; + // A provisional setup harness supplies no Cursor model or permissions. + // Legacy Cursor conversion still retains its provider-specific defaults. + if (input.nextAdapterConfig.provider === "cursor_cloud" && input.previousAdapterType !== "cursor_cloud") return input.nextAdapterConfig; const defaults = paperclipRunnerTransitionConfig(input.previousAdapterType, input.previousAdapterConfig.model, input.nextAdapterConfig.provider); if (!["claude_local", "codex_local", "opencode_local", "pi_local", "grok_local"].includes(input.previousAdapterType) && !isPaperclipRunnerProvider(input.nextAdapterConfig.provider)) { @@ -2206,6 +2210,7 @@ export function agentRoutes( ); const incompleteModelSetup = input.setupOnly && input.adapterType === "paperclip_runner" && (normalizedAdapterConfig.provider === "opencode" + || normalizedAdapterConfig.provider === "cursor_cloud" || (normalizedAdapterConfig.provider === "acpx" && ["pi", "kimi"].includes(String(normalizedAdapterConfig.acpxAgent)))) && (normalizedAdapterConfig.model === undefined || normalizedAdapterConfig.model === ""); // Inactive drafts can precede model selection. Test/hire always validate the complete profile. @@ -3275,7 +3280,25 @@ export function agentRoutes( } const testingSetup = savedAgent ? isAgentSetupDraft(savedAgent) : false; const poolBinding = requestedBinding ?? (savedAgent?.runtimeConfig?.aiConnection ? aiRuntimeConnectionBindingSchema.parse(savedAgent.runtimeConfig.aiConnection) : undefined); - const requestedEnvironmentId = await resolveAdapterTestEnvironmentId( + const prospectiveCursorDefault = type === "cursor_cloud" && !savedAgent + && (req.body.runner === undefined || req.body.runner === "auto") + && managedCursorApiDefaultEnabled((await instanceSettings.get()).experimental); + const testRunner = prospectiveCursorDefault ? "paperclip" : req.body.runner; + let providerApiTest; + try { + // An explicit Legacy switch must resolve its Computer and credentials + // before runner selection converts the saved native adapter. + providerApiTest = testRunner === "legacy" ? null : resolveProviderApiPlacement({ agent: { + adapterType: type === "cursor_cloud" && testRunner === "paperclip" ? "paperclip_runner" : type, + adapterConfig: type === "cursor_cloud" && testRunner === "paperclip" + ? { ...inputAdapterConfig, provider: "cursor_cloud" } : inputAdapterConfig, + } }); + } catch (error) { + if (error instanceof PaperclipRunnerProviderProfileError) throw unprocessable(error.message, { code: error.code }); + throw error; + } + if (providerApiTest) assertProviderApiPlacementPolicy(await instanceSettings.getGeneral()); + const requestedEnvironmentId = providerApiTest ? null : await resolveAdapterTestEnvironmentId( companyId, // Omission tests the saved selection. Explicit null tests a prospective // change back to the instance default. @@ -3321,7 +3344,7 @@ export function agentRoutes( ? { adapterType: type, adapterConfig: adapterConfigForTest } : await resolveNewAgentRunnerForCompany(db, companyId, { adapterType: type, adapterConfig: adapterConfigForTest, - runner: req.body.runner, + runner: testRunner, defaultEnvironmentId: requestedEnvironmentId, }); type = selection.adapterType; diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index 66123a51e52..e5d9c9f648e 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -1,3 +1,4 @@ +import { verifiedIdleCapabilities } from "../middleware/cloud-control.js"; import { trackIdleWork } from "../services/task-admission.js"; import { Router, type Request } from "express"; import { companies, type Db } from "@paperclipai/db"; @@ -336,7 +337,8 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, diagnostic => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup), diagnostic }), pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup, scheduledRoutines, scheduledIssueMonitors, - ...(req.query.idleSleepDiagnostics === "1" ? [{ collectDiagnostics: true }] as const : [] as const)); + { incomingWake: verifiedIdleCapabilities(req), deadlineCapability: verifiedIdleCapabilities(req), + ...(req.query.idleSleepDiagnostics === "1" ? { collectDiagnostics: true } : {}) }); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } diff --git a/server/src/services/acknowledged-native-stop.ts b/server/src/services/acknowledged-native-stop.ts index e937f3df2a7..09124602f0d 100644 --- a/server/src/services/acknowledged-native-stop.ts +++ b/server/src/services/acknowledged-native-stop.ts @@ -1,7 +1,12 @@ -import { and, eq } from "drizzle-orm"; -import { environmentLeases, heartbeatRuns, type Db } from "@paperclipai/db"; +import { and, desc, eq, inArray, isNotNull } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { parseObject } from "../adapters/utils.js"; +import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { hasNativeLocalProcessStop } from "./native-local-process-stop.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { validatePrpEvent } from "../vendor/paperclip-runner/index.js"; +import { nativeSha256 } from "./native-runtime/canonical.js"; +import { isConfirmedCursorCloudTerminalObservation } from "./native-runtime/cursor-cloud-runtime.js"; /** A server-recorded run-only Stop must not manufacture a recovery incident. */ export function hasAcknowledgedNativeRunStopReceipt(run: { @@ -37,6 +42,49 @@ export function isAcknowledgedNativeStop(run: Parameters): Promise { + if (run.runtimeMode !== "native" || run.driverKind !== "cursor_cloud_api" || !isAcknowledgedNativeStop(run) || + run.processPid !== null || run.processGroupId !== null || !run.nativeIssueId || !run.nativeSessionId || !run.runnerInstanceId) return false; + const intent = parseObject(run.resultJson?.nativeCancellation); + const requestedAt = typeof intent.recordedAt === "string" ? Date.parse(intent.recordedAt) : NaN; + if (!Number.isFinite(requestedAt) || !Array.isArray(intent.effects) || intent.effects.length !== 1 || + intent.effects[0] !== "release_run_resources" || + !String(intent.intentAuditId ?? "").trim() || !String(intent.acknowledgementAuditId ?? "").trim()) return false; + const profile = parseObject(run.runnerProfileJson); + const execution = parseObject(profile.nativeExecutionInput); + const binding = parseObject(execution.binding), session = parseObject(execution.session); + if (profile.executionPlacement === undefined || profile.backend !== "cursor_cloud_api" || + binding.companyId !== run.companyId || binding.runId !== run.id || binding.issueId !== run.nativeIssueId || + binding.agentId !== run.agentId || session.normalizedSessionId !== run.nativeSessionId || session.driverKind !== "cursor_cloud_api") return false; + try { + if (!resolveProviderApiPlacement({ run, agent: { adapterType: "process", adapterConfig: {} } })) return false; + } catch { return false; } + const [lease] = await db.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + )).limit(1); + if (lease || !await hasNativeLocalProcessStop(db, run.companyId, run.id)) return false; + const [terminal] = await db.select().from(heartbeatRunEvents).where(and( + eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), + isNotNull(heartbeatRunEvents.sourceEventId), + inArray(heartbeatRunEvents.eventType, ["turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]), + )).orderBy(desc(heartbeatRunEvents.seq)).limit(1); + const validated = validatePrpEvent(parseObject(terminal?.payload).prpEvent); + if (!terminal || !validated.ok || terminal.sourcePayloadSha256 !== nativeSha256(validated.event)) return false; + const event = validated.event; + return isConfirmedCursorCloudTerminalObservation({ eventType: terminal.eventType, payload: event.payload }) && terminal.agentId === run.agentId && + event.schema === "paperclip.prp.event.v1" && [1, 2].includes(Number(event.schemaVersion)) && + terminal.protocolSchemaVersion === event.schemaVersion && event.eventType === terminal.eventType && + event.sourceKind === "runner" && event.runId === run.id && event.normalizedSessionId === run.nativeSessionId && + event.sourceInstanceId === run.runnerInstanceId && terminal.sourceInstanceId === event.sourceInstanceId && + terminal.sourceEventId === event.sourceEventId && terminal.sourceSeq === event.sourceSeq && + Number.isSafeInteger(event.sourceSeq) && Number(event.sourceSeq) > 0 && + typeof event.turnId === "string" && Boolean(event.turnId); +} + /** A Stop acknowledgement alone does not prove provider cleanup completed. */ export async function acknowledgedNativeStopExecutionHasStopped(db: Db, run: Pick) { diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts index 5a34a5d430a..b3184891459 100644 --- a/server/src/services/activity.ts +++ b/server/src/services/activity.ts @@ -91,11 +91,26 @@ export function activityService(db: Db) { `.as("usageJson"); // A denied user needs the selected connection's name to repair their task. // Other readers need the connection's existing human sharing permission. + // A malformed present Stop receipt must stay unconfirmed, not disappear. const summarizedResultJson = (viewerUserId: string | null) => sql | null>` case when ${heartbeatRuns.resultJson} is null then null else jsonb_strip_nulls(jsonb_build_object( 'conversationReset', ${heartbeatRuns.resultJson} -> 'conversationReset', + 'executionCancellation', case when ${heartbeatRuns.resultJson} ? 'executionCancellation' + then jsonb_build_object('state', case + when jsonb_typeof(${heartbeatRuns.resultJson} -> 'executionCancellation') = 'object' + and ${heartbeatRuns.resultJson} #>> '{executionCancellation,state}' in ('requested', 'acknowledged') + then ${heartbeatRuns.resultJson} #>> '{executionCancellation,state}' else 'unknown' end) end, + 'nativeCancellation', case + when jsonb_typeof(${heartbeatRuns.resultJson} -> 'nativeCancellation') = 'object' + and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,schema}' = 'paperclip.native-cancellation.v1' + and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,runId}' = ${heartbeatRuns.id}::text + and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,companyId}' = ${heartbeatRuns.companyId}::text + and ${heartbeatRuns.resultJson} #>> '{nativeCancellation,dispatchState}' in ('pending', 'acknowledged') + then jsonb_build_object('schema', 'paperclip.native-cancellation.v1', + 'runId', ${heartbeatRuns.id}, 'companyId', ${heartbeatRuns.companyId}, + 'dispatchState', ${heartbeatRuns.resultJson} #>> '{nativeCancellation,dispatchState}') end, 'configurationIncomplete', case when ${heartbeatRuns.resultJson} -> 'configurationIncomplete' ->> 'reason' = 'workspace_base_ref_unresolved' then jsonb_build_object('reason', 'workspace_base_ref_unresolved') diff --git a/server/src/services/adapter-settings-postgres.test.ts b/server/src/services/adapter-settings-postgres.test.ts index 299c4b14f35..6198c163e05 100644 --- a/server/src/services/adapter-settings-postgres.test.ts +++ b/server/src/services/adapter-settings-postgres.test.ts @@ -8,7 +8,7 @@ import os from "node:os"; import path from "node:path"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; -const legacy = vi.hoisted(() => ({ get: vi.fn(() => [] as string[]), set: vi.fn() })); +const legacy = vi.hoisted(() => ({ extraTypes: [] as string[], get: vi.fn(() => [] as string[]), set: vi.fn() })); vi.mock("./adapter-plugin-store.js", () => ({ getDisabledAdapterTypes: legacy.get, setAdapterDisabled: legacy.set, @@ -16,7 +16,7 @@ vi.mock("./adapter-plugin-store.js", () => ({ getAdapterPluginByType: () => undefined, })); vi.mock("../adapters/registry.js", () => ({ - listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }], + listServerAdapters: () => [{ type: "codex_local" }, { type: "claude_local" }, { type: "paperclip_runner" }, ...legacy.extraTypes.map(type => ({ type }))], findServerAdapter: (type: string) => ({ type }), findActiveServerAdapter: (type: string) => ({ type }), isOverridePaused: () => false, @@ -56,6 +56,7 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos beforeEach(async () => { directory = await fs.mkdtemp(path.join(os.tmpdir(), "adapter-settings-import-")); await db.delete(instanceSettings).where(eq(instanceSettings.singletonKey, "adapter-settings:v1")); + legacy.extraTypes = []; legacy.get.mockClear(); legacy.set.mockClear(); vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-test-token"); @@ -188,6 +189,58 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos expect((await resolveNewAgentRunnerForCompany(db, "company", input)).adapterType).toBe("codex_local"); }); + it("refreshes API-only Cursor policy from PostgreSQL and authoritative operator declarations", async () => { + legacy.extraTypes = ["cursor_cloud"]; + await service().seedIfAbsent({ disabledTypes: [] }); + await db.insert(instanceSettings).values({ singletonKey: "default", general: {}, + experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }) + .onConflictDoUpdate({ target: instanceSettings.singletonKey, set: { general: {}, + experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } } }); + const draft = { adapterType: "cursor_cloud", runner: "auto" as const, setupOnly: true, adapterConfig: {}, disabledTypes: [] }; + const saved = { adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/acme/repo", lifecycleMode: "per_turn" } }; + const admitted = async (input = draft as typeof draft | typeof saved) => { + expect(await resolveNewAgentRunnerForCompany(db, "company", input)) + .toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } }); + }; + const denied = async (input = draft as typeof draft | typeof saved) => { + await expect(resolveNewAgentRunnerForCompany(db, "company", input)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } }); + }; + await admitted(); // The default managed tenant has no Computer or CLI probe. + for (const type of ["cursor_cloud", "paperclip_runner"]) { + await adapterSettingsService(peer, managed).setDisabled(type, true); + await denied(); + await denied(saved); + await adapterSettingsService(peer, managed).setDisabled(type, false); + await admitted(); + } + await adapterSettingsService(peer, managed).setDisabled("cursor_cloud", true); + const stored = await service().inspectStored(); + vi.stubEnv("PAPERCLIP_ADAPTERS", JSON.stringify([ + { adapterType: "cursor_cloud", enabled: true }, { adapterType: "paperclip_runner", enabled: true }, + ])); + await admitted(); // Operator allowlist supersedes, rather than unions, saved disables. + await admitted(saved); + expect(await service().inspectStored()).toEqual(stored); + for (const declaration of [ + [{ adapterType: "cursor_cloud", enabled: true }], + [{ adapterType: "paperclip_runner", enabled: true }], + [{ adapterType: "cursor_cloud", enabled: false }, { adapterType: "paperclip_runner", enabled: true }], + [{ adapterType: "cursor_cloud", enabled: true }, { adapterType: "paperclip_runner", enabled: false }], + ]) { + vi.stubEnv("PAPERCLIP_ADAPTERS", JSON.stringify(declaration)); + await denied(); + await denied(saved); + } + vi.stubEnv("PAPERCLIP_ADAPTERS", "invalid operator declaration"); + await expect(resolveNewAgentRunnerForCompany(db, "company", draft)).rejects.toThrow(); + vi.stubEnv("PAPERCLIP_ADAPTERS", undefined); + await denied(); // Removing the overlay restores the saved choice. + expect(await service().inspectStored()).toEqual(stored); + expect(legacy.get).not.toHaveBeenCalled(); + expect(legacy.set).not.toHaveBeenCalled(); + }); + it("rolls back the setting when its audit write fails", async () => { await expect(service().setDisabled("codex_local", true, async () => { throw new Error("audit failed"); })).rejects.toThrow("audit failed"); expect(await service().inspectStored()).toBeNull(); diff --git a/server/src/services/agent-deletion.ts b/server/src/services/agent-deletion.ts index ca18e8f02ec..245919fff3a 100644 --- a/server/src/services/agent-deletion.ts +++ b/server/src/services/agent-deletion.ts @@ -14,6 +14,7 @@ import { agents, agentApiKeys, agentRuntimeState, agentTaskSessions, agentWakeup caseHandoffs, principalPermissionGrants, type Db } from "@paperclipai/db"; import { conflict } from "../errors.js"; import { issueThreadInteractionService } from "./issue-thread-interactions.js"; +import { deleteAgentAdmissionHistory } from "../modules/execution-admission/index.js"; export async function deleteAgentDependencies(tx: Db, companyId: string, id: string) { const [retainedHandoff] = await tx.select({ id: caseHandoffs.id }).from(caseHandoffs).where(and( @@ -63,6 +64,11 @@ export async function deleteAgentDependencies(tx: Db, companyId: string, id: str await notifyInstructionCollection(tx, { companyId: companyId }); await notifyCancelledWorkspacePreservation(tx, { companyId: companyId }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId }); + if ((await deleteAgentAdmissionHistory(tx, { companyId, agentId: id })).kind === "unsettled") { + throw conflict("Wait for this agent's admitted task executions to settle before deleting it", { + code: "agent_execution_admission_pending", + }); + } await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.agentId, id)); await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, id)); await tx.delete(agentApiKeys).where(eq(agentApiKeys.agentId, id)); diff --git a/server/src/services/agent-harness-verification.ts b/server/src/services/agent-harness-verification.ts index 9a203f04e50..c8f7e21561a 100644 --- a/server/src/services/agent-harness-verification.ts +++ b/server/src/services/agent-harness-verification.ts @@ -10,6 +10,8 @@ import { aiConnectionService } from "./ai-connections.js"; import { withManagedAiProbe, stripAiAuthBindings } from "./ai-connection-runtime.js"; import { environmentService } from "./environments.js"; import { secretService } from "./secrets.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { assertProviderApiPlacementPolicy, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { museRunnerBroker } from "./muse-runner-broker.js"; import { dotRunnerBroker } from "./dot-runner-broker.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; @@ -51,6 +53,8 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan const binding = await museRunnerBroker(db).bindingForAgent(agent.companyId,agent.id); return binding?.status === "ready" && binding.paired && binding.receiverDetected && binding.backgroundReplyVerified && binding.id === agent.adapterConfig.museBindingId ? "complete" : "pending"; } + const providerApi = resolveProviderApiPlacement({ agent }); + if (providerApi) assertProviderApiPlacementPolicy(await instanceSettingsService(db).getGeneral()); const [company] = await db.select().from(companies).where(eq(companies.id, agent.companyId)); const responsibleUserId = agent.lifecycleOperation?.responsibleUserId ?? company.defaultResponsibleUserId; const context = { consumerType: "agent" as const, consumerId: agent.id, actorType: "system" as const, @@ -66,7 +70,7 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan } if (binding) config = { ...config, env: stripAiAuthBindings(config.env) }; config = (await secrets.resolveAdapterConfigForRuntime(agent.companyId, config, context, { adapterType: agent.adapterType })).config; - const environmentId = await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId, agent.adapterType === "paperclip_runner" && config.provider === "acpx" && config.acpxAgent === "copilot"); + const environmentId = providerApi ? null : await tests.resolveAdapterTestEnvironmentId(agent.companyId, agent.defaultEnvironmentId, agent.adapterType === "paperclip_runner" && config.provider === "acpx" && config.acpxAgent === "copilot"); if (environmentId) { await tests.assertAdapterTestEnvironmentForCompany(agent.companyId, environmentId, agent.adapterType); const environment = await environments.getById(environmentId); @@ -81,7 +85,9 @@ export function agentHarnessVerificationService(db: Db, manager: PluginWorkerMan adapterType: agent.adapterType, model: config.model, runnerProvider: config.provider, acpxAgent: config.acpxAgent, binding, }); - const target = await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId }); + const target = providerApi + ? { executionTarget: null, environmentName: null, fallbackChecks: [], release: async (_status?: "released" | "failed") => {} } + : await tests.resolveAdapterTestExecutionContext({ agentId: agent.id, companyId: agent.companyId, adapterType: agent.adapterType, environmentId }); let status: "released" | "failed" = "failed"; try { if (target.fallbackChecks.length) throw Object.assign(new Error("The test environment is unavailable"), { code: "test_environment_unavailable" }); diff --git a/server/src/services/agent-hiring.ts b/server/src/services/agent-hiring.ts index 16253ac025a..48e7ec83489 100644 --- a/server/src/services/agent-hiring.ts +++ b/server/src/services/agent-hiring.ts @@ -17,6 +17,8 @@ import { agentCredentialService } from "./agent-credentials.js"; import { agentIdentityService } from "./agent-identity.js"; import { initializePrimaryAgent } from "./primary-agent.js"; import { resolveAgentRunnerTaskDriverForCompany, resolveNewAgentRunnerForCompany } from "./agent-runner-selection.js"; +import { instanceSettingsService } from "./instance-settings.js"; +import { assertProviderApiPlacementPolicy, isProviderApiSetupDraft, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; export async function prepareAgentHire(db: Db, companyId: string, input: CreateAgentData, options?: CreateAgentOptions): Promise { await notifyAttentionRetentionWork(db, { companyId }); @@ -49,7 +51,11 @@ export async function prepareAgentHire(db: Db, companyId: string, input: CreateA ? await secretService(db).normalizeAdapterConfigForPersistence(companyId, data.adapterConfig, { adapterType }) : {}; const adapterConfig = normalizePaperclipRunnerAdapterConfig(adapterType, rawAdapterConfig); - if (adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) && + const profile = { adapterType, adapterConfig }; + const providerApiDraft = options?.setupOnly === true && isProviderApiSetupDraft(profile); + const providerApi = providerApiDraft ? null : resolveProviderApiPlacement({ agent: profile }); + if (providerApi) assertProviderApiPlacementPolicy(await instanceSettingsService(db).getGeneral()); + if (!providerApiDraft && !providerApi && adapterType === "paperclip_runner" && !["openai_dot", "grok_bot", "muse"].includes(String(adapterConfig.provider)) && (await resolveAgentRunnerTaskDriverForCompany(db, companyId, data.defaultEnvironmentId))?.warmTaskVersion === 1) { adapterConfig.lifecycleMode = "warm"; } diff --git a/server/src/services/agent-lifecycle.ts b/server/src/services/agent-lifecycle.ts index ea8fd4c4ad2..ef42fb865c5 100644 --- a/server/src/services/agent-lifecycle.ts +++ b/server/src/services/agent-lifecycle.ts @@ -24,13 +24,15 @@ import { trackIdleWork } from "./task-admission.js"; import { environmentService } from "./environments.js"; import { instanceSettingsService } from "./instance-settings.js"; import { pluginEnvironmentRequiresAgent } from "./plugin-environment-driver.js"; +import { isProviderApiSetupDraft } from "./native-runtime/provider-api-placement.js"; export { scheduleAgentLifecycle, processAgentLifecycle } from "../modules/agent-lifecycle/index.js"; export type { LifecycleDriver, LifecycleAgent } from "../modules/agent-lifecycle/index.js"; export function createAgentLifecycleEffects(_hooks: BudgetServiceHooks = {}): lifecycle.LifecycleEffects { return { - async setupRequiresPreparation(db, companyId, environmentId) { + async setupRequiresPreparation(db, companyId, environmentId, profile) { + if (isProviderApiSetupDraft(profile)) return false; const id = environmentId ?? (await instanceSettingsService(db).get()).defaultEnvironmentId; const environment = id ? await environmentService(db).getById(id) : null; if (!environment || !await pluginEnvironmentRequiresAgent(db, environment)) return false; diff --git a/server/src/services/agent-runner-selection.test.ts b/server/src/services/agent-runner-selection.test.ts index 41b722e36e6..831cb255783 100644 --- a/server/src/services/agent-runner-selection.test.ts +++ b/server/src/services/agent-runner-selection.test.ts @@ -46,18 +46,96 @@ describe("server-owned Codex runner selection", () => { const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo", repoStartingRef: "feature", runtimeEnvType: "cloud", env: { CURSOR_API_KEY: { type: "secret_ref", secretId: "saved-key", version: "latest" } }, instructionsFilePath: "/assigned/AGENTS.md", autoCreatePR: false }; expect(agentRunnerAvailability("cursor_cloud", target)).toEqual({ supportedRunners: ["paperclip", "legacy"], defaultRunner: "legacy" }); expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, target })).toEqual({ adapterType: "cursor_cloud", adapterConfig: config }); + for (const runner of [undefined, "auto"] as const) { + for (const driver of ["unavailable", "sandbox", "ssh"]) { + expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner, target: { driver } })) + .toThrow(expect.objectContaining({ status: 422, details: { code: "agent_runner_unavailable" } })); + } + } + expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "legacy", target: { driver: "sandbox" } })) + .toEqual({ adapterType: "cursor_cloud", adapterConfig: config }); const converted = resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target }); expect(converted).toEqual({ adapterType: "paperclip_runner", adapterConfig: { model: config.model, repoUrl: config.repoUrl, repoRef: "feature", cloudEnvType: "cloud", env: config.env, instructionsFilePath: config.instructionsFilePath, autoCreatePR: false, provider: "cursor_cloud", lifecycleMode: "per_turn" } }); expect(resolveNewAgentRunner({ ...converted, runner: "legacy", target })).toEqual({ adapterType: "cursor_cloud", adapterConfig: config }); }); - it("rejects unsupported Cursor Cloud conversions and remote controllers without changing the original", () => { + it("rejects unsupported Cursor Cloud settings and keeps API placement independent of the selected Computer", () => { const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo" }; for (const patch of [{ promptTemplate: "custom instructions" }, { runtimeEnvType: "pool" }, { model: "" }]) { expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: { ...config, ...patch }, runner: "paperclip", target })).toThrow(); } expect(config).toEqual({ model: "caller-model", repoUrl: "https://github.com/acme/repo" }); - expect(agentRunnerAvailability("cursor_cloud", { ...target, driver: "ssh" })).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" }); - expect(() => resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target: { ...target, driver: "ssh" } })).toThrow("unavailable"); + expect(agentRunnerAvailability("cursor_cloud", { ...target, driver: "ssh" })).toEqual({ supportedRunners: ["paperclip"], defaultRunner: "paperclip" }); + expect(resolveNewAgentRunner({ adapterType: "cursor_cloud", adapterConfig: config, runner: "paperclip", target: { ...target, driver: "ssh" } })).toMatchObject({ adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud" } }); + }); + it("saves native Cursor Cloud without probing or binding a managed Computer", async () => { + state.settings = { defaultEnvironmentId: "managed", experimental: { enableManagedSandboxOnly: true } }; + const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { + adapterType: "cursor_cloud", runner: "paperclip", defaultEnvironmentId: "managed", + adapterConfig: { model: "caller-model", repoUrl: "https://github.com/acme/repo" }, + }); + expect(saved).toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } }); + expect(state.getEnvironment).not.toHaveBeenCalled(); + expect(state.ssh).not.toHaveBeenCalled(); + expect(state.sandboxQualification).not.toHaveBeenCalled(); + }); + it("refreshes managed adapter policy before API-only Cursor admission without a Computer", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN", "synthetic-cloud-test-token"); + state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + state.managedEnvironment.mockResolvedValue(null); + const input = { adapterType: "cursor_cloud", runner: "auto" as const, setupOnly: true, adapterConfig: {}, disabledTypes: [] }; + expect(await resolveNewAgentRunnerForCompany({} as never, "company", input)) + .toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, adapterConfig: { provider: "cursor_cloud" } }); + for (const disabled of ["cursor_cloud", "paperclip_runner"]) { + state.disabled = [disabled]; + await expect(resolveNewAgentRunnerForCompany({} as never, "company", input)) + .rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } }); + } + await expect(resolveNewAgentRunnerForCompany({} as never, "company", { + adapterType: "paperclip_runner", adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/acme/repo", lifecycleMode: "per_turn" }, + })).rejects.toMatchObject({ status: 422, details: { code: "agent_runner_unavailable" } }); + expect(state.managedEnvironment).not.toHaveBeenCalled(); + expect(state.getEnvironment).not.toHaveBeenCalled(); + expect(state.resolveEnvironment).not.toHaveBeenCalled(); + expect(state.ssh).not.toHaveBeenCalled(); + expect(state.sandboxQualification).not.toHaveBeenCalled(); + }); + it.each([undefined, "auto"] as const)("defaults a new managed Cursor setup draft to the native API controller without a Computer (%s)", async runner => { + state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + state.managedEnvironment.mockResolvedValue(null); + const saved = await resolveNewAgentRunnerForCompany({} as never, "company", { + adapterType: "cursor_cloud", runner, setupOnly: true, adapterConfig: {}, + }); + expect(saved).toMatchObject({ adapterType: "paperclip_runner", defaultEnvironmentId: null, + adapterConfig: { provider: "cursor_cloud", lifecycleMode: "per_turn" } }); + expect(state.managedEnvironment).not.toHaveBeenCalled(); + expect(state.getEnvironment).not.toHaveBeenCalled(); + }); + it.each([ + { setupOnly: false, enabled: true }, + { setupOnly: true, enabled: false }, + ])("does not use the new-draft Cursor API default outside its admission scope (%j)", async ({ setupOnly, enabled }) => { + state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: enabled } }; + state.managedEnvironment.mockResolvedValue(null); + await expect(resolveNewAgentRunnerForCompany({} as never, "company", { + adapterType: "cursor_cloud", runner: "auto", setupOnly, adapterConfig: {}, + })).rejects.toMatchObject({ status: 422, details: { code: "managed_sandbox_unavailable" } }); + }); + it("preserves an explicit Legacy choice on a held Cursor setup draft", async () => { + state.settings = { experimental: { enableManagedSandboxOnly: true, enableNativeRunner: true } }; + expect(await resolveNewAgentRunnerForCompany({} as never, "company", { + adapterType: "cursor_cloud", runner: "legacy", setupOnly: true, adapterConfig: {}, + })).toEqual({ adapterType: "cursor_cloud", adapterConfig: {} }); + expect(state.managedEnvironment).not.toHaveBeenCalled(); + }); + it("refuses native Cursor save and availability under Kubernetes-only policy without provisioning", async () => { + state.settings = { general: { executionMode: "kubernetes" } }; + await expect(resolveNewAgentRunnerForCompany({} as never, "company", { adapterType: "cursor_cloud", runner: "paperclip", + adapterConfig: { model: "caller-model", repoUrl: "https://github.com/acme/repo" } })) + .rejects.toMatchObject({ status: 422, details: { code: "provider_api_kubernetes_execution_unsupported" } }); + const selected = await resolveAgentRunnerTargetForCompany({} as never, "company"); + expect(agentRunnerAvailability("cursor_cloud", selected).supportedRunners).toEqual([]); + expect(state.getEnvironment).not.toHaveBeenCalled(); + expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled(); }); it("keeps legacy Cursor Cloud environment bindings and rejects their unsupported native conversion", () => { const config = { model: "caller-model", repoUrl: "https://github.com/acme/repo", @@ -389,7 +467,7 @@ describe("server-owned Codex runner selection", () => { it("advertises Legacy for a forced Kubernetes target before lazy environment creation", async () => { state.settings = { general: { executionMode: "kubernetes" } }; const selected = await resolveAgentRunnerTargetForCompany({} as never, "company"); - expect(selected).toEqual({ driver: "sandbox" }); + expect(selected).toEqual({ driver: "sandbox", executionMode: "kubernetes" }); expect(agentRunnerAvailability("codex_local", selected, state.disabled)).toEqual({ supportedRunners: ["legacy"], defaultRunner: "legacy" }); expect(state.kubernetesEnvironment).toHaveBeenCalledWith("company"); expect(state.ensureKubernetesEnvironment).not.toHaveBeenCalled(); diff --git a/server/src/services/agent-runner-selection.ts b/server/src/services/agent-runner-selection.ts index 68802917de8..a836da2e190 100644 --- a/server/src/services/agent-runner-selection.ts +++ b/server/src/services/agent-runner-selection.ts @@ -1,3 +1,4 @@ +import { assertProviderApiPlacementPolicy, managedCursorApiDefaultEnabled, resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import type { Db } from "@paperclipai/db"; import { agentHarnessType, paperclipRunnerProfileForHarness, isEnvironmentDriverSupportedForAdapter, paperclipRunnerSupportsPlatform, type AgentRunnerChoice, type AgentRunnerAvailability } from "@paperclipai/shared"; import { normalizeLegacyRunnerProvider, PAPERCLIP_RUNNER_DEFAULT_MODELS } from "@paperclipai/adapter-utils"; @@ -16,6 +17,7 @@ import { localAgentRunnerTarget } from "../lib/agent-runner-platform.js"; import { normalizeClaudeRunnerOptions, type AgentConfigurationWarning } from "@paperclipai/shared"; import { usesManagedCloudSandboxImage } from "./sandbox-snapshot-qualification.js"; import { environmentCustomImageTemplateMatchesBaseConfig } from "./environment-custom-image-runtime.js"; +import type { ExecutionPolicy } from "./execution-allowlist.js"; export type RunnerTarget = { driver: string; platform?: string; architecture?: string; libc?: "glibc" | "musl" | "unknown"; glibcVersion?: string; @@ -23,6 +25,7 @@ export type RunnerTarget = { runnerTasks?: boolean; managedCloudImage?: boolean; warmTaskVersion?: 1; + executionMode?: ExecutionPolicy["executionMode"]; }; export const AGENT_RUNNER_SSH_PLATFORM_COMMAND = "uname -s; uname -m; if paperclip_glibc_version=$(getconf GNU_LIBC_VERSION 2>/dev/null) && [ -n \"$paperclip_glibc_version\" ]; then printf 'glibc\\n%s\\n' \"$paperclip_glibc_version\"; else case \"$(ldd --version 2>&1)\" in *musl*) printf 'musl\\n' ;; *) printf 'unknown\\n' ;; esac; fi"; export interface AgentRunnerSelection { @@ -52,10 +55,15 @@ function runnerAdaptersAvailable(harness: string, driver: string, disabledTypes: export function agentRunnerAvailability(harness: string, target: RunnerTarget = localAgentRunnerTarget(), disabledTypes: string[] = localDisabledTypes()): AgentRunnerAvailability { if (harness === "cursor_cloud") { - const supported = target.driver === "local" && runnerAdaptersAvailable(harness, target.driver, disabledTypes) - && (!target.platform || paperclipRunnerSupportsPlatform(harness, target.platform, target.architecture ?? "", target.libc, target.glibcVersion)); - return { supportedRunners: supported ? ["paperclip", "legacy"] : ["legacy"], defaultRunner: "legacy" }; + if (target.executionMode === "kubernetes") return { supportedRunners: [], defaultRunner: "legacy" }; + const controller = localAgentRunnerTarget(); + const supported = runnerAdaptersAvailable(harness, "local", disabledTypes) + && paperclipRunnerSupportsPlatform(harness, controller.platform, controller.architecture, controller.libc, controller.glibcVersion); + const legacy = isEnvironmentDriverSupportedForAdapter(harness, target.driver); + return { supportedRunners: [...(supported ? ["paperclip" as const] : []), ...(legacy ? ["legacy" as const] : [])], + defaultRunner: legacy || !supported ? "legacy" : "paperclip" }; } + if (target.driver === "unavailable") return { supportedRunners: [], defaultRunner: "legacy" }; if (target.runnerTasks || target.managedCloudImage) { return runnerAdaptersAvailable(harness, target.driver, disabledTypes) ? { supportedRunners: target.runnerTasks ? ["paperclip"] : ["paperclip", "legacy"], defaultRunner: "paperclip" } @@ -187,10 +195,16 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; if (native && (input.runner === undefined || input.runner === "paperclip" || (harness !== "codex_local" && input.runner !== "legacy"))) return { adapterType, adapterConfig: normalizeLegacyRunnerProvider(config) }; const availability = agentRunnerAvailability(harness, input.target, disabledTypes); const choice = input.runner ?? "auto"; - const useNative = choice === "paperclip" || (choice === "auto" && availability.defaultRunner === "paperclip"); + const useNative = choice === "paperclip" || (choice === "auto" && harness !== "cursor_cloud" && availability.defaultRunner === "paperclip"); if (!useNative) { if (harness === "cursor_cloud") { - if (!native) return { adapterType, adapterConfig: config }; + if (!native) { + if (choice !== "legacy" && !availability.supportedRunners.includes("legacy")) throw unprocessable( + "Automatic Cursor selection is unavailable for this environment. Choose a supported runner in Advanced.", + { code: "agent_runner_unavailable" }, + ); + return { adapterType, adapterConfig: config }; + } if (config.lifecycleMode !== undefined && config.lifecycleMode !== "per_turn") incompatible(["lifecycleMode"]); for (const [current, legacy] of [["cloudEnvType", "runtimeEnvType"], ["cloudEnvName", "runtimeEnvName"], ["repoRef", "repoStartingRef"], ["prUrl", "repoPullRequestUrl"]]) { if (config[current] !== undefined) config[legacy] = config[current]; @@ -363,7 +377,33 @@ function resolveAgentRunner(input: AgentRunnerSelection): { adapterType: string; } export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, input: AgentRunnerSelection & { defaultEnvironmentId?: string | null }) { - input = { ...input, disabledTypes: await adapterSettingsService(db).getDisabledTypes() }; + const disabledTypes = await adapterSettingsService(db).getDisabledTypes(); + input = { ...input, disabledTypes }; + const cursor = agentHarnessType(input.adapterType ?? "process", input.adapterConfig) === "cursor_cloud"; + const nativeCursor = cursor && input.runner !== "legacy" + && (input.adapterType === "paperclip_runner" || input.runner === "paperclip"); + const cursorSetupDefault = cursor && input.setupOnly === true + && (input.runner === undefined || input.runner === "auto"); + if (nativeCursor || cursorSetupDefault) { + const { instanceSettingsService } = await import("./instance-settings.js"); + const settings = await instanceSettingsService(db).get(); + // Only a held new setup may adopt the API-only managed default. Existing + // legacy agents still require an explicit conversion decision. + const useSetupDefault = cursorSetupDefault && managedCursorApiDefaultEnabled(settings.experimental); + if (nativeCursor || useSetupDefault) { + if (disabledTypes.includes("cursor_cloud") || disabledTypes.includes("paperclip_runner")) { + throw unprocessable("Cursor Cloud is disabled on this instance. Enable its adapter or choose another harness.", { code: "agent_runner_unavailable" }); + } + assertProviderApiPlacementPolicy(settings.general ?? {}); + const selected = resolveNewAgentRunner({ ...input, ...(useSetupDefault ? { runner: "paperclip" as const } : {}), target: localAgentRunnerTarget() }); + try { if (!input.setupOnly) resolveProviderApiPlacement({ agent: selected }); } + catch (error) { + if (error instanceof PaperclipRunnerProviderProfileError) throw unprocessable(error.message, { code: error.code }); + throw error; + } + return { ...selected, defaultEnvironmentId: null }; + } + } if (!paperclipRunnerProfileForHarness(agentHarnessType(input.adapterType ?? "process", input.adapterConfig)) || input.runner === "legacy" || (input.adapterType === "paperclip_runner" && (input.runner === undefined || input.runner === "paperclip"))) return resolveNewAgentRunner(input); const target = await resolveAgentRunnerTargetForCompany(db, companyId, input.defaultEnvironmentId, input.disabledTypes); @@ -372,7 +412,8 @@ export async function resolveNewAgentRunnerForCompany(db: Db, companyId: string, export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null, disabledTypes?: string[]): Promise { await waitForExternalAdapters(); - const { environment, driver } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId); + const { environment, driver, executionMode } = await resolveAgentRunnerEnvironmentForCompany(db, companyId, defaultEnvironmentId); + const withPolicy = (value: RunnerTarget): RunnerTarget => ({ ...value, ...(executionMode ? { executionMode } : {}) }); let target: RunnerTarget = driver === "local" ? localAgentRunnerTarget() : { driver }; if (driver === "plugin" && environment) { @@ -386,16 +427,16 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri environmentId: environment.id, provider: typeof environment.config.provider === "string" ? environment.config.provider : null, }); - if (!template?.templateRef) return { driver, managedCloudImage: true }; + if (!template?.templateRef) return withPolicy({ driver, managedCloudImage: true }); try { const parsed = parseEnvironmentDriverConfig(environment); if (parsed.driver === "sandbox" && !environmentCustomImageTemplateMatchesBaseConfig({ template, baseConfig: parsed.config, secretRefExcludePaths: await resolveSandboxProviderSecretRefPaths(db, template.provider), - })) return { driver, managedCloudImage: true }; + })) return withPolicy({ driver, managedCloudImage: true }); } catch { - return { driver }; + return withPolicy({ driver }); } } const { readSandboxPlatformQualification } = await import("./environment-probe.js"); @@ -413,7 +454,7 @@ export async function resolveAgentRunnerTargetForCompany(db: Db, companyId: stri const [os = "", arch = "", libc, glibcVersion] = output.trim().split(/\r?\n/); target = { driver, platform: os === "Linux" ? "linux" : os === "Darwin" ? "darwin" : "unknown", architecture: arch === "x86_64" ? "x64" : ["aarch64", "arm64"].includes(arch) ? "arm64" : arch, ...(os === "Linux" && libc !== undefined ? { libc: libc === "glibc" || libc === "musl" ? libc : "unknown", ...(libc === "glibc" && glibcVersion !== undefined ? { glibcVersion } : {}) } : {}) }; } - return target; + return withPolicy(target); } export async function resolveAgentRunnerEnvironmentForCompany(db: Db, companyId: string, defaultEnvironmentId?: string | null) { @@ -432,16 +473,19 @@ async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defa const { instanceSettingsService } = await import("./instance-settings.js"); const { environmentService } = await import("./environments.js"); const settings = await instanceSettingsService(db).get(); + const executionMode = settings.general?.executionMode; const envs = environmentService(db); const environmentId = defaultEnvironmentId ?? settings.defaultEnvironmentId; let environment = environmentId ? await envs.getById(environmentId) : null; if (settings.experimental?.enableManagedSandboxOnly && (!environment || environment.driver === "local")) { environment = await envs.findManagedSandboxEnvironment(companyId); - if (!environment) throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", { code: "managed_sandbox_unavailable" }); + if (!environment) throw unprocessable("The managed sandbox is unavailable. Restore Paperclip Computer and retry.", { + code: "managed_sandbox_unavailable", ...(executionMode ? { executionMode } : {}), + }); } if (settings.general?.executionMode === "kubernetes") { environment = await envs.findKubernetesEnvironment(companyId); - if (!environment) return { environment: null, driver: "sandbox" }; + if (!environment) return { environment: null, driver: "sandbox", executionMode }; } if (environmentId && !environment) throw unprocessable("Environment not found."); if (environment) { @@ -449,7 +493,7 @@ async function resolveAgentEnvironmentForCompany(db: Db, companyId: string, defa if (boundCompanyIds.length > 0 && !boundCompanyIds.includes(companyId)) throw forbidden("The selected environment belongs to another company.", { code: "environment_company_mismatch" }); if (environment.status === "archived") throw unprocessable("Environment is archived."); } - return { environment, driver: environment?.driver ?? "local" }; + return { environment, driver: environment?.driver ?? "local", executionMode }; } // The plugin's manifest declaration decides whether its environment runs diff --git a/server/src/services/budget-tracing.ts b/server/src/services/budget-tracing.ts index f4b9212ca8f..535ef5617e1 100644 --- a/server/src/services/budget-tracing.ts +++ b/server/src/services/budget-tracing.ts @@ -3,7 +3,7 @@ import { withDatabaseOperation } from "@paperclipai/db"; type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover" | "budget.accounting.settlement" | "budget.accounting.transaction" - | "budget.admission.evaluate" | "budget.admission.reserve" + | "budget.admission.evaluate" | "budget.admission.inspect" | "budget.admission.reconcile" | "budget.admission.reserve" | "budget.cancellation.deliver" | "budget.enforcement.deliver" | "budget.enforcement.scope" | "budget.exposure.evaluate"; diff --git a/server/src/services/budgets.ts b/server/src/services/budgets.ts index 3d5c2c00352..da4b8da1d2b 100644 --- a/server/src/services/budgets.ts +++ b/server/src/services/budgets.ts @@ -34,7 +34,7 @@ import type { } from "@paperclipai/shared"; import { HttpError, notFound, unprocessable } from "../errors.js"; import { logger } from "../middleware/logger.js"; -import { withAccountingTransaction } from "./accounting-transaction.js"; +import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js"; import { logActivity, type LogActivityInput, type ActivityPublication } from "./activity-log.js"; import { computeObservedSpend, computeObservedSpends, type ObservedSpend } from "./budget-exposure.js"; import { notifyDeliveryWork, DELIVERY_QUEUES } from "./delivery-work-notifications.js"; @@ -198,11 +198,66 @@ export function budgetPoliciesForRun(companyId: string, agentId: string | null, ); } +type AdmissionContext = { issueId?: string | null; projectId?: string | null }; + +async function readAdmissionScopes(db: Db, companyId: string, agentId: string | null, context?: AdmissionContext) { + const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [ + { scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []), + ...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []), + ]; + const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId))); + if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found"); + const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId), + budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null))); + return { scopes, records, policies }; +} + +function scopePauseBlock(scopeType: BudgetScopeType, scopeId: string, scope: ScopeRecord, hasPolicies: boolean) { + if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") { + return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` }; + } + if (scope.paused && scope.pauseReason === "budget" && !hasPolicies) { + return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." }; + } + return null; +} + +async function inspectInvocationEligibility(db: Db, companyId: string, agentId: string | null, context?: AdmissionContext) { + const { scopes, records, policies } = await readAdmissionScopes(db, companyId, agentId, context); + if (policies.length > 0) { + if (records.some(scope => scope.budgetHeld)) return { needsReconciliation: true } as const; + const observations = await computeObservedSpends(db, policies); + if (policies.some((policy, index) => observedBlocks(policy, observations[index]) + || (policy.isActive && policy.hardStopEnabled && policy.amount > 0 && observations[index].recoveringRunCount > 0))) { + return { needsReconciliation: true } as const; + } + const incidents = await db.select({ policyId: budgetIncidents.policyId, thresholdType: budgetIncidents.thresholdType }) + .from(budgetIncidents).where(and(eq(budgetIncidents.companyId, companyId), + inArray(budgetIncidents.policyId, policies.map(policy => policy.id)), eq(budgetIncidents.status, "open"))); + const byId = new Map(policies.map((policy, index) => [policy.id, { policy, observed: observations[index] }])); + if (incidents.some(incident => { + const { policy, observed } = byId.get(incident.policyId)!; + return incident.thresholdType === "hard" || shouldResolveSoftIncident(policy, observed); + })) return { needsReconciliation: true } as const; + } + for (const [index, { scopeType, scopeId }] of scopes.entries()) { + const block = scopePauseBlock(scopeType, scopeId, records[index], + policies.some(policy => policy.scopeType === scopeType && policy.scopeId === scopeId)); + if (block) return { needsReconciliation: false, block } as const; + } + return { needsReconciliation: false, block: null } as const; +} + function unpricedUsageBlocks(policy: PolicyRow, observed: ObservedSpend) { return policy.unpricedUsagePolicy !== "allow" && (observed.unpricedEventCount > 0 || observed.unknownBrowserRunCount > 0); } +function shouldResolveSoftIncident(policy: PolicyRow, observed: ObservedSpend) { + return !policy.isActive || !policy.notifyEnabled + || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0; +} + function observedBlocks(policy: PolicyRow, observed: Awaited>) { if (!policy.isActive || !policy.hardStopEnabled || policy.amount <= 0) return false; return compareCents(observed.totalExact, policy.amount) >= 0 || (observed.pendingRunCount - observed.recoveringRunCount) > 0 || unpricedUsageBlocks(policy, observed); @@ -654,18 +709,12 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica // A closed UTC window, inactive policy, or budget raise releases only // budget-owned pauses. Manual pauses remain operator decisions. await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "hard"); - if (!policy.isActive || !policy.notifyEnabled || compareCents(observed.totalExact, Math.ceil(policy.amount * policy.warnPercent / 100)) < 0) { + if (shouldResolveSoftIncident(policy, observed)) { await resolveOpenIncidentsForPolicy(policy.id, policy.companyId, "approved", "budget_service", "soft"); } } - // Agent status has its own admission gate and error code. Preserve a - // manual pause without misclassifying it as a budget failure. - if (scopeType !== "agent" && scope.paused && scope.pauseReason !== "budget") { - return { scopeType, scopeId, scopeName: scope.name, reason: `${scopeType === "company" ? "Company" : "Project"} is paused and cannot start new work.` }; - } - if (scope.paused && scope.pauseReason === "budget" && !policies.length) { - return { scopeType, scopeId, scopeName: scope.name, reason: "Budget pause requires a policy or an explicit operator resume." }; - } + const pauseBlock = scopePauseBlock(scopeType, scopeId, scope, policies.length > 0); + if (pauseBlock) return pauseBlock; if (scope.budgetHeld && policies[0]) await resumeScopeFromBudget(policies[0]); if (recoveringPolicy) return { scopeType, scopeId, scopeName: scope.name, reason: "New work must wait while a native run recovers its unfinished accounting." }; @@ -748,14 +797,7 @@ export function budgetServiceInTransaction(db: Db, publications: ActivityPublica async function evaluateAdmissionInTransaction(companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) { - const scopes: Array<{ scopeType: BudgetScopeType; scopeId: string }> = [ - { scopeType: "company", scopeId: companyId }, ...(agentId ? [{ scopeType: "agent" as const, scopeId: agentId }] : []), - ...(context?.projectId ? [{ scopeType: "project" as const, scopeId: context.projectId }] : []), - ]; - const records = await Promise.all(scopes.map(scope => resolveScopeRecord(db, scope.scopeType, scope.scopeId))); - if (records.some(scope => scope.companyId !== companyId)) throw notFound("Budget scope not found"); - const policies = await db.select().from(budgetPolicies).where(and(eq(budgetPolicies.companyId, companyId), - budgetPoliciesForRun(companyId, agentId, context?.projectId ?? null))); + const { scopes, records, policies } = await readAdmissionScopes(db, companyId, agentId, context); const observations = await computeObservedSpends(db, policies); const reservationIndexes = policies.map((policy, i) => policy.isActive && policy.hardStopEnabled && policy.metric === "billed_cents" && policy.amount > 0 ? i : -1).filter(i => i >= 0); const admission = (block: Awaited>) => ({ block, @@ -1188,8 +1230,13 @@ export function budgetService(db: Db, _hooks: BudgetServiceHooks = {}) { upsertPolicy: (companyId: string, input: BudgetPolicyUpsertInput, actorUserId: string | null) => mutate(companyId, service => service.upsertPolicy(companyId, input, actorUserId)), evaluateCostEvent: (event: typeof costEvents.$inferSelect) => mutate(event.companyId, service => service.evaluateCostEvent(event)), - getInvocationBlock: (companyId: string, agentId: string | null, context?: { issueId?: string | null; projectId?: string | null }) => - mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context)), + getInvocationBlock: async (companyId: string, agentId: string | null, context?: AdmissionContext) => { + const eligibility = await withBudgetSpan("budget.admission.inspect", () => + withAccountingReadSnapshot(db, companyId, tx => inspectInvocationEligibility(tx, companyId, agentId, context))); + if (!eligibility.needsReconciliation) return eligibility.block; + return withBudgetSpan("budget.admission.reconcile", () => + mutate(companyId, service => service.getInvocationBlock(companyId, agentId, context))); + }, resolveIncident: (companyId: string, incidentId: string, input: BudgetIncidentResolutionInput, actorUserId: string) => mutate(companyId, service => service.resolveIncident(companyId, incidentId, input, actorUserId)), }; diff --git a/server/src/services/chat-channels.ts b/server/src/services/chat-channels.ts index 804359109e5..010e9670202 100644 --- a/server/src/services/chat-channels.ts +++ b/server/src/services/chat-channels.ts @@ -136,6 +136,7 @@ import { notifyChatDeliveryWork, notifyChatPublicationWork } from "./chat-work-n import { voiceSessionStore } from "./voice/voice-session-store.js"; import { createChatRuntime, type ChatRuntime } from "./chat-runtime.js"; +import { registerIdleLocalWorkProbe } from "./idle-local-work.js"; import type { ChatEndpointRuntime } from "./voice/voice-runtime.js"; import { fastResponseService, fastResponseSourceCurrent } from "./fast-responses.js"; @@ -517,6 +518,11 @@ export function chatChannelService(db: Db, options: ChatChannelServiceOptions) { version: string; } >(); + // SDK initialize() and the final authority read continue after the registry's + // replacement tail settles. Neither that work nor a failed retirement may + // disappear just because the archived endpoint is no longer returned by get(). + const unregisterRuntimeIdleProbe = registerIdleLocalWorkProbe(() => + runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle() ? "present" : "none"); const { recordCurrentMicrosoftTeamsRoute } = createTeamsRoutes(db, { runtime, runtimeCallbackEndpoint }); const persistence = createChatSdkStatePersistence(db); const fetchImpl = options.fetch ?? globalThis.fetch; @@ -3622,12 +3628,17 @@ return { // Stop every provider runtime before releasing leader rows. This gives a // standby an immediate takeover path without overlapping Gateway sockets. await runtime.shutdown(); + await Promise.allSettled([...runtimeInitializations.values()].map(initialization => initialization.promise)); await slackRegistration.close(); await Promise.all( ownedDiscordGateways.map((ownership) => releaseDiscordGatewayLeaseRow(ownership), ), ); + if (runtimeInitializations.size > 0 || runtime.hasUnsettledLifecycle()) { + throw new Error("Chat runtime lifecycle is still unsettled after shutdown"); + } + unregisterRuntimeIdleProbe(); }, }; } diff --git a/server/src/services/chat-runtime.ts b/server/src/services/chat-runtime.ts index b404a229e9b..881810ae54d 100644 --- a/server/src/services/chat-runtime.ts +++ b/server/src/services/chat-runtime.ts @@ -23,6 +23,11 @@ export class ChatRuntime { return [...this.endpoints.values()]; } + /** Retirement failures retain their owner even after get/list hide it. */ + hasUnsettledLifecycle(): boolean { + return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0; + } + private enqueueLifecycle( endpointId: string, operation: () => Promise, diff --git a/server/src/services/chat-sdk-runtime.test.ts b/server/src/services/chat-sdk-runtime.test.ts index 89e9a9c3587..a956946dc6e 100644 --- a/server/src/services/chat-sdk-runtime.test.ts +++ b/server/src/services/chat-sdk-runtime.test.ts @@ -241,6 +241,8 @@ import { type CreateChatSdkEndpointRuntimeOptions, } from "./chat-sdk-runtime.js"; +import { createChatRuntime } from "./chat-runtime.js"; + const persistence: ChatSdkStatePersistence = { async compareAndSet() { return true; @@ -1152,6 +1154,31 @@ describe("Chat SDK endpoint runtime", () => { await registry.shutdown(); }); + it.each([ + { name: "SDK", create: createChatSdkRuntime }, + { name: "shared", create: createChatRuntime }, + ])("keeps $name lifecycle work visible while hidden retirement fails, until its owner retries", async ({ create }) => { + const registry = create(); + const first = await registry.replaceEndpoint(registryOptions()); + expect(registry.hasUnsettledLifecycle()).toBe(false); + const held = deferred(); + const shutdown = vi.spyOn(first, "shutdown").mockReturnValueOnce(held.promise).mockResolvedValue(undefined); + const removal = registry.removeEndpoint("endpoint-1"); + const result = removal.catch((error: Error) => error.message); + expect(registry.hasUnsettledLifecycle()).toBe(true); + await vi.waitFor(() => expect(shutdown).toHaveBeenCalledOnce()); + expect(registry.get("endpoint-1")).toBeNull(); + expect(registry.list()).toEqual([]); + expect(registry.hasUnsettledLifecycle()).toBe(true); + held.reject(new Error("retirement uncertain")); + expect(await result).toBe("retirement uncertain"); + expect(registry.hasUnsettledLifecycle()).toBe(true); + await registry.removeEndpoint("endpoint-1"); + expect(shutdown).toHaveBeenCalledTimes(2); + expect(registry.hasUnsettledLifecycle()).toBe(false); + await registry.shutdown(); + }); + it("retains failed retirement ownership and only constructs a replacement after an explicit successful retry", async () => { const registry = createChatSdkRuntime(); const first = await registry.replaceEndpoint(registryOptions()); diff --git a/server/src/services/chat-sdk-runtime.ts b/server/src/services/chat-sdk-runtime.ts index cffdd09f5e4..6afd8fb844c 100644 --- a/server/src/services/chat-sdk-runtime.ts +++ b/server/src/services/chat-sdk-runtime.ts @@ -3195,6 +3195,11 @@ export class ChatSdkRuntime { return [...this.endpoints.values()]; } + /** Retirement failures retain their owner even after get/list hide it. */ + hasUnsettledLifecycle(): boolean { + return this.retiringEndpoints.size > 0 || this.lifecycleTails.size > 0; + } + private enqueueLifecycle( endpointId: string, operation: () => Promise, diff --git a/server/src/services/cloud-runtime-identity.ts b/server/src/services/cloud-runtime-identity.ts index 80b3333b36f..cffe6a8dbfd 100644 --- a/server/src/services/cloud-runtime-identity.ts +++ b/server/src/services/cloud-runtime-identity.ts @@ -1,3 +1,4 @@ +import { parseIdleCapabilities, type IdleCapabilities } from "./idle-incoming-wake-capability.js"; import { createPublicKey, timingSafeEqual, verify, type JsonWebKey } from "node:crypto"; import { eq } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -491,6 +492,7 @@ export type CloudControlClaims = { sub: string; action: CloudControlAction; requestId: string; + idleCapabilities?: IdleCapabilities; iat: number; exp: number; }; @@ -572,6 +574,10 @@ export function verifyCloudControlAssertion(input: { ) { throw new Error("Cloud control assertion is expired or has an invalid lifetime"); } + if (payload.idleCapabilities !== undefined) { + if (payload.action !== "task-drain:read") throw new Error("Idle capabilities require task-drain:read"); + payload.idleCapabilities = parseIdleCapabilities(payload.idleCapabilities); + } // Consumed LAST, only after every other check passed: a rejected // assertion must not burn its request id, or an attacker could deny a // legitimate call by replaying a mangled copy of it first. diff --git a/server/src/services/company-deletion.ts b/server/src/services/company-deletion.ts index 66e81df7673..2d341072d2f 100644 --- a/server/src/services/company-deletion.ts +++ b/server/src/services/company-deletion.ts @@ -20,6 +20,7 @@ import { eq, inArray } from "drizzle-orm"; import { createAgentLifecycle, processAgentLifecycle } from "./agent-lifecycle.js"; import { requestRestorePointDeletion } from "./git-restore-point-ledger.js"; import { agentLifecycleCompanyDeletion } from "../modules/agent-lifecycle/company-deletion.js"; +import { executionAdmissionCompanyDeletion } from "../modules/execution-admission/company-deletion.js"; import { companies, agents, @@ -117,6 +118,7 @@ export async function deleteCompany(db: Db, id: string) { await notifyInstructionCollection(tx, { companyId: id }); await notifyCancelledWorkspacePreservation(tx, { companyId: id }); await notifyUnsafeWorkspaceExportRecovery(tx, { companyId: id }); + await executionAdmissionCompanyDeletion.deleteCompanyData(tx, id); await tx.delete(heartbeatRuns).where(eq(heartbeatRuns.companyId, id)); await tx.delete(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, id)); await tx.delete(agentApiKeys).where(eq(agentApiKeys.companyId, id)); diff --git a/server/src/services/connection-event-wake.ts b/server/src/services/connection-event-wake.ts index 87840eb0979..cd7bd24da51 100644 --- a/server/src/services/connection-event-wake.ts +++ b/server/src/services/connection-event-wake.ts @@ -10,3 +10,6 @@ export function wakeConnectionEvents(owner: object): boolean { wake(); return true; } + +/** Inspection never emits a wake or polls the broker. */ +export const hasConnectionEventWakeReceiver = (owner: object): boolean => receivers.has(owner); diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index 4e57d9b0c88..146e551f358 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -1,4 +1,5 @@ -import { isPreDispatchReviewWait } from "./pre-dispatch-review-wait.js"; +import { isPreDispatchReviewWait, neverStartedDispatchCancellationPredicate } from "./pre-dispatch-review-wait.js"; +import { canonicalExecutionUuid } from "./execution-identity.js"; import { preDispatchExecutionWaitRecoveryActionPredicate } from "./pre-dispatch-execution-wait.js"; import { and, desc, eq, gt, inArray, not, or, sql } from "drizzle-orm"; import { conversationRecoveryActionPredicate, getConversationOwnershipBlocker } from "./conversation-continuation.js"; @@ -97,6 +98,17 @@ export function executionBlockerPredicate() { return and( not(conversationRecoveryActionPredicate()!), not(preDispatchExecutionWaitRecoveryActionPredicate()), + sql`not (${issueRecoveryActions.cause} = 'legacy_execution_requires_reconciliation' and exists ( + select 1 from ${heartbeatRuns} + where ${heartbeatRuns.companyId} = ${issueRecoveryActions.companyId} + and ${heartbeatRuns.id} = ${canonicalExecutionUuid(sql`coalesce(${issueRecoveryActions.evidence}->>'runId', ${issueRecoveryActions.evidence}->>'sourceRunId')`)} + and ${issueRecoveryActions.sourceIssueId} = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)} + and (${heartbeatRuns.issueId} is null or ${heartbeatRuns.issueId} = ${issueRecoveryActions.sourceIssueId}) + and ${neverStartedDispatchCancellationPredicate()} + and ${heartbeatRuns.errorCode} <> 'issue_continuation_waiting_on_review' + and coalesce(${issueRecoveryActions.evidence}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive' + and coalesce(${issueRecoveryActions.evidence}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA} + ))`, or(unreconciledNativeExportPredicate(), and( inArray(issueRecoveryActions.cause, [...EXECUTION_RECONCILIATION_CAUSES]), or(inArray(issueRecoveryActions.status, ["active", "escalated"]), diff --git a/server/src/services/github-connection-events.ts b/server/src/services/github-connection-events.ts index af936353bdd..e84092ed826 100644 --- a/server/src/services/github-connection-events.ts +++ b/server/src/services/github-connection-events.ts @@ -146,7 +146,7 @@ function normalizeLeasedPayload(event: LeasedEvent): Record { return compact(base); } -function bindingRows(rows: Array<{ +export function githubConnectionEventBindings(rows: Array<{ grant: typeof connectionGrants.$inferSelect; connection: typeof toolConnections.$inferSelect; }>): GitHubBinding[] { @@ -259,7 +259,7 @@ export function githubConnectionEventService( eq(toolConnections.enabled, true), sql`exists (select 1 from companies where id = ${connectionGrants.companyId} and status <> 'archived')`, )); - return bindingRows(rows); + return githubConnectionEventBindings(rows); } async function applyPullRequestEvent(database: Db, companyId: string, event: LeasedEvent, effects: Array<(committedDb: Db) => Promise>) { diff --git a/server/src/services/heartbeat-run-events.ts b/server/src/services/heartbeat-run-events.ts index c65b078bc68..1f895cc5365 100644 --- a/server/src/services/heartbeat-run-events.ts +++ b/server/src/services/heartbeat-run-events.ts @@ -89,7 +89,10 @@ export async function appendHeartbeatRunEvent( input: AppendHeartbeatRunEventInput, ): Promise { const projectsRetention = input.eventType === "lifecycle" && Boolean(input.message?.startsWith(RETRY_EXHAUSTION_MESSAGE_PREFIX)); - const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType); + // Terminal proof may arrive after controller retirement and a quiet Stop + // check. Register the scoped wake before writing it, in the same transaction. + const projectsRecovery = ["native.process_start_requested", "native.process_identity_recorded", "native.local_process_stopped"].includes(input.eventType) + || Boolean(input.nativeSource && ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(input.eventType)); const projectsReassessment = readNativeEventEvidenceOutcome(input.payload?.prpEvent) !== null; const projectsProgress = isSafeNativeChatProgressEventType(input.eventType); const projectsResponse = isNativeResponseEvidence(input.eventType, input.payload?.prpEvent); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 4ddcfc16a7e..68bf5fd630a 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1,3 +1,4 @@ +import { resolveProviderApiPlacement } from "./native-runtime/provider-api-placement.js"; import { registerDeferredReceiptWork } from "./heartbeat/deferred-receipt-work.js"; import { reconcileTaskChatRetryReceipts } from "./task-retry.js"; import { registerIssueRecoveryWork } from "./recovery/work.js"; @@ -2266,7 +2267,8 @@ export function heartbeatService( const defaultIsolatedWorkspacesEnabled = isolatedWorkspacesEnabled && experimentalInstanceSettings.enableIsolatedWorkspacesByDefault; - const externalProvider = heartbeatExternalProvider({ run, agent }); + const providerApiPlacement = resolveProviderApiPlacement({ run, agent }); + const externalProvider = heartbeatExternalProvider({ run, agent }) ?? providerApiPlacement?.provider; const [taskWorkspaceIntentRow] = issueId && !externalProvider ? await db.select({ intent: issues.workspaceSelection }).from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))) : []; const freshWorkspace = await readFreshWorkspaceReadiness(db, run); @@ -2968,7 +2970,7 @@ export function heartbeatService( runtimeConfig = { ...runtimeConfig, env: gitExecutionEnv }; for (const key of MANAGED_GITHUB_TOKEN_KEYS) secretKeys.add(key); context.githubAuthenticationMode = useHostGitHub ? "host" : "managed"; - if (!useHostGitHub) { + if (!useHostGitHub && !providerApiPlacement) { const githubLaunchers = await timePreparation("github_launchers", () => prepareHeartbeatGitHubLaunchers({ native: agent.adapterType === "paperclip_runner", githubConfigured: githubSelection.configured, @@ -2990,7 +2992,7 @@ export function heartbeatService( runtimeConfig = { ...runtimeConfig, env: githubLaunchers.env }; secretKeys.add("PAPERCLIP_GITHUB_BROKER_TOKEN"); } - context.paperclipEnvironment = { + context.paperclipEnvironment = selectedEnvironment && activeEnvironmentLease ? { id: selectedEnvironment.id, name: selectedEnvironment.name, driver: selectedEnvironment.driver, @@ -3017,7 +3019,7 @@ export function heartbeatService( : undefined, } : {}), - }; + } : { placement: providerApiPlacement }; await db .update(heartbeatRuns) .set({ @@ -3244,8 +3246,8 @@ export function heartbeatService( executionWorkspace, persistedExecutionWorkspace, executionTarget, - environmentDriver: selectedEnvironment.driver, - leaseMetadata: activeEnvironmentLease.lease.metadata, + environmentDriver: selectedEnvironment?.driver ?? "provider_api", + leaseMetadata: activeEnvironmentLease?.lease.metadata ?? {}, })); const adapterEnv = Object.fromEntries( Object.entries(parseObject(runtimeConfig.env)).filter( @@ -3932,6 +3934,9 @@ export function heartbeatService( await settleNativeRunProcess(db, run.id).catch(error => { logger.warn({ err: error, runId: run.id }, "failed to release settled native process identity"); }); + await acknowledgeRemoteStop(run.id, run.companyId).catch(error => { + logger.warn({ err: error, runId: run.id }, "failed to acknowledge Stop after native process settlement"); + }); } await db.update(heartbeatRuns).set({ executionStage: sql`case when ${heartbeatRuns.status} = 'cancelled' then 'settled' else ${heartbeatRuns.executionStage} end`, diff --git a/server/src/services/heartbeat/deferred-receipt-work.ts b/server/src/services/heartbeat/deferred-receipt-work.ts index 9cca7528228..b799f10b0a7 100644 --- a/server/src/services/heartbeat/deferred-receipt-work.ts +++ b/server/src/services/heartbeat/deferred-receipt-work.ts @@ -6,7 +6,7 @@ import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import type { DeferredExecutionScope, DeferredReceiptRecoveryOptions } from "../deferred-execution-notifications.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; -type Entry = { id: string; companyId: string; dueAt: number }; +type Entry = { id: string; companyId: string; dueAt: number; immediate: boolean }; type NativeIntentOptions = DeferredExecutionScope & { onDeferredDeadline(at: number): void; onError(error: unknown): void; @@ -39,6 +39,11 @@ export function registerDeferredReceiptWork( }, decodeScope: hint => hint === "*" ? undefined : JSON.parse(hint), matches: (entry, scope) => !scope || entry.companyId === scope.companyId && (!scope.requestId || entry.id === scope.requestId), + // A busy company can invalidate a paginated scan before the pending + // batch drains. Keep unread ready receipts ahead of freshly reloaded + // receipts already checked; actual wait/claim deadlines still replace. + mergeEntry: (pending, loaded) => pending.immediate && loaded.immediate + ? { ...loaded, dueAt: Math.min(pending.dueAt, loaded.dueAt) } : loaded, load: async scope => { const key = scope ? JSON.stringify({ companyId: scope.companyId, requestId: scope.requestId }) : "*"; if (!enabled) { cursors.delete(key); return []; } @@ -76,12 +81,15 @@ export function registerDeferredReceiptWork( )).orderBy(asc(agentWakeupRequests.id)).limit(100); const complete = rows.length < 100; if (complete) cursors.delete(key); else cursors.set(key, rows.at(-1)!.id); - return { rows: rows.map(row => ({ id: row.id, companyId: row.companyId, - dueAt: kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000 + const now = Date.now(); + return { rows: rows.map(row => { + const deadline = kind === "native" && row.status === "claimed" && row.claimedAt ? row.claimedAt.getTime() + 60_000 : kind === "native" && row.error ? row.updatedAt.getTime() + 30_000 : kind === "deferred" && row.delayed && !row.interruptedLegacy ? row.updatedAt.getTime() + 30_000 - : Date.now(), - })), complete }; + : null; + const immediate = deadline === null || deadline <= now; + return { id: row.id, companyId: row.companyId, dueAt: immediate ? now : deadline, immediate }; + }), complete }; }, check: async entry => { if (!enabled) return null; @@ -101,7 +109,7 @@ export function registerDeferredReceiptWork( : kind === "deferred" && current.status === "deferred_issue_execution" && !current.interruptedLegacy && (current.payload?.queuedCommentInterrupt || current.payload?.executionWait) ? current.updatedAt.getTime() + 30_000 : null; - if (due !== null && due > Date.now()) return { ...entry, dueAt: due }; + if (due !== null && due > Date.now()) return { ...entry, dueAt: due, immediate: false }; } const errors: unknown[] = []; let next: number | null = null; @@ -117,7 +125,7 @@ export function registerDeferredReceiptWork( catch (error) { failed(error); } } if (errors.length) throw new AggregateError(errors, `Deferred receipt ${entry.id} could not settle`); - return next === null ? null : { ...entry, dueAt: next }; + return next === null ? null : { ...entry, dueAt: next, immediate: false }; }, }, canRun); } diff --git a/server/src/services/heartbeat/environment-selection.ts b/server/src/services/heartbeat/environment-selection.ts index 41ba6e44d20..58c2713c9c2 100644 --- a/server/src/services/heartbeat/environment-selection.ts +++ b/server/src/services/heartbeat/environment-selection.ts @@ -1,3 +1,4 @@ +import { assertProviderApiPlacementPolicy, resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { heartbeatExternalProvider, nativeExecutionUsesTaskWorkspace } from "./external-provider.js"; import { heartbeatRuns, issues, type Db, type agents } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; @@ -106,16 +107,18 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron : null; // This retained name covers both external bridges; neither adopts task files. const isDotRun = heartbeatExternalProvider({ run, agent }) !== null; + const providerApiPlacement = resolveProviderApiPlacement({ run, agent }); + const controllerOnly = isDotRun || providerApiPlacement !== null; const persistedNativeExecutionWorkspaceId = - persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null; + providerApiPlacement ? null : persistedNativeExecutionInput?.binding.executionWorkspaceId ?? null; const fresh = input.task.freshWorkspace?.mode === "fresh_isolated_workspace"; - const requestedExecutionWorkspaceId = isDotRun ? null : + const requestedExecutionWorkspaceId = controllerOnly ? null : persistedNativeExecutionWorkspaceId ?? readNonEmptyString(issueRef?.executionWorkspaceId); let existingExecutionWorkspace = requestedExecutionWorkspaceId ? await executionWorkspacesSvc.getStoredById(agent.companyId, requestedExecutionWorkspaceId) : null; - if (existingExecutionWorkspace && issueId && !isDotRun && !fresh + if (existingExecutionWorkspace && issueId && !controllerOnly && !fresh && existingExecutionWorkspace.id === readNonEmptyString(issueRef?.executionWorkspaceId) && (existingExecutionWorkspace.status === "archived" || existingExecutionWorkspace.status === "cleanup_failed") && (readGitCacheExpiration(existingExecutionWorkspace as Parameters[0])?.workspaceRemoved @@ -154,7 +157,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron const workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: requestedExecutionWorkspaceId, - issueExecutionWorkspacePreference: isDotRun ? null : nativeRecoveryExecutionWorkspaceId + issueExecutionWorkspacePreference: controllerOnly ? null : nativeRecoveryExecutionWorkspaceId ? "reuse_existing" : (issueRef?.executionWorkspacePreference ?? null), existingExecutionWorkspaceStatus: @@ -168,11 +171,11 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron : null; const workspaceAuthorizationActor = { type: "agent" as const, agentId: agent.id, companyId: agent.companyId, source: "agent_jwt" as const, runId: run.id, onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }; - if (!isDotRun && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { + if (!controllerOnly && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { const workspaceRequiringAccess = reusableExistingExecutionWorkspace ?? boundSourceWorkspace; if (workspaceRequiringAccess && issueId) { await assertTaskWorkspaceExecutionAccess(db, workspaceAuthorizationActor, agent.companyId, { issueId, workspaceId: workspaceRequiringAccess.id }); - } else if (!isDotRun && executionProjectId && (hasSelectedWorkspaceSource || workspaceProjectContext?.hasWorkspace)) { + } else if (!controllerOnly && executionProjectId && (hasSelectedWorkspaceSource || workspaceProjectContext?.hasWorkspace)) { await assertTaskWorkspaceSourceProjectAccess(db, workspaceAuthorizationActor, agent.companyId, executionProjectId); } } @@ -191,10 +194,12 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron const managedSandboxOnly = (await instanceSettings.getExperimental()).enableManagedSandboxOnly === true; - const managedSandboxEnvironment = managedSandboxOnly + const managedSandboxEnvironment = managedSandboxOnly && !providerApiPlacement ? await environmentsSvc.findManagedSandboxEnvironment(agent.companyId) : null; - const environmentResolution = resolveExecutionWorkspaceEnvironmentId({ + const environmentResolution = providerApiPlacement + ? { environmentId: null, source: "default" as const } + : resolveExecutionWorkspaceEnvironmentId({ lowTrustIssueEnvironmentId: trustPreset.kind === "low_trust_review" ? parseIssueExecutionWorkspaceSettings(issueContext?.executionWorkspaceSettings, {includeEnvironmentId: true})?.environmentId : null, agentDefaultEnvironmentId: agent.defaultEnvironmentId, @@ -218,8 +223,9 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron }; const executionForcedToKubernetes = isExecutionForcedToKubernetes(executionPolicy); + if (providerApiPlacement) assertProviderApiPlacementPolicy(executionPolicy); let selectedEnvironmentId = environmentResolution.environmentId; - if (executionForcedToKubernetes) { + if (executionForcedToKubernetes && !providerApiPlacement) { let kubernetesEnvironment = await environmentsSvc.findKubernetesEnvironment(agent.companyId); if (!kubernetesEnvironment) { @@ -292,7 +298,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron : selectedEnvironmentId ? await environmentsSvc.getById(selectedEnvironmentId) : null; - const nativeChatWorkspaceScope = isDotRun ? null : await findNativeChatWorkspaceScope(db, { + const nativeChatWorkspaceScope = controllerOnly ? null : await findNativeChatWorkspaceScope(db, { adapterType: agent.adapterType, environmentDriver: selectedEnvironmentForConfig?.driver ?? null, companyId: agent.companyId, @@ -391,7 +397,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron // project or issue settings request serialization. Sandbox protection still uses // the existing holder staleness and workspace_busy retry ladder. if ( - !isDotRun && issueRef?.projectWorkspaceId && + !controllerOnly && issueRef?.projectWorkspaceId && effectiveExecutionWorkspaceMode === "shared_workspace" ) { const workspaceHolder = await findSharedWorkspaceHolder({ @@ -472,18 +478,18 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron ); } } - if (!isDotRun && taskWorkspaceIntent && !boundSourceWorkspace && !nativeRecoveryExecutionWorkspaceId) { + if (!controllerOnly && taskWorkspaceIntent && !boundSourceWorkspace && !nativeRecoveryExecutionWorkspaceId) { await executionWorkspacesSvc.validateStoredSelectionForRun({ companyId: agent.companyId, actor: { type: "agent", agentId: agent.id, companyId: agent.companyId, runId: run.id, source: "agent_jwt", onBehalfOfUserId: responsibleUserId === "local-board" ? null : responsibleUserId }, selection: taskWorkspaceIntent.selection, issueId, preserveImplicitDefaultSettings: ["project", "task_default"].includes(taskWorkspaceIntent.source) }); } - const explicitTaskDirectory = !isDotRun && taskWorkspaceIntent?.selection.kind === "task_directory"; + const explicitTaskDirectory = !controllerOnly && taskWorkspaceIntent?.selection.kind === "task_directory"; const explicitDirectoryStrategies = [ ...(reusableExistingExecutionWorkspace?.providerType === "adapter_managed" ? [config.workspaceStrategy] : []), issueAssigneeOverrides?.adapterConfig?.workspaceStrategy, projectExecutionWorkspacePolicy?.workspaceStrategy, issueExecutionWorkspaceSettings?.workspaceStrategy]; - const boundTaskDirectory = !isDotRun && !fresh && !nativeRecoveryExecutionWorkspaceId && issueId !== null && issueRef !== null && + const boundTaskDirectory = !controllerOnly && !fresh && !nativeRecoveryExecutionWorkspaceId && issueId !== null && issueRef !== null && requestedShouldReuseExisting && requestedExecutionWorkspaceMode === "isolated_workspace" && !workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId && taskWorkspaceIntent?.selection.kind !== "configured_source" && @@ -503,7 +509,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron if (retainedDirectory && !boundTaskDirectory) throw new WorkspaceValidationFailure("The saved Retry task directory is unavailable.", { workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId }, }); - const useIsolatedTaskDirectory = !isDotRun && issueRef !== null && (reuseTaskDirectory || (fresh && !workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId) || shouldUseIsolatedTaskDirectory({ + const useIsolatedTaskDirectory = !controllerOnly && issueRef !== null && (reuseTaskDirectory || (fresh && !workspaceProjectContext?.hasWorkspace && !issueRef.projectWorkspaceId) || shouldUseIsolatedTaskDirectory({ trustPreset: trustPreset.kind, environmentDriver: selectedEnvironmentForConfig?.driver ?? null, mode: requestedExecutionWorkspaceMode, @@ -520,6 +526,7 @@ export async function selectHeartbeatEnvironment(db: Db, input: HeartbeatEnviron persistedRunnerProfile, persistedNativeExecutionInput, isDotRun, + providerApiPlacement, existingExecutionWorkspace, nativeRecoveryExecutionWorkspaceId, workspaceReuseRequest, diff --git a/server/src/services/heartbeat/external-provider.ts b/server/src/services/heartbeat/external-provider.ts index 6dfb45a4516..5cb656a23a0 100644 --- a/server/src/services/heartbeat/external-provider.ts +++ b/server/src/services/heartbeat/external-provider.ts @@ -24,5 +24,5 @@ type TaskWorkspaceExecution = Exclude)?.deadlineAt)) || Date.now(), })); } else { + // Retirement may commit before Stop acknowledgement. This is only a + // work hint; the owner rechecks the complete provider/process proof. + const pendingApiStop = and(eq(heartbeatRuns.status, "cancelled"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"), + sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`, + sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`); const candidates = await db.select({ id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, nextAttemptAt: nativeRunFinalizations.nextAttemptAt, leaseExpiresAt: nativeRunFinalizations.leaseExpiresAt }) .from(heartbeatRuns).leftJoin(nativeRunFinalizations, and(eq(nativeRunFinalizations.runId, heartbeatRuns.id), @@ -76,9 +81,9 @@ export function registerLiveRecoveryWork(db: Db, coordinator: ReturnType'nativeQuestionCancellation' is null`) : and(eq(heartbeatRuns.runtimeMode, "native"), inArray(heartbeatRuns.status, ["failed", "cancelled", "timed_out", "interrupted"]), - isNotNull(heartbeatRuns.finishedAt), or(isNotNull(heartbeatRuns.processPid), - isNotNull(heartbeatRuns.processGroupId), isNotNull(heartbeatRuns.processStartedAt)), - settledNativeProcessIdentityCondition(db)), + isNotNull(heartbeatRuns.finishedAt), or(pendingApiStop, + and(or(isNotNull(heartbeatRuns.processPid), isNotNull(heartbeatRuns.processGroupId), + isNotNull(heartbeatRuns.processStartedAt)), settledNativeProcessIdentityCondition(db)))), )).orderBy(asc(heartbeatRuns.id)).limit(100); rows = candidates.map(row => ({ id: row.id, companyId: row.companyId, dueAt: kind === "native" ? Math.max(Date.now(), row.nextAttemptAt?.getTime() ?? 0, row.leaseExpiresAt?.getTime() ?? 0) : Date.now() })); diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index 48d1a532c27..40351ddfb1a 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -2775,7 +2775,8 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD scope.onDeferredDeadline(Date.now() + 30_000); return; } - await releaseSettledNativeRunProcess(db, scope.runId, scope); + const outcome = await releaseSettledNativeRunProcess(db, scope.runId, scope); + if (outcome !== "pending") await acknowledgeRemoteStop(scope.runId, scope.companyId); }, orphanLease: scope => sweepOrphanedActiveLeases({ ...scope, backoffMs: 5 * 60_000 }), shutdown: scope => sweepLegacyShutdownWorkspaceSettlements(db, new Date(), scope), diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index bbcd348b1ca..fef333bc61a 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; -import { activityLog, agents, agentWakeupRequests, caseEvents, caseHandoffs, cases, companies, environmentLeases, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { activityLog, agents, agentWakeupRequests, caseEvents, caseHandoffs, cases, companies, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issues, nativeRunFinalizations } from "@paperclipai/db"; import { eq, inArray, sql } from "drizzle-orm"; import { afterEach, describe, expect, it, vi } from "vitest"; import { runningProcesses } from "../../adapters/index.js"; @@ -24,9 +24,18 @@ import { caseHandoffService } from "../case-handoffs.js"; import { withQueuedCommentIdsInWakePayload } from "../issue-queued-comment-queue.js"; import { instanceSettingsService } from "../instance-settings.js"; import { issueService } from "../issues.js"; +import { buildNativeExecutionInput } from "../native-runtime/native-execution-input.js"; +import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; +import { acknowledgedNativeApiStopExecutionHasStopped } from "../acknowledged-native-stop.js"; +import { nativeSha256 } from "../native-runtime/canonical.js"; import { callbacks, runControlPostgresSupport, setupRunControlDatabase } from "./__tests__/run-control-fixtures.js"; vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); +vi.mock("../acknowledged-native-stop.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, acknowledgedNativeApiStopExecutionHasStopped: vi.fn(actual.acknowledgedNativeApiStopExecutionHasStopped) }; +}); vi.mock("../adapter-execution-control.js", async (importOriginal) => { const actual = await importOriginal(); return { @@ -628,6 +637,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat releasedAt: proof === "missing" ? null : new Date(), cleanupStatus: proof === "missing" ? null : "success", metadata: proof === "missing" ? {} : { remoteExecutionTermination: receipt } }); const control = createHeartbeatRunControl(db, callbacks(db)); + vi.mocked(publishLiveEvent).mockClear(); await control.acknowledgeRemoteStop(run.id, company.id); await control.acknowledgeRemoteStop(run.id, company.id); const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); @@ -635,6 +645,149 @@ describe.skipIf(!runControlPostgresSupport.supported)("heartbeat run-control dat if (proof === "confirmed") expect(stopped.resultJson?.executionCancellation).toMatchObject({ proof: "provider_termination_receipt" }); expect(stopped.resultJson?.workspaceRestoreFailure).toBe("restore_failed"); expect(stopped.resultJson?.workspaceRestoreRecovery).toMatchObject({ leaseIds: [leaseId] }); + expect(vi.mocked(publishLiveEvent).mock.calls).toEqual(proof === "confirmed" ? [[{ + companyId: company.id, + type: "heartbeat.run.status", + payload: expect.objectContaining({ runId, agentId: agent.id, status: "cancelled" }), + }]] : []); + }); + + it.each(["confirmed", "missing_process_stop", "later_start", "missing_terminal", "unproven_terminal", + "foreign_terminal", "foreign_binding", "missing_placement", "unexpected_lease", "running_pid", + "wrong_intent", "pending_ack", "unresolved_mode", "missing_hash", "wrong_hash", "malformed_terminal", + "race_event_start", "race_profile_binding", "race_benign_event", + "natural_completed", "natural_cancelled", "natural_error", "natural_expired", + "mismatched_terminal_status", "local_failure", "unproven_cancelled", + "pre_stop_completed", "pre_stop_cancelled", "pre_stop_error", "pre_stop_expired", "pre_stop_interrupted", + "pre_stop_later_turn", "pre_stop_unproven", + ] as const)("publishes an API-only native Stop only after its exact provider and controller stopped: %s", async proof => { + const { company, agent } = await fixture(); + const [issue] = await db.insert(issues).values({ companyId: company.id, title: "API-only Stop", assigneeAgentId: agent.id }).returning(); + const runId = randomUUID(), sessionId = randomUUID(), runnerId = randomUUID(); + const recordedAt = new Date(Date.now() - 1_000).toISOString(); + const nativeExecutionInput = buildNativeExecutionInput({ companyId: company.id, agentId: agent.id, + runId: proof === "foreign_binding" ? randomUUID() : runId, provider: "cursor_cloud", model: "explicit-model", + cursorCloud: { schema: "paperclip.cursor-cloud.config.v1", repos: [{ url: "https://github.com/example/qualification" }] }, + issue: { id: issue.id, identifier: "RC-1", title: issue.title, description: null, workMode: "standard" }, + taskPrompt: "Stop the foreground command", normalizedSessionId: sessionId, + workspace: { id: randomUUID(), cwd: "/private/controller", repoUrl: null, repoRef: null, branchName: null }, + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Stop the foreground command", criteria: [{ id: "objective", requirement: "Stop the foreground command" }] } }, + runtimeContext: nativeRuntimeContextFixture(), + }); + const profile = { backend: "cursor_cloud_api", nativeExecutionInput, + ...(proof === "missing_placement" ? {} : { executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } }) }; + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId: company.id, agentId: agent.id, + nativeIssueId: issue.id, runtimeMode: proof === "unresolved_mode" ? "legacy" : "native", driverKind: "cursor_cloud_api", + nativeSessionId: sessionId, runnerInstanceId: runnerId, runnerProfileJson: profile, + status: "cancelled", finishedAt: new Date(), processPid: proof === "running_pid" ? process.pid : null, + resultJson: { executionCancellation: { state: "requested", requestedAt: recordedAt }, + cancelledByActorType: "user", cancelledByUserId: "operator", nativeCancellation: { + schema: "paperclip.native-cancellation.v1", runId: proof === "wrong_intent" ? randomUUID() : runId, + companyId: company.id, issueId: issue.id, scope: "run", reasonCode: "cancellation_run_only", + dispatchState: proof === "pending_ack" ? "pending" : "acknowledged", dispatched: true, + effects: ["release_run_resources"], recordedAt, acknowledgedAt: new Date().toISOString(), + intentAuditId: randomUUID(), acknowledgementAuditId: randomUUID(), + } }, + }).returning(); + if (proof === "unexpected_lease") await db.insert(environmentLeases).values({ companyId: company.id, issueId: issue.id, + heartbeatRunId: runId, provider: "local", status: "released", leasePolicy: "retain_on_failure", + releasedAt: new Date(), cleanupStatus: "success" }); + if (proof !== "missing_terminal") { + const preStop = proof.startsWith("pre_stop_"); + const terminalProof = proof === "pre_stop_later_turn" || proof === "pre_stop_unproven" ? "natural_completed" + : preStop ? `natural_${proof.slice("pre_stop_".length)}` : proof; + const terminalAt = preStop ? new Date(Date.parse(recordedAt) - 1_000) : new Date(); + const terminalType = terminalProof === "natural_cancelled" || terminalProof === "unproven_cancelled" ? "turn.cancelled" + : ["natural_error", "natural_expired", "local_failure"].includes(terminalProof) ? "turn.failed" + : terminalProof === "natural_completed" || terminalProof === "mismatched_terminal_status" ? "turn.completed" : "turn.interrupted"; + const terminalError = terminalProof === "natural_error" ? { code: "cursor_cloud_error", message: "Cursor Cloud run failed" } + : terminalProof === "natural_expired" ? { code: "cursor_cloud_expired", message: "Cursor Cloud run expired" } + : terminalProof === "local_failure" ? { code: "process_lost", message: "Synthetic local process loss" } : null; + const event = { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: terminalType, + sourceKind: "runner", sourceInstanceId: runnerId, sourceEventId: `${runnerId}:${runId}:1`, sourceSeq: 1, + runId, turnId: "api-stop-turn", normalizedSessionId: proof === "foreign_terminal" ? randomUUID() : sessionId, + emittedAt: terminalAt.toISOString(), priority: 1, + payload: { status: terminalProof === "mismatched_terminal_status" ? "interrupted" : terminalType.slice("turn.".length), error: terminalError, + ...(terminalProof === "natural_cancelled" ? { providerTerminalState: "completed", reason: "cancelled_after_provider_completed" } : {}), + ...(terminalProof === "unproven_cancelled" ? { providerTerminalState: "unknown", reason: "operator_cancel_requested" } : {}), + ...(proof === "unproven_terminal" || proof === "pre_stop_unproven" ? { providerTerminalObserved: false } : {}) } }; + await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId, + eventType: event.eventType, payload: { prpEvent: event }, nativeSource: { + sourceInstanceId: runnerId, sourceEventId: event.sourceEventId, sourceSeq: 1, + protocolSchemaVersion: 1, canonicalPayload: event, + } }); + if (preStop) await db.update(heartbeatRunEvents).set({ createdAt: terminalAt }).where(eq(heartbeatRunEvents.runId, runId)); + if (proof === "pre_stop_later_turn") { + const later = { ...event, eventType: "turn.started", sourceEventId: `${runnerId}:${runId}:2`, sourceSeq: 2, + turnId: "later-api-turn", emittedAt: new Date().toISOString(), payload: { status: "running" } }; + await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId, + eventType: later.eventType, payload: { prpEvent: later }, nativeSource: { + sourceInstanceId: runnerId, sourceEventId: later.sourceEventId, sourceSeq: 2, + protocolSchemaVersion: 1, canonicalPayload: later, + } }); + } + if (["missing_hash", "wrong_hash", "malformed_terminal"].includes(proof)) { + const malformed = { ...event, priority: undefined }; + await db.update(heartbeatRunEvents).set({ + sourcePayloadSha256: proof === "missing_hash" ? null : proof === "wrong_hash" ? "0".repeat(64) : nativeSha256(malformed), + ...(proof === "malformed_terminal" ? { payload: { prpEvent: malformed } } : {}), + }).where(eq(heartbeatRunEvents.runId, runId)); + } + } + const control = createHeartbeatRunControl(db, callbacks(db)); + vi.mocked(publishLiveEvent).mockClear(); + if (proof === "confirmed") { + await control.acknowledgeRemoteStop(run.id, company.id); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0] + .resultJson?.executionCancellation).toMatchObject({ state: "requested" }); + expect(publishLiveEvent).not.toHaveBeenCalled(); + } + if (proof !== "missing_process_stop") await appendHeartbeatRunEvent(db, { companyId: company.id, + agentId: agent.id, runId, eventType: "native.local_process_stopped", payload: { processPid: 424242, processGroupId: 424242 } }); + if (proof === "later_start") await appendHeartbeatRunEvent(db, { companyId: company.id, + agentId: agent.id, runId, eventType: "native.process_start_requested", payload: { targetKind: "local" } }); + if (proof.startsWith("race_")) { + const actual = await vi.importActual("../acknowledged-native-stop.js"); + vi.mocked(acknowledgedNativeApiStopExecutionHasStopped).mockImplementationOnce(async (...args) => { + const proven = await actual.acknowledgedNativeApiStopExecutionHasStopped(...args); + expect(proven).toBe(true); + if (proof === "race_event_start" || proof === "race_benign_event") { + const [before] = await db.select({ cursor: heartbeatRuns.nextEventSeq }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + await appendHeartbeatRunEvent(db, { companyId: company.id, agentId: agent.id, runId, + eventType: proof === "race_event_start" ? "native.process_start_requested" : "lifecycle", + payload: proof === "race_event_start" ? { targetKind: "local" } : { reason: "synthetic finalization log" } }); + const [after] = await db.select({ cursor: heartbeatRuns.nextEventSeq }).from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(after.cursor).toBe(before.cursor + 1); + } else { + await db.update(heartbeatRuns).set({ runnerProfileJson: { + ...profile, + nativeExecutionInput: { ...nativeExecutionInput, binding: { ...nativeExecutionInput.binding, issueId: randomUUID() } }, + } }).where(eq(heartbeatRuns.id, runId)); + } + return proven; + }); + } + await control.acknowledgeRemoteStop(run.id, company.id); + if (proof === "race_benign_event") expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0] + .resultJson?.executionCancellation).toMatchObject({ state: "acknowledged" }); + await control.acknowledgeRemoteStop(run.id, company.id); + const [stopped] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + const acknowledged = ["confirmed", "race_benign_event", "natural_completed", "natural_cancelled", "natural_error", "natural_expired", + "pre_stop_completed", "pre_stop_cancelled", "pre_stop_error", "pre_stop_expired", "pre_stop_interrupted"].includes(proof); + expect(stopped.resultJson?.executionCancellation).toMatchObject({ state: acknowledged ? "acknowledged" : "requested" }); + if (acknowledged) expect(stopped.resultJson?.executionCancellation).toMatchObject({ proof: "provider_api_stop_receipt" }); + expect(vi.mocked(publishLiveEvent).mock.calls).toEqual(acknowledged ? [[{ + companyId: company.id, + type: "heartbeat.run.status", + payload: expect.objectContaining({ runId, agentId: agent.id, status: "cancelled" }), + }]] : []); + if (acknowledged) { + const payload = vi.mocked(publishLiveEvent).mock.calls[0]![0].payload; + expect(payload).not.toHaveProperty("resultJson"); + expect(payload).not.toHaveProperty("executionCancellation"); + expect(payload).not.toHaveProperty("runnerProfileJson"); + } }); it("keeps pre-Stop input pending until a later user message authorizes one continuation", async () => { diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 835ac1fa078..7555e16fe92 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -12,6 +12,8 @@ import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED, readNativeLocalProc import { isHeartbeatRunTerminalStatus } from "./run-lifecycle.js"; import { pendingCaseHandoffForRun, cancelCaseHandoffsForIssue, isCaseHandoffSourceReleased } from "./case-handoff-lifecycle.js"; import { publishActivity, type ActivityPublication } from "../activity-log.js"; +import { buildHeartbeatRunStatusLiveEventPayload } from "../heartbeat-run-status-payload.js"; +import { publishLiveEvent } from "../live-events.js"; import { DEFERRED_WAKE_CONTEXT_KEY } from "./queue.js"; import { CANCELLABLE_HEARTBEAT_RUN_STATUSES } from "./recovery.js"; import { @@ -37,6 +39,7 @@ import { settleSlackConversation } from "../slack-conversation-lifecycle.js"; import { readQueuedInteractionResponse } from "../queued-interaction-response.js"; import { isAcknowledgedNativeStop, + acknowledgedNativeApiStopExecutionHasStopped, acknowledgedNativeStopExecutionHasStopped, } from "../acknowledged-native-stop.js"; import { completeTerminatedRemoteNativeSessionCleanup } from "../../vendor/paperclip-runner/index.js"; @@ -506,27 +509,61 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont completeTerminatedRemoteNativeSessionCleanup({ companyId, runId, remoteCleanupScope }); } } - const stopped = state?.status === "cancelled" ? await getRun(runId) : null; - // Native interrupt acknowledgement is not physical containment. Publish the - // shared Stop receipt only after every exact remote lease confirms teardown. - // Keep workspace preservation separate: a stopped provider can still owe files. - if (stopped?.status === "cancelled" && - parseObject(stopped.resultJson?.executionCancellation).state === "requested" && - (stopped.runtimeMode === "native" ? isAcknowledgedNativeStop(stopped) - : stopped.runtimeMode === "legacy" && await runUsedConversationAdapter(db, stopped)) && - await remoteExecutionHasStopped(db, companyId, runId)) { - await db.update(heartbeatRuns).set({ - resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ - executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), - state: "acknowledged", acknowledgedAt: new Date().toISOString(), - proof: "provider_termination_receipt" }, - ...(stopped.runtimeMode === "legacy" ? { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } : {}), - })}::jsonb`, - updatedAt: new Date(), - }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), - eq(heartbeatRuns.status, "cancelled"), - sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`, - stopped.runtimeMode === "native" ? sql`${heartbeatRuns.resultJson}->'nativeCancellation' = ${JSON.stringify(stopped.resultJson?.nativeCancellation)}::jsonb` : undefined)); + // Finalization can append a harmless event after proof was read. Retry + // only a lost API compare-and-set, with fresh complete proof each time. + for (let attempt = 0; attempt < 3; attempt++) { + const stopped = state?.status === "cancelled" ? await getRun(runId) : null; + // Native interrupt acknowledgement is not physical containment. Require + // exact remote lease teardown, or the API provider's terminal receipt and + // the local controller's retirement, before publishing shared Stop proof. + // Keep workspace preservation separate: a stopped provider can still owe files. + if (stopped?.status === "cancelled" && stopped.companyId === companyId && + parseObject(stopped.resultJson?.executionCancellation).state === "requested" && + (stopped.runtimeMode === "native" ? isAcknowledgedNativeStop(stopped) + : stopped.runtimeMode === "legacy" && await runUsedConversationAdapter(db, stopped))) { + const apiStopped = stopped.runtimeMode === "native" && await acknowledgedNativeApiStopExecutionHasStopped(db, stopped); + if (!apiStopped && !await remoteExecutionHasStopped(db, companyId, runId)) return; + const acknowledged = await db.update(heartbeatRuns).set({ + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ + executionCancellation: { ...parseObject(stopped.resultJson?.executionCancellation), + state: "acknowledged", acknowledgedAt: new Date().toISOString(), + proof: apiStopped ? "provider_api_stop_receipt" : "provider_termination_receipt" }, + ...(stopped.runtimeMode === "legacy" ? { conversationContinuation: CONVERSATION_CONTINUATION_POLICY } : {}), + })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.status, "cancelled"), + sql`${heartbeatRuns.resultJson}->'executionCancellation'->>'state' = 'requested'`, + apiStopped ? and(eq(heartbeatRuns.runtimeMode, "native"), eq(heartbeatRuns.driverKind, "cursor_cloud_api"), + eq(heartbeatRuns.nextEventSeq, stopped.nextEventSeq), + eq(heartbeatRuns.agentId, stopped.agentId), eq(heartbeatRuns.nativeIssueId, stopped.nativeIssueId!), + isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId), + eq(heartbeatRuns.nativeSessionId, stopped.nativeSessionId!), eq(heartbeatRuns.runnerInstanceId, stopped.runnerInstanceId!), + sql`${heartbeatRuns.runnerProfileJson}->'executionPlacement' = ${JSON.stringify({ kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" })}::jsonb`, + sql`${heartbeatRuns.runnerProfileJson}->>'backend' = 'cursor_cloud_api'`, + sql`jsonb_build_object( + 'companyId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'companyId', + 'runId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'runId', + 'agentId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'agentId', + 'issueId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'binding'->'issueId' + ) = ${JSON.stringify({ companyId, runId, agentId: stopped.agentId, issueId: stopped.nativeIssueId })}::jsonb`, + sql`jsonb_build_object( + 'normalizedSessionId', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'session'->'normalizedSessionId', + 'driverKind', ${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'session'->'driverKind' + ) = ${JSON.stringify({ normalizedSessionId: stopped.nativeSessionId, driverKind: "cursor_cloud_api" })}::jsonb`, + sql`${heartbeatRuns.runnerProfileJson}->'nativeExecutionInput'->'provider'->>'kind' = 'cursor_cloud'`, + sql`not exists (select 1 from ${environmentLeases} where ${environmentLeases.companyId} = ${heartbeatRuns.companyId} + and ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id})`) : undefined, + stopped.runtimeMode === "native" ? sql`${heartbeatRuns.resultJson}->'nativeCancellation' = ${JSON.stringify(stopped.resultJson?.nativeCancellation)}::jsonb` : undefined)).returning({ id: heartbeatRuns.id }); + if (acknowledged.length > 0) { + publishLiveEvent({ + companyId: stopped.companyId, + type: "heartbeat.run.status", + payload: buildHeartbeatRunStatusLiveEventPayload(stopped), + }); + } + if (!apiStopped || acknowledged.length > 0) return; + } else return; } } diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index bcbe9cd71ba..6a0898f46f0 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -135,7 +135,7 @@ export interface HeartbeatRuntimeExecutionInput { executionTarget: Selection["workspace"]["executionTarget"]; workspaceOperationRecorder: ReturnType["createRecorder"]>; nativeWorkspaceSync: Selected["nativeWorkspaceSync"]; - workspaceRestoreSource: Parameters[3]; + workspaceRestoreSource: Parameters[3] | null; remoteExecution: unknown; }; config: Pick & { @@ -937,6 +937,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe input.effects.onWorkspaceRestoreFailure(requiredWorkspaceRestoreEvidence); // Retention is the fallback even if recording this receipt fails. input.effects.onProviderResourceDisposition("stop_and_retain"); + if (workspaceRestoreSource === null) throw new Error("legacy_workspace_restore_source_unavailable"); await recordLegacyWorkspaceRestoreFailure(db, run, requiredWorkspaceRestoreEvidence, workspaceRestoreSource); } for (const stream of ["stdout", "stderr"] as const) { diff --git a/server/src/services/heartbeat/runtime-selection.test.ts b/server/src/services/heartbeat/runtime-selection.test.ts index d3d3a8ba4c3..032a80e73c6 100644 --- a/server/src/services/heartbeat/runtime-selection.test.ts +++ b/server/src/services/heartbeat/runtime-selection.test.ts @@ -2,10 +2,11 @@ import { randomUUID } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { activityLog, agents, companies, completionContracts, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { activityLog, agents, companies, completionContracts, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, executionWorkspaces, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { CHAT_CONTROL_RECOVERY_ADMISSION_KEY } from "../chat-control-recovery-stop.js"; import { buildNativeRuntimeContext } from "../native-runtime/runtime-context.js"; import { nativeRuntimeContextFixture } from "../native-runtime/runtime-context.test-fixture.js"; @@ -28,7 +29,8 @@ vi.mock("../native-runtime/native-workspace-sync.js", async (original) => ({ prepareNativeWorkspaceSync: vi.fn(async () => null), })); -const support = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUNTIME_SELECTION_TEST_DATABASE_URL"); +const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () => { let database: Awaited>; let db: Db; @@ -36,7 +38,9 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = beforeAll(async () => { instructionRoot = await mkdtemp(join(tmpdir(), "paperclip-selection-instructions-")); await writeFile(join(instructionRoot, "AGENTS.md"), "Complete the assigned task.\n"); - database = await startEmbeddedPostgresTestDatabase("paperclip-runtime-selection"); + if (externalDatabaseUrl) await applyPendingMigrations(externalDatabaseUrl); + database = externalDatabaseUrl ? { connectionString: externalDatabaseUrl, cleanup: async () => {} } + : await startEmbeddedPostgresTestDatabase("paperclip-runtime-selection"); db = createDb(database.connectionString); await db.execute(sql`set client_min_messages = warning`); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); @@ -78,6 +82,65 @@ describe.skipIf(!support.supported)("heartbeat runtime selection boundary", () = const coordinators = (id: string) => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, id)); const sandbox = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", reusableLeaseConfigured: false }; + it.each([false, true])("persists the Cursor provider API placement and completion authority without a Computer lease (ephemeral: %s)", async ephemeral => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", String(ephemeral)); + const { input, run, issue } = await fixture(); + input.agent = { ...input.agent, adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification" } }; + await db.update(agents).set({ adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id)); + input.workspace.activeEnvironmentLease = null; + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig, + agent: input.agent, issue, target: null, workspaceId: null }); + const result = await selectHeartbeatRuntime(db, input); + expect(result.selected).toBe(true); + if (!result.selected) throw new Error("Expected native Cursor selection"); + expect(result.nativeExecution?.provider).toMatchObject({ kind: "cursor_cloud", model: "caller-model" }); + expect(await read(run.id)).toMatchObject({ runtimeMode: "native", driverKind: "cursor_cloud_api", runnerProfileJson: { + backend: "cursor_cloud_api", executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" }, + nativeExecutionInput: result.nativeExecution, + } }); + expect(await db.select().from(completionContracts)).toHaveLength(1); + expect(await coordinators(run.id)).toHaveLength(1); + expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled(); + expect(input.stopControllerLease).toHaveBeenCalledOnce(); + }); + + it.each(["cursor_cloud_api", null] as const)("restores a historical Cursor workspace binding without allocating a Computer (driver: %s)", async driverKind => { + const { input, run, issue } = await fixture(); + input.agent = { ...input.agent, adapterConfig: { provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification" } }; + await db.update(agents).set({ adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id)); + const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId, sourceIssueId: issue.id, + name: "Previously admitted Cursor workspace", mode: "shared_workspace", strategyType: "project_primary", cwd: "/previous-cursor-workspace" }).returning(); + input.workspace.persistedExecutionWorkspace = workspace; + input.workspace.activeEnvironmentLease = null; + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig, + agent: input.agent, issue, target: null, workspaceId: workspace.id }); + const selected = await selectHeartbeatRuntime(db, input); + if (!selected.selected || !selected.nativeExecution) throw new Error("Expected native Cursor selection"); + expect(selected.nativeExecution.binding.executionWorkspaceId).toBe(workspace.id); + const saved = await read(run.id); + const historicalProfile: Record = { ...saved.runnerProfileJson, sessionCheckpoint: { retained: "Cursor provider checkpoint" } }; + delete historicalProfile.executionPlacement; + await db.update(heartbeatRuns).set({ runnerProfileJson: historicalProfile, driverKind }).where(eq(heartbeatRuns.id, run.id)); + input.run = await read(run.id); + input.nativeRuntimeResolution = resolveHeartbeatNativeRuntimeMode({ persisted: input.run, enabled: true, runtimeConfig: {}, adapterConfig: input.agent.adapterConfig, + agent: input.agent, issue, target: null, workspaceId: null }); + input.session.persistedRunnerProfile = historicalProfile; + input.session.persistedNativeExecutionInput = selected.nativeExecution; + input.workspace.persistedExecutionWorkspace = null; + input.workspace.executionWorkspace.cwd = "/private-cursor-controller"; + vi.mocked(buildNativeRuntimeContext).mockClear(); + const resumed = await selectHeartbeatRuntime(db, input); + expect(resumed).toMatchObject({ selected: true, nativeExecution: selected.nativeExecution, nativeRunnerInstanceId: selected.nativeRunnerInstanceId }); + expect(await read(run.id)).toMatchObject({ driverKind: "cursor_cloud_api" }); + expect((await read(run.id)).runnerProfileJson).toMatchObject({ ...historicalProfile, + executionPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } }); + expect(await db.select().from(executionWorkspaces)).toEqual([workspace]); + expect(await db.select().from(completionContracts)).toHaveLength(1); + expect(await coordinators(run.id)).toHaveLength(1); + expect(prepareNativeWorkspaceSync).not.toHaveBeenCalled(); + expect(buildNativeRuntimeContext).not.toHaveBeenCalled(); + }); + it("persists and admits a standing-chat response contract without requiring the unavailable action", async () => { const { input, issue, agent, run } = await fixture(); const [conversation] = await db.update(issues).set({ title: "Chat with CEO", description: null, diff --git a/server/src/services/heartbeat/runtime-selection.ts b/server/src/services/heartbeat/runtime-selection.ts index 8ea310b1eb7..b73ca32b3aa 100644 --- a/server/src/services/heartbeat/runtime-selection.ts +++ b/server/src/services/heartbeat/runtime-selection.ts @@ -1,3 +1,4 @@ +import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js"; import { museRunnerBroker } from "../muse-runner-broker.js"; import { assertEphemeralNativeLifecycle, readRemoteWorkspaceAuthority } from "../remote-workspaces.js"; @@ -123,7 +124,7 @@ export interface HeartbeatRuntimeSelectionInput { persistedExecutionWorkspace: { id: string; metadata?: Record | null } | null; executionWorkspace: Omit[0]["workspace"], "id">; executionTarget: Parameters[0]["target"]; - activeEnvironmentLease: Pick[0], "lease">; + activeEnvironmentLease: Pick[0], "lease"> | null; isDotRun: boolean; projectRepositoryPaths: string[]; onCheckpoint?: Parameters[0]["onCheckpoint"]; @@ -279,7 +280,11 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele throw new Error("native_runtime_ineligible: issue is required"); } const nativeExecutionWorkspaceId = - persistedExecutionWorkspace?.id ?? run.id; + persistedNativeExecutionInput?.provider.kind === "cursor_cloud" + // Older Cursor runs keep their admitted binding identity. Their task + // files and Computer are not part of the recovered API controller. + ? persistedNativeExecutionInput.binding.executionWorkspaceId + : persistedExecutionWorkspace?.id ?? run.id; const nativeReviewContext = readNativeReviewAssignmentContext(context); const nativeReview = nativeReviewContext ? await measureRunPhase("runtime_selection.review", () => getNativeReviewAssignment(db, { companyId: agent.companyId, issueId: issueRef.id, agentId: agent.id, @@ -514,10 +519,10 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele ((executionTarget?.kind === "remote" && executionTarget.transport === "sandbox" && !!executionTarget.runnerTask) || supportsManagedNativeWarmSession({ ...nativeRuntimeResolution.profile, acpxAgent: parseObject(agent.adapterConfig).acpxAgent }, managedAiRuntime?.attribution)); const effectiveLifecyclePolicy = persistedNativeExecutionInput?.session.lifecyclePolicy ?? - (["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession + (["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) || managedAiRuntime && !supportsManagedWarmSession ? { mode: "per_turn" as const, idleTimeoutMs: null } : environmentLifecyclePolicy ?? agentLifecyclePolicy); - if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend)) { + if (!["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend)) { assertEphemeralNativeLifecycle(effectiveLifecyclePolicy.mode); } if ( @@ -774,7 +779,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele }), ) ?? `# ${issueRef.identifier ?? issueRef.id}: ${issueRef.title}`, - !["openai_dot_mcp", "grok_bot_mcp", "muse_external"].includes(nativeRuntimeResolution.profile.backend) && projectRepositoryPaths.length > 0 + !["openai_dot_mcp", "grok_bot_mcp", "muse_external", "cursor_cloud_api"].includes(nativeRuntimeResolution.profile.backend) && projectRepositoryPaths.length > 0 ? `## Project repositories\nThe task workspace also contains these editable Git repositories:\n${projectRepositoryPaths.map((repo) => `- ${repo}`).join("\n")}` : null, ].filter(Boolean).join("\n\n"), @@ -970,6 +975,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele runnerProfileJson: { ...nativeRuntimeResolution.profile, ...lockedProfile, + ...(nativeExecution?.provider.kind === "cursor_cloud" ? { executionPlacement: resolveProviderApiPlacement({ agent, run }) } : {}), nativePreparationFencingVersion: !lockedRun.runnerInstanceId && !lockedRun.nativeSessionId ? NATIVE_PREPARATION_FENCING_VERSION : lockedProfile.nativePreparationFencingVersion, @@ -1058,7 +1064,8 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele }); if (!nativeSelected) return { selected: false }; stopControllerLease(); - nativeWorkspaceSync = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null + if (nativeExecution?.provider.kind !== "cursor_cloud" && !activeEnvironmentLease) throw new Error("native_environment_lease_unavailable"); + nativeWorkspaceSync = nativeExecution?.provider.kind === "cursor_cloud" || readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata) ? null : await measureRunPhase("runtime_selection.workspace_sync", () => prepareNativeWorkspaceSync({ db, runId: run.id, @@ -1066,7 +1073,7 @@ export async function selectHeartbeatRuntime(db: Db, input: HeartbeatRuntimeSele workspaceId: nativeExecutionWorkspaceId, workspaceLocalDir: executionWorkspace.cwd, target: executionTarget, - lease: activeEnvironmentLease.lease, + lease: activeEnvironmentLease!.lease, restartRecovery: runOptions.nativeRestartRecovery, sameRunRecovery: Boolean(runOptions.nativeLeaseOwner), resourceDisposition: providerResourceDispositionForRun, diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 8a1d010f8a7..3a218e50b8e 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -126,6 +126,56 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace }; return { input, source, issue, run, envOrchestrator }; } + it.each([false, true])("places Cursor Cloud in its provider API without a Computer under managed-only policy (ephemeral: %s)", async ephemeral => { + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", String(ephemeral)); + const { input, source, issue, run } = await fixture(); + input.agent = { ...input.agent, adapterType: "paperclip_runner", adapterConfig: { + provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", + } }; + await db.update(agents).set({ adapterType: input.agent.adapterType, adapterConfig: input.agent.adapterConfig }).where(eq(agents.id, input.agent.id)); + const settingsBefore = await instanceSettingsService(db).get(); + try { + await instanceSettingsService(db).updateExperimental({ enableManagedSandboxOnly: true, enableNativeRunner: true }); + const selectionInput = { + run: input.run, agent: input.agent, + task: { issueRef: issue, issueContext: issue, issueId: issue.id, context: input.task.context, + executionProjectId: issue.projectId, workspaceProjectContext: { hasWorkspace: true }, boundSourceWorkspace: null, + hasSelectedWorkspaceSource: true, taskWorkspaceIntent: null, responsibleUserId: null }, + policy: { ...input.policy }, config: { config: input.agent.adapterConfig, sourceWorkspaceConfig: input.agent.adapterConfig, issueAssigneeOverrides: null }, + services: { environmentsSvc: environmentService(db), executionWorkspacesSvc: executionWorkspaceService(db), + instanceSettings: instanceSettingsService(db), findSharedWorkspaceHolder: async () => null, findExecutionWorkspaceHolder: async () => null }, + }; + const selection = await selectHeartbeatEnvironment(db, selectionInput); + expect(selection).toMatchObject({ providerApiPlacement: { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" }, + selectedEnvironmentId: null, selectedEnvironmentForConfig: null, executionPolicy: { managedSandboxOnly: true } }); + input.environment = { ...input.environment, selectedEnvironmentId: selection.selectedEnvironmentId, + selectedEnvironmentForConfig: selection.selectedEnvironmentForConfig, environmentResolution: selection.environmentResolution, + resolvedInstanceSettings: selection.resolvedInstanceSettings }; + input.policy.executionPolicy = selection.executionPolicy; + const acquire = vi.spyOn(input.services.envOrchestrator, "acquireForRun"); + const realize = vi.spyOn(input.services.envOrchestrator, "realizeForRun"); + const resolveWorkspace = vi.spyOn(input.services, "resolveWorkspaceForRun"); + const result = await prepareHeartbeatWorkspace(db, input); + expect(result).toMatchObject({ selectedEnvironment: null, activeEnvironmentLease: null, executionTarget: null, + persistedExecutionWorkspace: null, projectRepositoryPaths: [], hostExecutionWorkspaceConfig: {}, skipHostProvision: true }); + expect(result.executionWorkspace.cwd).toContain(`/cursor-cloud-controllers/${input.agent.companyId}/${input.agent.id}/${issue.id}`); + expect(result.executionWorkspace).toMatchObject({ projectId: null, repoUrl: null, repoRef: null, created: false }); + expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n"); + expect(acquire).not.toHaveBeenCalled(); expect(realize).not.toHaveBeenCalled(); expect(resolveWorkspace).not.toHaveBeenCalled(); + expect(await db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, run.id))).toEqual([]); + expect(await db.select().from(executionWorkspaces)).toEqual([]); + expect((await instanceSettingsService(db).getExperimental()).enableManagedSandboxOnly).toBe(true); + await instanceSettingsService(db).updateGeneral({ executionMode: "kubernetes" }); + await expect(selectHeartbeatEnvironment(db, selectionInput)).rejects.toMatchObject({ status: 422, + details: { code: "provider_api_kubernetes_execution_unsupported" } }); + } finally { + await instanceSettingsService(db).updateGeneral({ executionMode: settingsBefore.general.executionMode ?? "any" }); + await instanceSettingsService(db).updateExperimental({ + enableManagedSandboxOnly: settingsBefore.experimental.enableManagedSandboxOnly, + enableNativeRunner: settingsBefore.experimental.enableNativeRunner, + }); + } + }); it.each(["unconfigured", "operator-disabled"] as const)("auto-enrollment keeps %s native sandboxes on their existing host path", async mode => { const { input, source, envOrchestrator } = await fixture(); input.agent.adapterType = "paperclip_runner"; @@ -1713,7 +1763,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace if (transport !== "daytona") { await expect(prepareNativeWorkspaceSync({ db, runId: input.run.id, companyId: issue.companyId, workspaceId: first.persistedExecutionWorkspace!.id, workspaceLocalDir: first.executionWorkspace.cwd, - target, lease: result.activeEnvironmentLease.lease })).resolves.toBeNull(); + target, lease: result.activeEnvironmentLease!.lease })).resolves.toBeNull(); } expect(admittedRoot).not.toBe(first.executionWorkspace.cwd); } else { @@ -1817,7 +1867,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace const result = await prepareHeartbeatWorkspace(db, input); expect((await readIssue(issue.id)).executionWorkspaceId).toBe(result.persistedExecutionWorkspace!.id); expect(result.persistedExecutionWorkspace!.name).toBe("Realized workspace"); - result.activeEnvironmentLease.lease.metadata!.remoteCwd = "/rebound"; - expect(result.workspaceRestoreSource.metadata?.remoteCwd).toBe("/original"); + result.activeEnvironmentLease!.lease.metadata!.remoteCwd = "/rebound"; + expect(result.workspaceRestoreSource!.metadata?.remoteCwd).toBe("/original"); }); }); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 838e8c75fb2..15958e0c6b9 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1,3 +1,4 @@ +import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, RemoteWorkspaceWriterActiveError, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WorkspaceBusyDeferral } from "./retries.js"; import { WORKSPACE_BUSY_RETRY_REASON } from "../../modules/run-dispatch/index.js"; @@ -127,10 +128,10 @@ export interface HeartbeatWorkspacePreparationInput { executionPolicy: Parameters[0]; }; environment: { - selectedEnvironmentId: string; + selectedEnvironmentId: string | null; localEnvironment: Environment; selectedEnvironmentForConfig: Environment | null; - environmentResolution: ExecutionWorkspaceEnvironmentResolution; + environmentResolution: ExecutionWorkspaceEnvironmentResolution | { environmentId: null; source: "default" }; resolvedInstanceSettings: Awaited["get"]>>; }; config: { @@ -169,13 +170,14 @@ function readNonEmptyString(value: unknown): string | null { export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) { const sourceIssueId = input.reuse.reusableExistingExecutionWorkspace?.sourceIssueId ?? input.reuse.existingExecutionWorkspace?.sourceIssueId ?? input.task.issueId; - if (!sourceIssueId || input.reuse.isDotRun) return prepareHeartbeatWorkspaceLocked(db, input); + if (!sourceIssueId || input.reuse.isDotRun || resolveProviderApiPlacement(input)) return prepareHeartbeatWorkspaceLocked(db, input); return withIsolatedRepositorySourceLock(db, input.agent.companyId, sourceIssueId, () => prepareHeartbeatWorkspaceLocked(db, input)); } async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspacePreparationInput) { const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input; + const providerApiPlacement = resolveProviderApiPlacement(input); const fresh = input.task.freshWorkspace?.mode === "fresh_isolated_workspace"; const timeWorkspace = createHeartbeatPreparationTiming(run.id); const { @@ -217,7 +219,8 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } = input.reuse; // An ordinary clean task directory already has a new per-attempt identity. // It is not a repository source and must not pass through Git copy admission. - const freshNeedsSeed = fresh && !isDotRun && !useIsolatedTaskDirectory && + const controllerOnly = isDotRun || providerApiPlacement !== null; + const freshNeedsSeed = fresh && !controllerOnly && !useIsolatedTaskDirectory && !(nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId); const { envOrchestrator, @@ -237,14 +240,14 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } } const enrolledRemoteAuthority = readRemoteWorkspaceAuthority((reusableExistingExecutionWorkspace ?? existingExecutionWorkspace)?.metadata) !== null; - let remoteAuthority = !isDotRun && shouldUseRemoteWorkspaces({ + let remoteAuthority = !controllerOnly && shouldUseRemoteWorkspaces({ adapterType: agent.adapterType, environmentDriver: selectedEnvironmentForConfig?.driver ?? null, experimentalEnabled: (process.env.PAPERCLIP_EPHEMERAL_SERVER === "true" || resolvedInstanceSettings.experimental.remoteWorkspaceStorageEnabled) && Boolean(issueId && issueRef) && !projectExecutionWorkspacePolicy?.storageProviderPluginId, existingWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace, }); - assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, + if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, adapterType: agent.adapterType, pluginTaskWorkspace, remoteAuthority }); const assertRemotePreparationOwned = run.runtimeMode === "native" ? () => assertNativeSessionPreparationAuthority({ @@ -260,7 +263,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace "Remote workspace storage requires a task.", { reason: "remote_workspace_task_required" }); const remoteSourceState: { value: Extract>, { authority: "remote" }> | null } = { value: null }; const pluginConfig = parseObject(selectedEnvironmentForConfig?.config); - if (fresh && !isDotRun && pluginTaskWorkspace) { + if (fresh && !controllerOnly && pluginTaskWorkspace) { const plugin = typeof pluginConfig.pluginKey === "string" ? await pluginRegistryService(db).getByKey(pluginConfig.pluginKey) : null; const driver = plugin?.manifestJson.environmentDrivers?.find(value => value.driverKey === pluginConfig.driverKey); if (driver?.freshWorkspaceVersion !== 1) throw new WorkspaceValidationFailure( @@ -298,10 +301,11 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } : null, resolveSelectedEnvironmentDriver: async () => { + if (providerApiPlacement) return "provider_api"; const preflightEnvironment = await envOrchestrator.resolveEnvironment( { companyId: agent.companyId, - selectedEnvironmentId, + selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id, localEnvironmentId: localEnvironment.id, }, ); @@ -329,7 +333,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace remoteAuthority = false; // Source resolution can retain a configured host folder. Check the // resolved authority before materializing it or acquiring a sandbox. - assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, + if (!providerApiPlacement) assertEphemeralWorkspaceExecution({ externalConversation: isDotRun, adapterType: agent.adapterType, pluginTaskWorkspace, remoteAuthority }); } else { @@ -354,10 +358,10 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace }; } } - if (isDotRun) { + if (controllerOnly) { // This is private controller storage, never a provider filesystem. // External providers receive workspace.access=none and cwd=null. - const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", (persistedNativeExecutionInput?.provider.kind ?? mergedConfig.provider) === "muse" ? "muse-controllers" : "dot-controllers", agent.companyId, run.id); + const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", providerApiPlacement ? "cursor-cloud-controllers" : (persistedNativeExecutionInput?.provider.kind ?? mergedConfig.provider) === "muse" ? "muse-controllers" : "dot-controllers", agent.companyId, ...(providerApiPlacement ? [agent.id, issueId ?? run.id] : [run.id])); await fs.mkdir(cwd, { recursive: true, mode: 0o700 }); return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null, workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] }; @@ -497,7 +501,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace let providerFresh: PluginFreshWorkspace | undefined; let previousProviderWorkspaceId: string | null = null; let previousProviderBindingRevision = 0; - if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && !isDotRun && issueId) { + if (input.task.freshWorkspace?.mode === "fresh_isolated_workspace" && !controllerOnly && issueId) { const [task] = await db.select({ workspaceId: issues.executionWorkspaceId, revision: issues.workspaceBindingRevision }).from(issues) .where(and(eq(issues.id, issueId), eq(issues.companyId, agent.companyId))); previousProviderWorkspaceId = task?.workspaceId ?? null; @@ -513,7 +517,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace previousWorkspaceId: previousProviderWorkspaceId }); } } - const pluginPrepared = isDotRun || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, { + const pluginPrepared = controllerOnly || remoteAuthority ? null : await preparePluginWorkspace(db, input.services.workspaceWorkers, { companyId: agent.companyId, projectId: resolvedWorkspace.projectId, projectWorkspaceId: resolvedWorkspace.workspaceId, issueId, runId: run.id, agentId: agent.id, responsibleUserId, existingWorkspaceId: providerFresh ? providerFresh.previousWorkspaceId : reusableExistingExecutionWorkspace?.id ?? issueRef?.executionWorkspaceId ?? null, @@ -544,7 +548,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace boundWorkspace: reusableExistingExecutionWorkspace ?? existingExecutionWorkspace, })); const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh" || hostProvisionDriver === "plugin"; - const hostExecutionWorkspaceConfig = isDotRun ? {} : + const hostExecutionWorkspaceConfig = controllerOnly ? {} : stripHostWorkspaceProvision({ config: fresh && !remoteAuthority ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig, skipHostProvision, @@ -726,7 +730,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace warnings: [], created: false, branchCreatedByRuntime: false, baseRefSha: pluginPrepared.metadata?.workspaceProviderBaseSha as string | null, } as RealizedExecutionWorkspace, reusedExecutionWorkspace: Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } - : isDotRun || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision( + : controllerOnly || pluginTaskWorkspace || remoteAuthority ? { executionWorkspace: { ...executionWorkspaceBase, strategy: pluginTaskWorkspace || remoteAuthority ? latestWorkspaceStrategyType : "project_primary" as const, cwd: freshTaskReservation?.cwd ?? resolvedWorkspace.cwd, branchName: remoteSourceState.value?.source.kind === "git" ? remoteSourceState.value!.source.branchName ?? null : null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: (pluginTaskWorkspace || remoteAuthority) && workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace && Boolean(reusableExistingExecutionWorkspace), policy: workspaceReuseProvisioningPolicy } : await timeWorkspace("workspace_provision", () => provisionExecutionWorkspaceForFreshnessDecision( { requestedShouldReuseExisting, existingExecutionWorkspaceId: @@ -854,7 +858,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace const resolvedProjectWorkspaceId = resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree" ? reusableExistingExecutionWorkspace.projectWorkspaceId - : isDotRun ? null : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; + : controllerOnly ? null : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null; let persistedExecutionWorkspace: ExecutionWorkspace | null = null; let issueExecutionWorkspaceIdForRun = issueRef?.executionWorkspaceId ?? null; @@ -875,7 +879,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace const bindIssueToPersistedExecutionWorkspace = async ( workspace: ExecutionWorkspace | null, ) => { - if (isDotRun || !issueId || !workspace || nativeRecoveryExecutionWorkspaceId) { + if (controllerOnly || !issueId || !workspace || nativeRecoveryExecutionWorkspaceId) { return; } const nextIssueWorkspaceMode = @@ -996,6 +1000,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace try { if (remoteAuthority) { if (!remoteSourceState.value || !issueId) throw new Error("remote_workspace_source_missing"); + if (!selectedEnvironmentId) throw new Error("remote_workspace_environment_missing"); await assertRemotePreparationOwned(); const reserved = await reserveRemoteWorkspace({ db, companyId: agent.companyId, issueId, runId: run.id, agentId: agent.id, @@ -1053,7 +1058,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace ), }, )) - : !isDotRun && !persistedNativeExecutionInput && (resolvedProjectId || issueRef) + : !controllerOnly && !persistedNativeExecutionInput && (resolvedProjectId || issueRef) ? await timeWorkspace("workspace_record_create", () => executionWorkspacesSvc.create({ companyId: agent.companyId, projectId: resolvedProjectId, @@ -1178,7 +1183,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace const projectRepositoryPaths: string[] = []; if (remoteAuthority) projectRepositoryPaths.push(...(remoteSourceState.value?.source.repositories ?? []) .filter(repository => !repository.readOnly).map(repository => repository.relativePath)); - if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { + if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !controllerOnly && persistedExecutionWorkspace && issueId && !persistedNativeExecutionInput && !nativeRecoveryExecutionWorkspaceId) { const taskRepositories = await timeWorkspace("workspace_repository_admission", () => executionWorkspacesSvc.prepareTaskRepositoriesForAdmission({ companyId: agent.companyId, issueId, workspaceId: persistedExecutionWorkspace!.id, cwd: executionWorkspace.cwd, @@ -1231,6 +1236,21 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace }) .where(eq(heartbeatRuns.id, run.id)); } + if (providerApiPlacement) { + await controllerLease.assertOwned(); + return { + resolvedWorkspace, hostExecutionWorkspaceConfig, skipHostProvision: true, + latestWorkspaceConfigMetadata, workspaceConfigFreshness, workspaceOperationRecorder, + executionWorkspace, reusedExecutionWorkspace, resolvedWorkspaceReusePolicy, + persistedExecutionWorkspace: null, projectRepositoryPaths: [], + environmentAcquireStartedAtMs: Date.now(), selectedEnvironment: null, + activeEnvironmentLease: null, + realizationResult: { remoteExecution: null, executionTarget: null, persistedExecutionWorkspace: null, + lease: null, workspaceRealization: null }, + environmentRealizeEndedAtMs: Date.now(), workspaceRealization: null, + executionTarget: null, workspaceRestoreSource: null, + }; + } const environmentAcquireStartedAtMs = Date.now(); let acquiredEnvironment: Awaited< ReturnType @@ -1254,7 +1274,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace } acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ companyId: agent.companyId, - selectedEnvironmentId, + selectedEnvironmentId: selectedEnvironmentId ?? localEnvironment.id, localEnvironmentId: localEnvironment.id, adapterType: agent.adapterType, adapterConfig: parseObject(agent.adapterConfig), @@ -1423,7 +1443,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace if (wake) wake.executionContinuation = continuation; } } - if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !isDotRun && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId) + if (!pluginPrepared && !pluginTaskWorkspace && !remoteAuthority && !controllerOnly && issueId && (persistedNativeExecutionInput || nativeRecoveryExecutionWorkspaceId) && !(executionTarget?.kind === "remote" && (executionTarget.transport === "ssh" || (executionTarget.transport === "sandbox" && executionTarget.runnerTask)))) { const workspaceId = persistedNativeExecutionInput?.binding.executionWorkspaceId ?? nativeRecoveryExecutionWorkspaceId!; diff --git a/server/src/services/idle-api-activity.test.ts b/server/src/services/idle-api-activity.test.ts new file mode 100644 index 00000000000..e0e54ddfe83 --- /dev/null +++ b/server/src/services/idle-api-activity.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { IdleApiActivity } from "./idle-api-activity.js"; + +describe("authenticated API idle activity", () => { + it("is passive without successful authentication and expires without timers or dependency work", () => { + let monotonic = 1; + const activity = new IdleApiActivity(() => monotonic); + expect(activity.active()).toBe(false); + activity.record(); + expect(activity.active()).toBe(true); + monotonic += 599_999; + expect(activity.active()).toBe(true); + monotonic++; + expect(activity.active()).toBe(false); + }); + it("extends only from the actual latest successful use and ignores wall clock changes", () => { + let monotonic = 1; + const activity = new IdleApiActivity(() => monotonic); + activity.record(); + monotonic += 300_000; + activity.record(); + monotonic += 300_001; + expect(activity.active()).toBe(true); + monotonic += 299_999; + expect(activity.active()).toBe(false); + }); +}); diff --git a/server/src/services/idle-api-activity.ts b/server/src/services/idle-api-activity.ts new file mode 100644 index 00000000000..6c4879201f0 --- /dev/null +++ b/server/src/services/idle-api-activity.ts @@ -0,0 +1,12 @@ +/** Authenticated API use is real activity; rejecting an unknown key is not. */ +export class IdleApiActivity { + private until = 0; + constructor(private readonly now: () => number = () => performance.now()) {} + record(): void { + this.until = Math.max(this.until, this.now() + 10 * 60_000); + } + active(): boolean { + return this.now() < this.until; + } +} +export const idleApiActivity = new IdleApiActivity(); diff --git a/server/src/services/idle-api-wake-admission.test.ts b/server/src/services/idle-api-wake-admission.test.ts new file mode 100644 index 00000000000..604eb2cb4ef --- /dev/null +++ b/server/src/services/idle-api-wake-admission.test.ts @@ -0,0 +1,33 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { apiWakeFingerprint, createApiWakeAdmission, parseApiWakeAdmission, matchesApiWakeAdmission } from "./idle-api-wake-admission.js"; + +const token = "api-wake-vector-token"; +const keyHash = createHash("sha256").update(token).digest("hex"); +const input = { stackId: "vector-stack", ownerId: "00000000-0000-4000-8000-000000000001", capabilityGeneration: "a".repeat(64), challenge: "c".repeat(64) }; +const fingerprint = apiWakeFingerprint({ ...input, keyHash }); +const receipt = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: null }] }); +describe("API wake receipt wire vectors", () => { + it("uses exact domain-separated vectors and exports neither token nor database hash", () => { + expect(fingerprint).toBe("6478041b90f4665eeca8403186fefedc456bf1322f38dab9fb89212eecaafd43"); + expect(JSON.stringify(receipt)).not.toContain(token); + expect(JSON.stringify(receipt)).not.toContain(keyHash); + expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token, now: new Date("2099-01-01").getTime() })).toBe(true); + expect(matchesApiWakeAdmission(receipt, { stackId: "other", token, now: 0 })).toBe(false); + expect(matchesApiWakeAdmission(receipt, { stackId: input.stackId, token: "unknown", now: 0 })).toBe(false); + }); + it("enforces board expiry and canonical unique bounded complete receipts", () => { + const changedChallenge = createApiWakeAdmission({ ...input, challenge: "b".repeat(64), keys: receipt.keys }); + expect(matchesApiWakeAdmission(changedChallenge, { stackId: input.stackId, token, now: 0 })).toBe(false); + const expiring = createApiWakeAdmission({ ...input, keys: [{ fingerprint, expiresAt: new Date(1000).toISOString() }] }); + expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 999 })).toBe(true); + expect(matchesApiWakeAdmission(expiring, { stackId: input.stackId, token, now: 1000 })).toBe(false); + for (const invalid of [ + { ...receipt, generation: "b".repeat(64) }, { ...receipt, challenge: "b".repeat(64) }, + { ...receipt, extra: true }, { ...receipt, keys: [...receipt.keys, ...receipt.keys] }, + { ...receipt, keys: Array.from({ length: 1001 }, () => receipt.keys[0]) }, + { ...receipt, keys: [{ fingerprint, expiresAt: "2099-01-01" }] }, + { ...receipt, keys: [{ fingerprint: "d".repeat(64), expiresAt: null }, { fingerprint: "a".repeat(64), expiresAt: null }] }, + ]) expect(() => parseApiWakeAdmission(invalid)).toThrow(); + }); +}); diff --git a/server/src/services/idle-api-wake-admission.ts b/server/src/services/idle-api-wake-admission.ts new file mode 100644 index 00000000000..26ed499a231 --- /dev/null +++ b/server/src/services/idle-api-wake-admission.ts @@ -0,0 +1,56 @@ +import { createHash } from "node:crypto"; + +export type ApiWakeClaim = { generation: string; fingerprint: string }; +export interface ApiWakeAdmission { + version: 1; + stackId: string; + ownerId: string; + capabilityGeneration: string; + challenge: string; + generation: string; + keys: { fingerprint: string; expiresAt: string | null }[]; +} +const digest = (value: unknown): value is string => typeof value === "string" && /^[a-f0-9]{64}$/.test(value); +const object = (value: unknown): value is Record => Boolean(value && typeof value === "object" && !Array.isArray(value)); +const exact = (value: Record, keys: string[]) => Object.keys(value).length === keys.length && Object.keys(value).every(key => keys.includes(key)); +const generation = (value: Omit) => createHash("sha256").update(JSON.stringify([ + "paperclip-api-wake-receipt-v1", value.version, value.stackId, value.ownerId, value.capabilityGeneration, value.challenge, + value.keys.map(key => [key.fingerprint, key.expiresAt]), +])).digest("hex"); +export function apiWakeFingerprint(input: { stackId: string; ownerId: string; challenge: string; keyHash: string }): string { + if (!digest(input.keyHash) || !digest(input.challenge)) throw new Error("Invalid API wake fingerprint input"); + return createHash("sha256").update(JSON.stringify(["paperclip-api-wake-v1", input.stackId, input.ownerId, input.challenge, input.keyHash])).digest("hex"); +} +/** Strict bounded canonical wire contract. Parsing never grants tenant permissions. */ +export function parseApiWakeAdmission(value: unknown): ApiWakeAdmission { + if (!object(value) || !exact(value, ["version", "stackId", "ownerId", "capabilityGeneration", "challenge", "generation", "keys"]) + || value.version !== 1 || typeof value.stackId !== "string" || !/^[A-Za-z0-9_-]{1,200}$/.test(value.stackId) + || typeof value.ownerId !== "string" || !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(value.ownerId) + || !digest(value.capabilityGeneration) || !digest(value.challenge) || !digest(value.generation) + || !Array.isArray(value.keys) || value.keys.length > 1000) throw new Error("Invalid API wake receipt"); + let previous = ""; + for (const key of value.keys) { + if (!object(key) || !exact(key, ["fingerprint", "expiresAt"]) || !digest(key.fingerprint) || key.fingerprint <= previous + || !(key.expiresAt === null || typeof key.expiresAt === "string" && key.expiresAt.length <= 30 && Number.isFinite(Date.parse(key.expiresAt)) + && new Date(key.expiresAt).toISOString() === key.expiresAt)) throw new Error("Invalid API wake receipt key"); + previous = key.fingerprint; + } + const parsed = value as unknown as ApiWakeAdmission; + if (generation(parsed) !== parsed.generation) throw new Error("API wake receipt generation changed"); + return { version: 1, stackId: parsed.stackId, ownerId: parsed.ownerId, capabilityGeneration: parsed.capabilityGeneration, + challenge: parsed.challenge, generation: parsed.generation, + keys: parsed.keys.map(key => ({ fingerprint: key.fingerprint, expiresAt: key.expiresAt })) }; +} +export function createApiWakeAdmission(input: Omit): ApiWakeAdmission { + const canonical = { version: 1 as const, ...input, keys: [...input.keys].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)) }; + return parseApiWakeAdmission({ ...canonical, generation: generation(canonical) }); +} +export function authorizeApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): ApiWakeClaim | undefined { + if (receipt.stackId !== input.stackId || input.token.length < 1 || input.token.length > 8192 || /\s/.test(input.token)) return; + const fingerprint = apiWakeFingerprint({ ...receipt, keyHash: createHash("sha256").update(input.token).digest("hex") }); + return receipt.keys.some(key => key.fingerprint === fingerprint && (key.expiresAt === null || Date.parse(key.expiresAt) > input.now)) + ? { generation: receipt.generation, fingerprint } : undefined; +} +export function matchesApiWakeAdmission(receipt: ApiWakeAdmission, input: { stackId: string; token: string; now: number }): boolean { + return authorizeApiWakeAdmission(receipt, input) !== undefined; +} diff --git a/server/src/services/idle-incoming-wake-capability.test.ts b/server/src/services/idle-incoming-wake-capability.test.ts new file mode 100644 index 00000000000..c3b9cd70da6 --- /dev/null +++ b/server/src/services/idle-incoming-wake-capability.test.ts @@ -0,0 +1,113 @@ +import { generateKeyPairSync, sign } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { parseIdleCapabilities } from "./idle-incoming-wake-capability.js"; +import { + verifyCloudControlAssertion, + resetCloudControlReplayFenceForTests, + CLOUD_CONTROL_AUDIENCE, + CLOUD_CONTROL_JWS_TYPE, + CLOUD_RUNTIME_IDENTITY_ISSUER, +} from "./cloud-runtime-identity.js"; +const incoming = { + version: 1 as const, + ownerId: "00000000-0000-4000-8000-000000000001", + generation: "a".repeat(64), + apiKeyProof: { version: 1 as const, challenge: "c".repeat(64) }, + routes: ["tenant-api-http-v2" as const], +}; +const invalid: unknown[] = [ + null, + {}, + { other: true }, + { deadlineCatalog: "other" }, + { incomingWake: { ...incoming, version: 2 } }, + { incomingWake: { ...incoming, routes: ["all"] } }, + { + incomingWake: { + ...incoming, + routes: ["tenant-api-http-v2", "tenant-api-http-v2"], + }, + }, + { incomingWake: { ...incoming, ownerId: "other" } }, + { incomingWake: { ...incoming, extra: true } }, + { + incomingWake: { + ...incoming, + githubBroker: { + instanceId: "instance", + bindingsFingerprint: "b".repeat(64), + }, + }, + }, + { incomingWake: { ...incoming, routes: ["github-broker-events-v1"] } }, +]; + +describe("signed idle capabilities", () => { + it.each(invalid)("rejects unknown/partial capabilities: %j", (value) => { + expect(() => parseIdleCapabilities(value)).toThrow(); + }); + it("detaches and freezes the exact known catalogs", () => { + const source = { + incomingWake: { ...incoming, routes: [...incoming.routes] }, + deadlineCatalog: "domain_deadlines_v1", + }; + const parsed = parseIdleCapabilities(source); + source.incomingWake.routes.length = 0; + expect(parsed.incomingWake?.routes).toEqual(["tenant-api-http-v2"]); + expect(Object.isFrozen(parsed.incomingWake?.routes)).toBe(true); + }); + it("binds capability to signed stack, action and replay fence", () => { + resetCloudControlReplayFenceForTests(); + const pair = generateKeyPairSync("ed25519"); + const now = new Date("2099-01-01T00:00:00Z"); + const jwk = { + ...pair.publicKey.export({ format: "jwk" }), + kid: "idle", + use: "sig", + alg: "EdDSA", + }; + const env = { + PAPERCLIP_CLOUD_STACK_ID: "stack", + PAPERCLIP_CLOUD_RUNTIME_IDENTITY_JWKS: JSON.stringify({ keys: [jwk] }), + }; + const encode = (v: unknown) => + Buffer.from(JSON.stringify(v)).toString("base64url"); + const token = (overrides: object = {}) => { + const header = encode({ + alg: "EdDSA", + typ: CLOUD_CONTROL_JWS_TYPE, + kid: "idle", + }); + const body = encode({ + v: 1, + iss: CLOUD_RUNTIME_IDENTITY_ISSUER, + aud: CLOUD_CONTROL_AUDIENCE, + sub: "stack", + action: "task-drain:read", + requestId: "same", + iat: now.getTime() / 1000, + exp: now.getTime() / 1000 + 60, + idleCapabilities: { + incomingWake: incoming, + deadlineCatalog: "domain_deadlines_v1", + }, + ...overrides, + }); + return `${header}.${body}.${sign(null, Buffer.from(`${header}.${body}`), pair.privateKey).toString("base64url")}`; + }; + const verify = (compactJws: string) => + verifyCloudControlAssertion({ + compactJws, + env, + now, + expectedAction: "task-drain:read", + }); + expect(() => verify(token({ sub: "foreign" }))).toThrow(); + expect(() => verify(token({ action: "task-drain:start" }))).toThrow(); + expect(() => + verify(token({ idleCapabilities: { deadlineCatalog: "unknown" } })), + ).toThrow(); + expect(verify(token()).idleCapabilities?.incomingWake).toEqual(incoming); + expect(() => verify(token())).toThrow(/already been used/); + }); +}); diff --git a/server/src/services/idle-incoming-wake-capability.ts b/server/src/services/idle-incoming-wake-capability.ts new file mode 100644 index 00000000000..48dfe19c654 --- /dev/null +++ b/server/src/services/idle-incoming-wake-capability.ts @@ -0,0 +1,93 @@ +/** Closed Cloud-owned ingress catalog. No credential material belongs in this proof. */ +export interface IdleIncomingWakeCapability { + version: 1; + ownerId: string; + generation: string; + routes: ("tenant-api-http-v2" | "github-broker-events-v1")[]; + apiKeyProof?: { version: 1; challenge: string }; + githubBroker?: { instanceId: string; bindingsFingerprint: string }; +} +export interface IdleCapabilities { + incomingWake?: IdleIncomingWakeCapability; + deadlineCatalog?: "domain_deadlines_v1"; +} +const object = (v: unknown): v is Record => + Boolean(v && typeof v === "object" && !Array.isArray(v)); +const exact = (v: Record, keys: string[]) => + Object.keys(v).every((k) => keys.includes(k)); +const digest = (v: unknown): v is string => + typeof v === "string" && /^[a-f0-9]{64}$/.test(v); +export function parseIdleCapabilities(value: unknown): IdleCapabilities { + if ( + !object(value) || + !exact(value, ["incomingWake", "deadlineCatalog"]) || + !Object.keys(value).length + ) + throw new Error("Invalid idle capabilities"); + if ( + value.deadlineCatalog !== undefined && + value.deadlineCatalog !== "domain_deadlines_v1" + ) + throw new Error("Unknown idle deadline catalog"); + if (value.incomingWake !== undefined) { + const v = value.incomingWake; + if ( + !object(v) || + !exact(v, [ + "version", + "ownerId", + "generation", + "routes", + "githubBroker", + "apiKeyProof", + ]) || + v.version !== 1 || + typeof v.ownerId !== "string" || + !/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test( + v.ownerId, + ) || + !digest(v.generation) || + !Array.isArray(v.routes) || + v.routes.length < 1 || + v.routes.length > 2 || + v.routes.some( + (r) => !["tenant-api-http-v2", "github-broker-events-v1"].includes(r), + ) || + new Set(v.routes).size !== v.routes.length || + [...v.routes].sort().join() !== v.routes.join() + ) + throw new Error("Invalid incoming wake capability"); + if (v.routes.includes("tenant-api-http-v2")) { + const proof = v.apiKeyProof; + if (!object(proof) || Object.keys(proof).length !== 2 || !exact(proof, ["version", "challenge"]) + || proof.version !== 1 || !digest(proof.challenge)) throw new Error("Invalid API wake challenge"); + } else if (v.apiKeyProof !== undefined) throw new Error("Unexpected API wake challenge"); + if (v.routes.includes("github-broker-events-v1")) { + const b = v.githubBroker; + if ( + !object(b) || + !exact(b, ["instanceId", "bindingsFingerprint"]) || + typeof b.instanceId !== "string" || + !/^[A-Za-z0-9_-]{1,200}$/.test(b.instanceId) || + !digest(b.bindingsFingerprint) + ) + throw new Error("Invalid GitHub wake binding"); + } else if (v.githubBroker !== undefined) + throw new Error("Unexpected GitHub wake binding"); + } + // Return a detached immutable snapshot; request callers cannot mutate a verified claim. + const parsed = JSON.parse(JSON.stringify(value)) as IdleCapabilities; + if (parsed.incomingWake) { + Object.freeze(parsed.incomingWake.routes); + if (parsed.incomingWake.apiKeyProof) Object.freeze(parsed.incomingWake.apiKeyProof); + if (parsed.incomingWake.githubBroker) + Object.freeze(parsed.incomingWake.githubBroker); + Object.freeze(parsed.incomingWake); + } + return Object.freeze(parsed); +} + +declare const verified: unique symbol; +export type VerifiedIdleCapabilities = IdleCapabilities & { + readonly [verified]: true; +}; diff --git a/server/src/services/idle-incoming-wake.test.ts b/server/src/services/idle-incoming-wake.test.ts new file mode 100644 index 00000000000..d432a3083fe --- /dev/null +++ b/server/src/services/idle-incoming-wake.test.ts @@ -0,0 +1,184 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Db } from "@paperclipai/db"; +import { + readIdleIncomingWakeProof, + incomingWakeFingerprint, +} from "./idle-incoming-wake.js"; +import { registerConnectionEventWake } from "./connection-event-wake.js"; +import type { VerifiedIdleCapabilities } from "./idle-incoming-wake-capability.js"; + +const ownerId = "00000000-0000-4000-8000-000000000001"; +function fixture(count: number) { + const endpoints = Array.from({ length: count }, (_, i) => ({ + endpoint: { + id: `endpoint-${i}`, + companyId: "company", + status: "active", + activatedAt: new Date(0), + botExternalId: `bot-${i}`, + providerAccountId: `app-${i}`, + setup: { + step: "complete", + github: { + cloudRegistrationId: String(i).padStart(43, "a"), + registrationStatus: "completed", + }, + }, + }, + connection: { id: `connection-${i}` }, + })); + const registrations = endpoints.map(({ endpoint }) => ({ + endpointId: endpoint.id, + companyId: endpoint.companyId, + userId: "owner", + cloudId: endpoint.setup.github.cloudRegistrationId, + })); + const tuples = registrations.map((row) => [ + `github-app:${row.cloudId}`, + row.companyId, + row.userId, + "github.bot", + `gateway:${row.cloudId}`, + row.cloudId, + row.cloudId, + ]); + const capability = { + incomingWake: { + version: 1, + ownerId, + generation: "a".repeat(64), + routes: ["github-broker-events-v1"], + githubBroker: { + instanceId: "fixture-instance", + bindingsFingerprint: incomingWakeFingerprint(tuples), + }, + }, + } as VerifiedIdleCapabilities; + return { endpoints, registrations, capability }; +} +/** Named DB boundary for cardinality/elapsed guards; PostgreSQL fixture covers real SQL separately. */ +function database(replies: Array<() => unknown[]>) { + let index = 0; + const select = vi.fn(() => { + const reply = replies[index++]; + const query = { + from: () => query, + innerJoin: () => query, + where: () => query, + limit: async () => reply?.() ?? [], + }; + return query; + }); + const execute = vi.fn(async () => []); + return { + db: { select, execute } as unknown as Pick, + select, + execute, + }; +} +const stops: Array<() => void> = []; +function input( + db: object, + capabilities: VerifiedIdleCapabilities, + now?: () => number, +) { + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_INSTANCE_ID", "fixture-instance"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SIGN_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_SEAL_PRIVATE_KEY", "fixture"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_ENVIRONMENT", "test"); + vi.stubEnv("PAPERCLIP_CLOUD_CONNECTOR_BASE_URL", "http://127.0.0.1"); + stops.push(registerConnectionEventWake(db, () => {})); + return { ownerId, receiverOwner: db, capabilities, now }; +} +afterEach(() => { + for (const stop of stops.splice(0)) stop(); + vi.unstubAllEnvs(); +}); + +describe("bounded incoming wake owner projection", () => { + it("uses three bounded selects for a thousand bots and retains exact per-bot authority", async () => { + const f = fixture(1000), + sql = database([() => [], () => f.endpoints, () => f.registrations]); + const result = await readIdleIncomingWakeProof( + sql.db, + input(sql.db, f.capability), + ); + expect(sql.select).toHaveBeenCalledTimes(3); + expect(sql.execute).toHaveBeenCalledTimes(4); + expect(result?.endpointIds).toHaveLength(1000); + expect(result?.connectionIds).toHaveLength(1000); + }); + it.each(["duplicate", "foreign", "missing"])( + "rejects %s registration authority", + async (fault) => { + const f = fixture(1); + let registrations = f.registrations; + if (fault === "duplicate") + registrations = [...registrations, ...registrations]; + if (fault === "foreign") + registrations = registrations.map((r) => ({ + ...r, + companyId: "foreign", + })); + if (fault === "missing") registrations = []; + const sql = database([() => [], () => f.endpoints, () => registrations]); + await expect( + readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)), + ).rejects.toThrow("registration authority"); + }, + ); + it("stops the complete inspection when its monotonic budget expires before another query", async () => { + const f = fixture(1); + let now = 0; + const sql = database([ + () => { + now = 2001; + return []; + }, + () => f.endpoints, + () => f.registrations, + ]); + await expect( + readIdleIncomingWakeProof( + sql.db, + input(sql.db, f.capability, () => now), + ), + ).rejects.toThrow("budget exhausted"); + expect(sql.select).toHaveBeenCalledTimes(1); + expect(sql.execute).toHaveBeenCalledTimes(1); + }); + it("fails closed at row limits before expanding work or issuing registration queries", async () => { + const f = fixture(1001), + sql = database([() => [], () => f.endpoints, () => f.registrations]); + await expect( + readIdleIncomingWakeProof(sql.db, input(sql.db, f.capability)), + ).rejects.toThrow("Too many GitHub endpoints"); + expect(sql.select).toHaveBeenCalledTimes(2); + }); +}); + +describe("bounded API admission projection", () => { + const api = { incomingWake: { version: 1, ownerId, generation: "a".repeat(64), routes: ["tenant-api-http-v2"], + apiKeyProof: { version: 1, challenge: "c".repeat(64) } } } as VerifiedIdleCapabilities; + it.each(["overflow", "duplicate", "malformed"])("refuses complete eligibility for %s key state", async kind => { + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack"); + const row = { keyHash: "a".repeat(64), expiresAt: null }; + const rows = kind === "overflow" ? Array.from({ length: 1001 }, () => row) + : kind === "duplicate" ? [row, row] : [{ keyHash: "not-a-digest", expiresAt: null }]; + const execute = vi.fn().mockResolvedValueOnce([]).mockResolvedValueOnce(rows); + await expect(readIdleIncomingWakeProof({ execute } as unknown as Pick, + { ownerId, receiverOwner: {}, capabilities: api })).rejects.toThrow(); + expect(execute).toHaveBeenCalledTimes(2); + }); + it("has an explicit complete empty-key receipt and shares the inspection time budget", async () => { + vi.stubEnv("PAPERCLIP_CLOUD_STACK_ID", "fixture-stack"); + const execute = vi.fn(async () => []); + const proof = await readIdleIncomingWakeProof({ execute } as unknown as Pick, + { ownerId, receiverOwner: {}, capabilities: api }); + expect(proof?.apiWakeAdmission?.keys).toEqual([]); + let clock = 0; + const slow = vi.fn(async () => { clock += 1100; return []; }); + await expect(readIdleIncomingWakeProof({ execute: slow } as unknown as Pick, + { ownerId, receiverOwner: {}, capabilities: api, now: () => clock })).rejects.toThrow(/budget exhausted/); + }); +}); diff --git a/server/src/services/idle-incoming-wake.ts b/server/src/services/idle-incoming-wake.ts new file mode 100644 index 00000000000..f0aa26e1b13 --- /dev/null +++ b/server/src/services/idle-incoming-wake.ts @@ -0,0 +1,359 @@ +import { apiWakeFingerprint, createApiWakeAdmission, type ApiWakeAdmission } from "./idle-api-wake-admission.js"; +import { createHash } from "node:crypto"; +import { and, eq, inArray, sql, type SQL } from "drizzle-orm"; +import { + chatEndpoints, + chatGitHubRegistrations, + connectionGrants, + toolConnections, + type Db, +} from "@paperclipai/db"; +import type { + IdleIncomingWakeCapability, + VerifiedIdleCapabilities, +} from "./idle-incoming-wake-capability.js"; +import { githubConnectionEventBindings } from "./github-connection-events.js"; +import { paperclipCloudConnectorConfigFromEnv } from "./paperclip-cloud-connector.js"; +import { hasConnectionEventWakeReceiver } from "./connection-event-wake.js"; + +export interface IdleIncomingWakeProof { + capability: IdleIncomingWakeCapability; + apiWakeAdmission?: ApiWakeAdmission; + connectionIds: string[]; + endpointIds: string[]; + /** A broker enrollment cannot discharge these credential continuity obligations. */ + requiredContinuityConnectionIds: string[]; +} +export const incomingWakeFingerprint = (bindings: string[][]) => + createHash("sha256") + .update( + JSON.stringify( + bindings + .map((b) => [...b]) + .sort((a, b) => + JSON.stringify(a) < JSON.stringify(b) + ? -1 + : JSON.stringify(a) > JSON.stringify(b) + ? 1 + : 0, + ), + ), + ) + .digest("hex"); + +/** Read current binding authority in the safety transaction. No network call, renewal or write. */ +export async function readIdleIncomingWakeProof( + tx: Pick, + input: { + capabilities?: VerifiedIdleCapabilities; + ownerId: string; + receiverOwner: object; + now?: () => number; + }, +): Promise { + const capability = input.capabilities?.incomingWake; + if (!capability) return; + if (capability.ownerId !== input.ownerId) + throw new Error("Incoming wake owner changed"); + const proof: IdleIncomingWakeProof = { + capability, + connectionIds: [], + endpointIds: [], + requiredContinuityConnectionIds: [], + }; + const monotonic = input.now ?? (() => performance.now()); + const started = monotonic(); + const remaining = () => { + const ms = 2_000 - (monotonic() - started); + if (!Number.isFinite(ms) || ms <= 0) + throw new Error("Incoming wake inspection budget exhausted"); + return Math.max(1, Math.floor(ms)); + }; + const beforeRead = async () => { + // One shared owner budget, not five seconds multiplied by the row count. + await tx.execute( + sql`SELECT set_config('statement_timeout', ${`${remaining()}ms`}, true)`, + ); + remaining(); + }; + if (capability.routes.includes("tenant-api-http-v2")) { + const stackId = process.env.PAPERCLIP_CLOUD_STACK_ID; + if (!stackId || !capability.apiKeyProof) throw new Error("API wake challenge is missing"); + await beforeRead(); + const rows = await tx.execute<{ keyHash: string; expiresAt: Date | null }>(sql` + SELECT key_hash AS "keyHash", NULL::timestamptz AS "expiresAt" FROM agent_api_keys WHERE revoked_at IS NULL + UNION ALL SELECT key_hash AS "keyHash", expires_at AS "expiresAt" FROM board_api_keys + WHERE revoked_at IS NULL AND (expires_at IS NULL OR expires_at > now()) LIMIT 1001`); + remaining(); + if (rows.length > 1000) throw new Error("Too many API keys to certify wake admission"); + proof.apiWakeAdmission = createApiWakeAdmission({ stackId, ownerId: input.ownerId, + capabilityGeneration: capability.generation, challenge: capability.apiKeyProof.challenge, + keys: rows.map(row => ({ fingerprint: apiWakeFingerprint({ stackId, ownerId: input.ownerId, + challenge: capability.apiKeyProof!.challenge, keyHash: row.keyHash }), + expiresAt: row.expiresAt === null ? null : new Date(row.expiresAt).toISOString() })), + }); + } + if (!capability.routes.includes("github-broker-events-v1")) { + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + remaining(); + return proof; + } + const broker = capability.githubBroker; + const config = paperclipCloudConnectorConfigFromEnv(); + if ( + !broker || + config?.instanceId !== broker.instanceId || + !hasConnectionEventWakeReceiver(input.receiverOwner) + ) + throw new Error("Incoming GitHub receiver is not bound"); + await beforeRead(); + const rows = await tx + .select({ grant: connectionGrants, connection: toolConnections }) + .from(connectionGrants) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, connectionGrants.connectionId), + eq(toolConnections.companyId, connectionGrants.companyId), + ), + ) + .where( + and( + eq(connectionGrants.status, "active"), + eq(toolConnections.status, "active"), + eq(toolConnections.enabled, true), + sql`${toolConnections.config}->>'sourceTemplateKey' = 'github'`, + sql`${toolConnections.config}->'oauth'->>'connectorProfile' = 'github.code'`, + sql`exists (select 1 from companies where id=${connectionGrants.companyId} and status <> 'archived')`, + ), + ) + .limit(1001); + if (rows.length > 1000) + throw new Error("Too many GitHub bindings to inspect"); + remaining(); + let bindingCount = 0; + for (const { grant, connection } of rows) { + const github = grant.providerTenant?.github; + if ( + !github || + !Array.isArray(github.installationIds) || + !github.installationIds.length || + github.installationIds.length > 1000 || + github.bindingSync || + github.continuityRetry || + github.installationIds.some( + (id) => typeof id !== "string" || !/^[0-9]+$/.test(id), + ) + ) + throw new Error("Unsettled GitHub authority"); + bindingCount += github.installationIds.length; + if (bindingCount > 1000) + throw new Error("Too many GitHub bindings to inspect"); + proof.requiredContinuityConnectionIds.push(connection.id); + } + const bindings = githubConnectionEventBindings(rows); + if ( + bindings.length !== + rows.reduce( + (n, r) => n + r.grant.providerTenant!.github!.installationIds.length, + 0, + ) + ) + throw new Error("Incomplete GitHub binding"); + const tuples = bindings.map((b) => [ + b.id, + b.companyId, + b.subject, + "github.code", + b.installationId, + b.connectionId, + b.grantId, + ]); + await beforeRead(); + const endpoints = await tx + .select({ endpoint: chatEndpoints, connection: toolConnections }) + .from(chatEndpoints) + .innerJoin( + toolConnections, + and( + eq(toolConnections.id, chatEndpoints.connectionId), + eq(toolConnections.companyId, chatEndpoints.companyId), + ), + ) + .where( + and( + eq(chatEndpoints.provider, "github"), + eq(chatEndpoints.status, "active"), + eq(toolConnections.status, "active"), + eq(toolConnections.enabled, true), + eq(toolConnections.transport, "chat_sdk"), + eq(toolConnections.connectionPurpose, "channel"), + sql`exists (select 1 from companies where id=${chatEndpoints.companyId} and status <> 'archived')`, + ), + ) + .limit(1001); + if (endpoints.length > 1000) + throw new Error("Too many GitHub endpoints to inspect"); + remaining(); + const brokerEndpointIds = endpoints + .filter( + ({ endpoint }) => + typeof endpoint.setup.github?.cloudRegistrationId === "string", + ) + .map(({ endpoint }) => endpoint.id); + const registrationUsers = new Map(); + if (brokerEndpointIds.length) { + await beforeRead(); + const registrations = await tx + .select({ + endpointId: chatGitHubRegistrations.endpointId, + companyId: chatGitHubRegistrations.companyId, + userId: chatGitHubRegistrations.userId, + cloudId: sql`${chatGitHubRegistrations.handoff}->>'cloudId'`, + }) + .from(chatGitHubRegistrations) + .where( + and( + inArray(chatGitHubRegistrations.endpointId, brokerEndpointIds), + eq(chatGitHubRegistrations.status, "completed"), + ), + ) + .limit(10001); + remaining(); + if (registrations.length > 10000) + throw new Error("Too many GitHub registrations to inspect"); + for (const registration of registrations) { + const key = JSON.stringify([ + registration.endpointId, + registration.companyId, + registration.cloudId, + ]); + const users = registrationUsers.get(key) ?? []; + users.push(registration.userId); + registrationUsers.set(key, users); + } + } + for (const { endpoint, connection } of endpoints) { + const registration = endpoint.setup.github?.cloudRegistrationId; + if (!registration || !/^[A-Za-z0-9_-]{43}$/.test(registration)) continue; + if ( + endpoint.setup.step !== "complete" || + endpoint.setup.github?.registrationStatus !== "completed" || + endpoint.setup.github.initialSetupPending || + endpoint.setup.github.initialRepositoryImportPending || + !endpoint.activatedAt || + !endpoint.botExternalId || + !endpoint.providerAccountId + ) + throw new Error("GitHub bot setup has unfinished work"); + const users = registrationUsers.get( + JSON.stringify([endpoint.id, endpoint.companyId, registration]), + ); + if (users?.length !== 1 || !users[0]) + throw new Error("GitHub registration authority changed"); + tuples.push([ + `github-app:${registration}`, + endpoint.companyId, + users[0], + "github.bot", + `gateway:${registration}`, + registration, + registration, + ]); + proof.connectionIds.push(connection.id); + proof.endpointIds.push(endpoint.id); + } + remaining(); + if (tuples.length > 1000) + throw new Error("Too many GitHub bindings to inspect"); + if (incomingWakeFingerprint(tuples) !== broker.bindingsFingerprint) + throw new Error("GitHub wake bindings changed"); + remaining(); + await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + remaining(); + return proof; +} + +/** Exact domain predicates selected by stable inventory reason, never SQL-text identity. */ +export function incomingWakeWorkQuery( + reason: string, + proof?: IdleIncomingWakeProof, + continuityConnectionIds: readonly string[] = [], +): SQL | undefined { + const connectionIds = [ + ...(proof?.connectionIds ?? []), + ...(proof?.requiredContinuityConnectionIds.filter((id) => + continuityConnectionIds.includes(id), + ) ?? []), + ]; + if (reason === "agent_api_keys") + return proof?.apiWakeAdmission !== undefined + ? sql`SELECT 1 WHERE false` + : sql`SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`; + if (reason === "board_api_keys") + return proof?.apiWakeAdmission !== undefined + ? sql`SELECT 1 WHERE false` + : sql`SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL + AND (expires_at IS NULL OR expires_at > now())`; + if (reason === "chat_endpoints") { + const ids = proof?.endpointIds ?? []; + // Only the supported GitHub webhook path gains a removed-history proof. + // Archival commits before cleanup, so status alone cannot discharge work. + return sql`SELECT 1 FROM chat_endpoints endpoint WHERE (((endpoint.provider = 'github' + AND endpoint.status = 'archived' AND endpoint.archived_at IS NOT NULL + AND jsonb_typeof(endpoint.setup) = 'object' + AND CASE WHEN jsonb_typeof(endpoint.setup->'runtimeGeneration') = 'number' + THEN (endpoint.setup->>'runtimeGeneration')::numeric >= 1 + AND trunc((endpoint.setup->>'runtimeGeneration')::numeric) = (endpoint.setup->>'runtimeGeneration')::numeric + ELSE false END + AND EXISTS (SELECT 1 FROM tool_connections removed_connection + WHERE removed_connection.id = endpoint.connection_id AND removed_connection.company_id = endpoint.company_id + AND removed_connection.connection_purpose = 'channel' AND removed_connection.transport = 'chat_sdk' + AND removed_connection.status = 'archived' AND removed_connection.enabled = false + AND removed_connection.credential_secret_refs = '[]'::jsonb) + AND NOT EXISTS (SELECT 1 FROM chat_endpoint_leases lease WHERE lease.endpoint_id = endpoint.id) + AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id) + AND NOT EXISTS (SELECT 1 FROM chat_github_registrations registration WHERE registration.endpoint_id = endpoint.id + AND (registration.company_id = endpoint.company_id AND registration.status = 'completed' + AND registration.consumed_at IS NOT NULL) IS NOT TRUE)) + OR (endpoint.provider = 'slack' + AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup' + AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb + AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb + OR jsonb_typeof(endpoint.setup->'command') = 'string') + AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL + AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL + AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL + AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL + AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id) + AND EXISTS (SELECT 1 FROM tool_connections draft_connection + WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id + AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk' + AND draft_connection.status = 'draft' AND draft_connection.enabled = false))) IS NOT TRUE) AND ${ + ids.length + ? sql`endpoint.id NOT IN (${sql.join( + ids.map((id) => sql`${id}::uuid`), + sql`, `, + )})` + : sql`true` + }`; + } + if (reason === "tool_connections") { + const ids = connectionIds; + return sql`SELECT 1 FROM tool_connections c WHERE (c.status <> 'archived' AND ( + (c.status = 'draft' AND c.enabled = false) + OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth') + OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote' + AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id + AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL))) + ) IS NOT TRUE) AND ${ + ids.length + ? sql`c.id NOT IN (${sql.join( + ids.map((id) => sql`${id}::uuid`), + sql`, `, + )})` + : sql`true` + }`; + } + return undefined; +} diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 6034d6515d8..70b8cd6a921 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -1,3 +1,4 @@ +import { idleApiActivity } from "./idle-api-activity.js"; import { promises as fs } from "node:fs"; import path from "node:path"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; @@ -83,7 +84,7 @@ export async function readIdleLocalWork(options: { backupCheckpoint?: boolean; d } let probes: IdleLocalWork; try { - const local = [...localWorkProbes].map(probe => probe()); + const local = [idleApiActivity.active() ? "present" : "none", ...[...localWorkProbes].map(probe => probe())]; probes = local.some(state => state !== "none" && state !== "present") ? "unknown" : local.includes("present") ? "present" : "none"; } catch { probes = "unknown"; } diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 0773269fb21..626d18a0ab8 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -1,3 +1,6 @@ +import type { ApiWakeAdmission } from "./idle-api-wake-admission.js"; +import { readIdleIncomingWakeProof, incomingWakeWorkQuery, type IdleIncomingWakeProof } from "./idle-incoming-wake.js"; +import type { VerifiedIdleCapabilities, IdleIncomingWakeCapability } from "./idle-incoming-wake-capability.js"; import { idleWorkSnapshot } from "./task-admission.js"; import { readIdleLocalWork, type IdleLocalWork } from "./idle-local-work.js"; import { readIdleRoutineSchedule, type ScheduledWake } from "./idle-scheduled-work.js"; @@ -38,10 +41,12 @@ export interface IdleSleepDrainStatus { } export interface IdleSleepSafety { - version: 1 | 2; + version: 1 | 2 | 3; backgroundWork: "none" | "present" | "unknown"; scheduledWake?: ScheduledWake; diagnostics?: IdleSleepDiagnostics; + incomingWake?: IdleIncomingWakeCapability; + apiWakeAdmission?: ApiWakeAdmission; } export type InspectIdlePlugins = (hold: { ownerId: string; expiresAt: number }) => Promise<{ @@ -85,9 +90,8 @@ const WORK_CHECKS = [ workCheck("prompt_injection_invocations", `SELECT 1 FROM prompt_injection_invocations WHERE status = 'running'`), // Persistent machine credentials can receive work without a human opening // the instance. They need an ingress wake contract before we can sleep. - workCheck("agent_api_keys", `SELECT 1 FROM agent_api_keys WHERE revoked_at IS NULL`), - workCheck("board_api_keys", `SELECT 1 FROM board_api_keys WHERE revoked_at IS NULL - AND (expires_at IS NULL OR expires_at > now())`), + workCheck("agent_api_keys", incomingWakeWorkQuery("agent_api_keys")!), + workCheck("board_api_keys", incomingWakeWorkQuery("board_api_keys")!), // Coalescing consumes a request into another run. Retain it as work until // both the request and that exact company/agent's run have finished. workCheck("agent_wakeup_requests", `SELECT 1 FROM agent_wakeup_requests w WHERE @@ -138,20 +142,7 @@ const WORK_CHECKS = [ // An unconfigured Slack draft has no provider identity or inbound runtime. // Setup still wakes through normal browser admission. Retain every enabled, // previously active, partially connected, foreign, or unsupported endpoint. - workCheck("chat_endpoints", `SELECT 1 FROM chat_endpoints endpoint WHERE (endpoint.provider = 'slack' - AND endpoint.status = 'draft' AND endpoint.setup->>'step' = 'provider_setup' - AND (endpoint.setup - ARRAY['step', 'command']) = '{}'::jsonb - AND (endpoint.setup->'command' IS NULL OR endpoint.setup->'command' = 'null'::jsonb - OR jsonb_typeof(endpoint.setup->'command') = 'string') - AND endpoint.activated_at IS NULL AND endpoint.provider_account_id IS NULL - AND endpoint.bot_external_id IS NULL AND endpoint.bot_username IS NULL - AND endpoint.last_event_at IS NULL AND endpoint.last_publication_at IS NULL - AND endpoint.health_message IS NULL AND endpoint.last_error IS NULL - AND NOT EXISTS (SELECT 1 FROM chat_slack_registrations registration WHERE registration.endpoint_id = endpoint.id) - AND EXISTS (SELECT 1 FROM tool_connections draft_connection - WHERE draft_connection.id = endpoint.connection_id AND draft_connection.company_id = endpoint.company_id - AND draft_connection.connection_purpose = 'channel' AND draft_connection.transport = 'chat_sdk' - AND draft_connection.status = 'draft' AND draft_connection.enabled = false)) IS NOT TRUE`), + workCheck("chat_endpoints", incomingWakeWorkQuery("chat_endpoints")!), workCheck("routines", `SELECT 1 FROM routines WHERE status NOT IN ('paused', 'archived')`), workCheck("routine_runs", `SELECT 1 FROM routine_runs WHERE status NOT IN ('completed', 'failed', 'skipped', 'cancelled')`), workCheck("plugin_jobs", `SELECT 1 FROM plugin_jobs WHERE status <> 'paused'`), @@ -160,13 +151,7 @@ const WORK_CHECKS = [ // Saved outbound MCP configuration and disabled drafts have no independent // listener. Keep local processes, channels and GitHub webhook bindings awake; // accepted calls, credential recovery and delivery work have separate fences. - workCheck("tool_connections", `SELECT 1 FROM tool_connections c WHERE c.status <> 'archived' AND ( - (c.status = 'draft' AND c.enabled = false) - OR (c.status = 'active' AND c.connection_purpose = 'ai' AND c.transport = 'runtime_auth') - OR (c.status = 'active' AND c.connection_purpose = 'tool' AND c.transport = 'mcp_remote' - AND NOT EXISTS (SELECT 1 FROM connection_grants g WHERE g.connection_id = c.id - AND (g.company_id <> c.company_id OR g.provider_tenant->'github' IS NOT NULL))) - ) IS NOT TRUE`), + workCheck("tool_connections", incomingWakeWorkQuery("tool_connections")!), // An explicit board escalation has no automatic retry. Retain the visible // action and all execution/cleanup checks while the board decides what to do. workCheck("issue_recovery_actions", `SELECT 1 FROM issue_recovery_actions a WHERE a.status NOT IN ('resolved', 'cancelled') @@ -295,6 +280,8 @@ function sameQuietHold(before: IdleSleepDrainStatus, after: IdleSleepDrainStatus * Only a caller holding the exact idle owner may receive an empty report. */ export interface IdleSleepSafetyOptions { + incomingWake?: VerifiedIdleCapabilities; + deadlineCapability?: VerifiedIdleCapabilities; collectDiagnostics?: true; } @@ -311,12 +298,15 @@ export async function readIdleSleepSafety( scheduledIssueMonitors = false, options: IdleSleepSafetyOptions = {}, ): Promise { + let incomingProof: IdleIncomingWakeProof | undefined; const diagnostic = options.collectDiagnostics ? createIdleSleepDiagnostics() : undefined; const report = (backgroundWork: IdleLocalWork, scheduledWake?: ScheduledWake): IdleSleepSafety => { const diagnostics = diagnostic?.report(); if (backgroundWork === "none" && diagnostics?.checks.some(check => check.state === "unknown" || check.state === "not_checked")) backgroundWork = "unknown"; - return { version: backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork, + return { version: backgroundWork === "none" && incomingProof ? 3 : backgroundWork === "none" && scheduledWake ? 2 : 1, backgroundWork, + ...(backgroundWork === "none" && incomingProof ? { incomingWake: incomingProof.capability, + ...(incomingProof.apiWakeAdmission ? { apiWakeAdmission: incomingProof.apiWakeAdmission } : {}) } : {}), ...(backgroundWork === "none" && scheduledWake ? { scheduledWake } : {}), ...(diagnostics ? { diagnostics } : {}) }; }; @@ -361,9 +351,14 @@ export async function readIdleSleepSafety( let scheduledWake: ScheduledWake | undefined; const blocked = await db.transaction(async (tx) => { await tx.execute(sql`SET LOCAL statement_timeout = '5s'`); + if (options.incomingWake?.incomingWake) { + if (!ownerId || ownerId !== before.ownerId) return undefined; + incomingProof = await readIdleIncomingWakeProof(tx, { capabilities: options.incomingWake, ownerId, receiverOwner: db }); + } const checks = WORK_CHECKS.filter(check => (!scheduledRoutines || check.reason !== "routines") - && (!scheduledIssueMonitors || check.reason !== "issues")); + && (!scheduledIssueMonitors || check.reason !== "issues")) + .map(check => ({ ...check, query: incomingWakeWorkQuery(check.reason, incomingProof) ?? check.query })); checks.push(workCheck("agents.company_hold", sql`SELECT 1 FROM agents JOIN companies ON companies.id = agents.company_id WHERE ${companyHoldMismatch()}`)); checks.push(workCheck("heartbeat_runs.budget_cancellation", sql`SELECT 1 FROM heartbeat_runs WHERE ${pendingBudgetRunCancellationPredicate()}`)); @@ -448,6 +443,9 @@ export async function readIdleSleepSafety( diagnostic?.record("inspection.hold", "unknown"); return report("present"); } + // Legacy alarm kinds cannot be silently combined with a new ingress proof. + // The negotiated deadline catalog supplies the composed v3 alarm separately. + if (incomingProof && scheduledWake) return report("unknown"); return report(local, scheduledWake); } catch { // Missing migrations, unknown state, malformed configuration, timeouts and diff --git a/server/src/services/isolated-task-directory-retention.ts b/server/src/services/isolated-task-directory-retention.ts index cb5a8dc8a37..b4c83c70e66 100644 --- a/server/src/services/isolated-task-directory-retention.ts +++ b/server/src/services/isolated-task-directory-retention.ts @@ -124,7 +124,7 @@ function restorePointObjectKey(companyId: string, workspaceId: string, generatio return `${companyId}/${NAMESPACE}/${workspaceId}/${generation}/task-directory/${id}.pcrp`; } -function validateRestorePointRef(ref: TaskDirectoryRestorePointRef | null | undefined, workspace: { companyId: string; id: string }) { +export function validateTaskDirectoryRestorePointRef(ref: TaskDirectoryRestorePointRef | null | undefined, workspace: { companyId: string; id: string }) { if (!ref || ref.schema !== TASK_DIRECTORY_RESTORE_POINT_SCHEMA || typeof ref.id !== "string" || !UUID.test(ref.id) || !Number.isSafeInteger(ref.generation) || ref.generation < 1 || ref.objectKey !== restorePointObjectKey(workspace.companyId, workspace.id, ref.generation, ref.id) @@ -136,6 +136,25 @@ function validateRestorePointRef(ref: TaskDirectoryRestorePointRef | null | unde return ref; } +/** Read a retained directory without reopening its workspace or replacing its + * original path. Administrative migration uses the same identity and byte checks + * as ordinary restore, and never treats an expired object as an empty folder. */ +export async function loadTaskDirectoryRestorePoint(input: { + storage: StorageService; + workspace: { companyId: string; id: string }; + ref: TaskDirectoryRestorePointRef; + signal?: AbortSignal; +}) { + const ref = validateTaskDirectoryRestorePointRef(input.ref, input.workspace); + if (ref.expiredAt) throw new TaskDirectoryRestoreFailure("restore_point_expired"); + const ciphertext = await readRestorePointObject(input.storage, input.workspace.companyId, ref.objectKey, input.signal).catch(() => { + throw new TaskDirectoryRestoreFailure("restore_point_unavailable"); + }); + if (ciphertext.length !== ref.byteSize) throw new TaskDirectoryRestoreFailure("ciphertext_size_mismatch"); + return openRestorePoint({ ciphertext, sha256: ref.sha256, key: ref.key, + identity: { companyId: input.workspace.companyId, workspaceId: input.workspace.id, restorePointId: ref.id, generation: ref.generation } }); +} + function skipReason(error: unknown) { if (error instanceof TaskDirectorySnapshotSkip) return error.reason; if (error instanceof IsolatedRepositorySourceBusy) return "source_busy"; @@ -341,7 +360,7 @@ export async function restoreIsolatedTaskDirectory(input: { const parent = path.dirname(cwd); await fs.mkdir(parent, { recursive: true, mode: 0o700 }); if (await fs.realpath(parent) !== parent) throw new TaskDirectoryRestoreFailure("task_directory_identity_changed"); - const ref = validateRestorePointRef(state.restorePoint, input.workspace); + const ref = validateTaskDirectoryRestorePointRef(state.restorePoint, input.workspace); if (await directoryState(cwd) === "present") { if (!state.removed) return { restored: false, expired: false }; if ((await fs.readdir(cwd)).length === 0) await fs.rmdir(cwd); @@ -355,12 +374,7 @@ export async function restoreIsolatedTaskDirectory(input: { await fs.mkdir(cwd, { mode: 0o700 }); return { restored: false, expired: true }; } - const ciphertext = await readRestorePointObject(input.storage, input.workspace.companyId, ref.objectKey, input.signal).catch(() => { - throw new TaskDirectoryRestoreFailure("restore_point_unavailable"); - }); - if (ciphertext.length !== ref.byteSize) throw new TaskDirectoryRestoreFailure("ciphertext_size_mismatch"); - const payload = await openRestorePoint({ ciphertext, sha256: ref.sha256, key: ref.key, - identity: { companyId: input.workspace.companyId, workspaceId: input.workspace.id, restorePointId: ref.id, generation: ref.generation } }); + const payload = await loadTaskDirectoryRestorePoint({ ...input, ref }); const staging = await materializeTaskDirectorySnapshot({ payload, target: cwd, expectedDigest: ref.digest, signal: input.signal }); try { await fs.rename(staging, cwd); diff --git a/server/src/services/native-runtime/agent-chat-response-recovery.test.ts b/server/src/services/native-runtime/agent-chat-response-recovery.test.ts index 918653a5fd1..b0d4ce5481a 100644 --- a/server/src/services/native-runtime/agent-chat-response-recovery.test.ts +++ b/server/src/services/native-runtime/agent-chat-response-recovery.test.ts @@ -65,6 +65,26 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => const input = { companyId, issueId, runId }; return { ...input, input, agentId, userId, wakeId, prpEvent, reply }; } + it("settles the exact claimed wake when recovery projects a committed native run", async () => { + // The provider and result are already committed. Recovery owns terminal + // projection because the live executor will not return to completeRun. + const f = await fixture(false, { succeed: false }); + const sourceWakeId = randomUUID(); + await db.insert(agentWakeupRequests).values({ id: sourceWakeId, companyId: f.companyId, + agentId: f.agentId, source: "issue", reason: "issue_assigned", status: "claimed", + runId: f.runId, claimedAt: new Date(), payload: { issueId: f.issueId } }); + await db.update(heartbeatRuns).set({ wakeupRequestId: sourceWakeId }).where(eq(heartbeatRuns.id, f.runId)); + + await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + const [wake] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, sourceWakeId)); + expect(run).toMatchObject({ status: "succeeded", nativePhase: "committed" }); + expect(run.finishedAt).not.toBeNull(); + expect(wake).toMatchObject({ status: "completed", runId: f.runId }); + expect(wake.finishedAt).not.toBeNull(); + }); + const comments = (runId: string) => db.select().from(issueComments).where(eq(issueComments.createdByRunId, runId)); const issue = async (issueId: string) => (await db.select().from(issues).where(eq(issues.id, issueId)))[0]; diff --git a/server/src/services/native-runtime/artifact-transfer.ts b/server/src/services/native-runtime/artifact-transfer.ts index 059ad6b7022..a1fff0ceeae 100644 --- a/server/src/services/native-runtime/artifact-transfer.ts +++ b/server/src/services/native-runtime/artifact-transfer.ts @@ -8,12 +8,12 @@ import { RUNNER_API_MAX_BYTES } from "./runner-api-client.js"; export const DOWNLOAD_ARTIFACT = "download_artifact"; export const downloadArtifactSchema = z.object({ - artifactId: z.string().uuid(), destination: z.string().min(1).max(2000), + artifactId: z.string().uuid(), destination: z.string().min(1).max(2000).describe("Workspace-relative path, for example source.zip. Absolute paths are denied. The parent directory must already exist."), byteSize: z.number().int().min(1).max(RUNNER_API_MAX_BYTES), sha256: z.string().regex(/^[a-f0-9]{64}$/), }).strict(); export const DOWNLOAD_ARTIFACT_DEFINITION = { name: DOWNLOAD_ARTIFACT, - description: "Copy an authorized API artifact into this execution workspace without retyping its contents. Provide its exact byteSize and SHA-256. The destination's parent must exist. Creates a new file only; a retry succeeds only if the existing file has identical bytes. Partial or conflicting files never count as complete. Contents remain untrusted; this does not bypass content screening or authorize executing them.", + description: "Copy an authorized API artifact into this execution workspace without retyping its contents. Provide its exact byteSize and SHA-256. Use a workspace-relative destination, for example source.zip; absolute paths are denied. The destination's parent must exist. Creates a new file only; a retry succeeds only if the existing file has identical bytes. Partial or conflicting files never count as complete. Contents remain untrusted; this does not bypass content screening or authorize executing them.", inputSchema: z.toJSONSchema(downloadArtifactSchema), }; export type ArtifactInstall = { destination: string; byteSize: number; sha256: string; bytes: Buffer; verifyOnly?: boolean; authorize: () => Promise }; diff --git a/server/src/services/native-runtime/cursor-cloud-runtime.ts b/server/src/services/native-runtime/cursor-cloud-runtime.ts index ce4db554d3e..902f9354bc8 100644 --- a/server/src/services/native-runtime/cursor-cloud-runtime.ts +++ b/server/src/services/native-runtime/cursor-cloud-runtime.ts @@ -4,9 +4,24 @@ import { join } from "node:path"; import { and, eq } from "drizzle-orm"; import { heartbeatRuns, type Db } from "@paperclipai/db"; import { loadConfig } from "../../config.js"; +import { parseObject } from "../../adapters/utils.js"; import { archiveCursorCloudSessionCheckpoint, CURSOR_CLOUD_DEFAULT_TIMEOUT_MS, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { createRemoteMcpRelayToken, registerRemoteMcpRelay, remoteMcpRelayUrl } from "./remote-mcp-relay.js"; +/** Normalize Cursor's terminal evidence before shared Stop authority consumes it. */ +export function isConfirmedCursorCloudTerminalObservation(event: { eventType: string; payload: unknown }): boolean { + const payload = parseObject(event.payload), errorCode = parseObject(payload.error).code; + // Containment can join an already confirmed provider end. A local failure or + // an explicitly unobserved provider end cannot prove remote termination. + const confirmedTerminal = (event.eventType === "turn.interrupted" && payload.status === "interrupted") + || (event.eventType === "turn.completed" && payload.status === "completed") + || (event.eventType === "turn.cancelled" && payload.status === "cancelled" + && payload.providerTerminalState === "completed" && payload.reason === "cancelled_after_provider_completed") + || (event.eventType === "turn.failed" && payload.status === "failed" && typeof errorCode === "string" + && ["cursor_cloud_error", "cursor_cloud_expired"].includes(errorCode)); + return confirmedTerminal && (payload.providerTerminalObserved === undefined || payload.providerTerminalObserved === true); +} + export function resolveCursorCloudPublicOrigin(): string | undefined { return loadConfig().authPublicBaseUrl; } diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 2accfe2a6ca..c4315273e81 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -4,16 +4,17 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agentTaskSessions, agents, companies, createDb, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; -import { DurablePrpControlPlane, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; +import { agentTaskSessions, agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; +import { DurablePrpControlPlane, parseNativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js"; import { createIsolatedTestHome } from "../../__tests__/helpers/isolated-test-home.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { nativeRunnerCheckpointWriter } from "./native-runner-checkpoint.js"; -import { createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js"; +import { assertNativeRetainedSandboxBinding, createNativeControllerRecovery, deferNativeControllerRecovery, importNativeControllerRecovery, NativeControllerRecoveryWaitingError, NATIVE_RETAINED_OWNER_WAIT, resolveNativeControllerRecovery } from "./native-controller-recovery.js"; import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; import { claimNativeRestartRecoveries } from "./native-restart-recovery.js"; import { readNativeControllerRecoveryInventory } from "./native-controller-recovery-inventory.js"; +import { inspectRemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js"; describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable discovery and import", () => { let database: Awaited>; @@ -132,6 +133,91 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: nextDeadline })).toEqual([]); }); + it.each(["committed", "workspace_finalizing"])("defers a remote warm continuation while its %s predecessor has a finalization lease", async phase => { + const fixture = await seed(), runId = randomUUID(), environmentId = randomUUID(); + const previous = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, + binding: { companyId, agentId, issueId: fixture.scope.issueId, runId: fixture.scope.runId, + executionWorkspaceId: randomUUID() }, + task: { identifier: "NCD-1", title: "Continue retained execution", description: null, + prompt: "Continue the saved work.", workMode: "standard" }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: fixture.scope.normalizedSessionId, driverKind: "codex_app_server", + protocolVersion: 1, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } }, + completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Continue the saved work.", criteria: [{ id: "objective", requirement: "Save the result." }] } }, + credentialBindings: [], interactionResponses: [], + }); + const current = { ...previous, binding: { ...previous.binding, runId } }; + const checkpoint: PersistedNativeSession = { ...fixture.snapshot, driverKind: "codex_app_server", + providerSessionId: "retained-provider-session", activeTurnId: null, pendingRuntimeRequests: [] }; + const metadata = createNativeControllerRecovery(checkpoint, compatibility, { runId: fixture.scope.runId, + runnerInstanceId: fixture.native.runnerInstanceId, normalizedSessionId: fixture.scope.normalizedSessionId }); + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...fixture.profile, nativeExecutionInput: previous, + sessionCheckpoint: checkpoint, nativeControllerRecovery: metadata } }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: fixture.scope.issueId, + nativeSessionId: fixture.scope.normalizedSessionId, status: "running", runtimeMode: "native", + runnerProfileJson: { nativeExecutionInput: current, sessionCheckpoint: { ...checkpoint, + identity: { ...checkpoint.identity, runId } } } }); + await db.insert(nativeRunFinalizations).values({ companyId, issueId: fixture.scope.issueId, runId, phase: "observed" }); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, fixture.scope.issueId)); + await db.insert(environments).values({ id: environmentId, name: `Retained ${environmentId}`, driver: "sandbox" }); + await db.insert(environmentLeases).values({ companyId, environmentId, issueId: fixture.scope.issueId, + heartbeatRunId: fixture.scope.runId, provider: "daytona", providerLeaseId: "retained-sandbox", status: "released", + metadata: { remoteCwd: "/workspace" } }); + const deadline = new Date(Date.now() + 60_000); + await db.update(nativeRunFinalizations).set({ phase, leaseOwner: "finalizer", leaseExpiresAt: deadline, + controllerPid: process.pid, controllerProcessStartedAt: new Date() }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + const identity = { runId: fixture.scope.runId, normalizedSessionId: fixture.scope.normalizedSessionId, + runnerInstanceId: fixture.native.runnerInstanceId, environmentLeaseId: fixture.native.environmentLeaseId, + turnId: fixture.native.turnId, itemId: fixture.native.itemId }; + const target = { kind: "remote", transport: "sandbox", environmentId, leaseId: randomUUID(), providerKey: "daytona", + remoteCwd: "/workspace", sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" } } as const; + const readSandbox = (override: Record = {}) => assertNativeRetainedSandboxBinding(db, + { ...fixture.scope, target: { ...target, ...override } as never }); + const inspect = (overrides: Partial[0]> = {}) => inspectRemoteWarmSessionRecovery({ db, execution: current, + target: target as never, + identity, readControlState: () => ({ identity, commands: [{ type: "turn.start", status: "completed" }], + committedEvents: [{ eventType: "run.terminal", envelope: identity }] }), + matchesExecution: saved => saved.binding.runId === fixture.scope.runId, + loadCheckpoint: () => checkpoint, canReuseProcess: () => true, ...overrides, + }); + const denied = "runner_state_identity_mismatch: prior_owner_terminal_state_indeterminate"; + await expect(inspect({ execution: { ...current, binding: { ...current.binding, companyId: randomUUID() } } })) + .rejects.toThrow(denied); + await expect(inspect({ matchesExecution: () => false })).rejects.toThrow(denied); + await expect(readSandbox({ remoteCwd: "/unrelated" })).rejects.toThrow("retained_sandbox_binding"); + for (const leaseExpiresAt of [null, new Date(Date.now() - 1_000)]) { + await db.update(nativeRunFinalizations).set({ leaseExpiresAt }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + await expect(inspect()).rejects.toThrow(denied); + await expect(readSandbox()).rejects.toThrow("retained_sandbox_binding"); + } + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: deadline }).where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + await expect(inspect()).rejects.toThrow(denied); + await db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, fixture.scope.runId)); + await expect(readSandbox()).rejects.toMatchObject({ retryAt: deadline }); + const waiting = await inspect().catch(error => error); + expect(waiting).toBeInstanceOf(NativeControllerRecoveryWaitingError); + expect(waiting.retryAt).toEqual(deadline); + await deferNativeControllerRecovery(db, { ...current.binding, retryAt: waiting.retryAt }); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0]) + .toMatchObject({ phase: "retryable_failure", attempt: 0, nextAttemptAt: deadline, resultId: null }); + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]); + await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null, leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + await expect(readSandbox()).resolves.toBeUndefined(); + await expect(inspect({ readControlState: () => ({ identity, commands: [{ status: "pending" }], + committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toThrow(denied); + const recovered = await inspect(); + expect(recovered.identity).toEqual(identity); + expect(recovered.checkpoint).toBe(checkpoint); + expect(recovered.reuseProcess).toBe(true); + const claims = await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId], now: deadline }); + expect(claims).toHaveLength(1); + expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "duplicate", runIds: [runId], now: deadline })).toEqual([]); + }); + it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => { const fixture = await seed(), runId = randomUUID(); const scope = { ...fixture.scope, runId }; diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index 4c5fd4ac419..223e5f69bdc 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -138,11 +138,17 @@ export async function assertNativeRetainedSandboxBinding(db: Db, input: { eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId), eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId), eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId))); - if (!owner || owner.leaseOwner || owner.phase !== "committed" || !owner.resultId || owner.recoveryState + if (!owner || !owner.resultId || owner.recoveryState || leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd || !["active", "released", "expired"].includes(leases[0].status)) { throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding"); } + if (owner.leaseOwner && owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()) { + throw new NativeControllerRecoveryWaitingError(owner.leaseExpiresAt); + } + if (owner.leaseOwner || owner.phase !== "committed") { + throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding"); + } } /** Only the existing bounded selector chooses history. Never search backwards diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index 1cbba5703cd..3a68b7d8f3b 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -3,7 +3,7 @@ import { notifyExternalAgentSettlement } from "../../modules/external-agents/ind import { notifyNativeRecoveryNotices } from "../native-recovery-notice-notifications.js"; import { resolveCommittedNativeRecoveryNotices } from "./native-recovery-notices.js"; import { notifyNativeRunProjection } from "../native-run-projection-notifications.js"; -import { projectCommittedRun, recoveredExecutionFailureMetadata } from "./native-run-projection.js"; +import { projectCommittedRun, recoveredExecutionFailureMetadata, settleNativeTerminalWake } from "./native-run-projection.js"; import { notifyNativeResultCommit } from "../native-result-commit-notifications.js"; import { notifyRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js"; import { notifyNativeFinalizationWork } from "../native-finalization-notifications.js"; @@ -465,6 +465,7 @@ async function recordRetryableFailure(input: { }) .where(eq(heartbeatRuns.id, input.run.id)) .returning(); + if (projectsTerminalStatus && updatedRun) await settleNativeTerminalWake(tx as unknown as Db, updatedRun); if (projectsTerminalStatus) terminalRunToEmit = updatedRun ?? null; if (supersededByNewerRun) { await issueRecoveryActionService( @@ -1679,6 +1680,7 @@ export async function finalizeNativeRun(input: { await notifyExecutionRunWork(tx, { companyId: run.companyId }); await notifyChatRunMilestones(tx, { companyId: run.companyId, issueId: coordinator.issueId }); const [after] = await projectRun(tx); + if (after && input.projectRunStatus) await settleNativeTerminalWake(tx, after); return after ? { before, after } : null; }); const updatedRun = projection?.after; diff --git a/server/src/services/native-runtime/native-run-projection.ts b/server/src/services/native-runtime/native-run-projection.ts index 2542b8bb1e3..706f80e7366 100644 --- a/server/src/services/native-runtime/native-run-projection.ts +++ b/server/src/services/native-runtime/native-run-projection.ts @@ -1,3 +1,5 @@ +import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js"; +import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeRecoveryNotices } from "../native-recovery-notice-notifications.js"; @@ -5,7 +7,7 @@ import { pendingNativeStatusEffectsCondition } from "./native-pending-effect-wor import { notifyNativeRunProjection } from "../native-run-projection-notifications.js"; import { randomUUID } from "node:crypto"; import { and, eq, exists, inArray, isNotNull, isNull, not, or, sql, type SQL } from "drizzle-orm"; -import { heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; +import { agentWakeupRequests, heartbeatRuns, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; import { notifyRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js"; import { notifyInstructionCollection } from "../instruction-collection-notifications.js"; import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js"; @@ -26,11 +28,41 @@ export function projectNativeTerminalRunStatus( +function claimedNativeWakeNeedsSettlement() { + return sql`exists (select 1 from ${agentWakeupRequests} where + ${agentWakeupRequests.id} = ${heartbeatRuns.wakeupRequestId} + and ${agentWakeupRequests.companyId} = ${heartbeatRuns.companyId} + and ${agentWakeupRequests.agentId} = ${heartbeatRuns.agentId} + and ${agentWakeupRequests.runId} = ${heartbeatRuns.id} + and ${agentWakeupRequests.status} = 'claimed')`; +} + +/** Settle only the exact persisted source claim inside terminal projection. */ +export async function settleNativeTerminalWake(tx: Db, run: Pick) { + if (run.runtimeMode !== "native" || !run.wakeupRequestId || !run.finishedAt + || !["succeeded", "failed", "cancelled"].includes(run.status)) return; + const claim = and(eq(agentWakeupRequests.id, run.wakeupRequestId), + eq(agentWakeupRequests.companyId, run.companyId), eq(agentWakeupRequests.agentId, run.agentId), + eq(agentWakeupRequests.runId, run.id), eq(agentWakeupRequests.status, "claimed")); + const [current] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests) + .where(claim).for("update").limit(1); + if (!current) return; + await notifyExternalConversationSettlement(tx, { companyId: run.companyId }); + await notifyStrandedAgentStatus(tx, { companyId: run.companyId, agentId: run.agentId }); + await tx.update(agentWakeupRequests).set({ + status: run.status === "succeeded" ? "completed" : run.status, + finishedAt: run.finishedAt, error: run.error, updatedAt: new Date(), + }).where(claim); +} + export function nativeRunProjectionNeedsRepair(projectedStatus: "succeeded" | "failed" | "cancelled" | SQL) { return and( eq(heartbeatRuns.runtimeMode, "native"), or( inArray(heartbeatRuns.status, ["queued", "running", "failed"]), + and(eq(heartbeatRuns.status, projectedStatus), + inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled"]), claimedNativeWakeNeedsSettlement()), and( eq(heartbeatRuns.status, "succeeded"), or( @@ -51,6 +83,7 @@ export function nativeRunProjectionNeedsRepair(projectedStatus: "succeeded" | "f or( sql`${heartbeatRuns.status} is distinct from ${projectedStatus}`, + claimedNativeWakeNeedsSettlement(), isNull(heartbeatRuns.finishedAt), sql`${heartbeatRuns.nativePhase} is distinct from 'committed'`, sql`${heartbeatRuns.resultJson}->>'finalizationPhase' is distinct from 'committed'`, @@ -164,6 +197,7 @@ export async function projectCommittedRun(input: { }) .where(needsRepair) .returning(); + if (after) await settleNativeTerminalWake(tx, after); return after ? { before, after } : null; }); diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index aa23a2d35b0..c90619e12c7 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1,5 +1,7 @@ import { renewNativeSessionExecutionLease, startNativeSessionExecutionLeaseRenewal, type NativeExecutionLeaseRenewal } from "./native-execution-lease.js"; import { enableRunFailureReporting } from "../../__tests__/helpers/run-failure-reporting.js"; +import { NativeControllerRecoveryWaitingError } from "./native-controller-recovery.js"; +import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js"; import { hasNativeLocalProcessStop } from "../native-local-process-stop.js"; import { randomUUID } from "node:crypto"; import { createRequire } from "node:module"; @@ -150,12 +152,13 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( expect(next.kind).toBe("bootstrap_incomplete"); }); - it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "stale-projection", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry", "diskless-warm-suspended", "diskless-warm-suspended-read-retry", "diskless-live-inspection"])("preserves remote warm continuity across completed turns: %s", async (scenario) => { - const hardRestart = !["warm", "same-server"].includes(scenario); + it.skipIf(!process.env.PAPERCLIP_TEST_RUNNER_CONTAINER).each(["warm", "hard", "finalizer-wait", "warm-suspended", "stale-projection", "legacy", "changed", "same-server", "diskless", "diskless-per-turn", "diskless-per-turn-read-retry", "diskless-warm-suspended", "diskless-warm-suspended-read-retry", "diskless-live-inspection"])("preserves remote warm continuity across completed turns: %s", async (scenario) => { + const hardRestart = !["warm", "same-server", "warm-suspended"].includes(scenario); const diskless = scenario.startsWith("diskless"); const perTurn = scenario.startsWith("diskless-per-turn"); - const warmSuspended = scenario.startsWith("diskless-warm-suspended"); + const warmSuspended = scenario === "warm-suspended" || scenario.startsWith("diskless-warm-suspended"); const resumedAfterSuspension = perTurn || warmSuspended; + const suspendedBeforeThird = perTurn || (diskless && warmSuspended); const container = process.env.PAPERCLIP_TEST_RUNNER_CONTAINER!; const tunnel = await createRunnerTestTlsTunnel({ port: Number(process.env.PAPERCLIP_TEST_RUNNER_PORT), key: process.env.PAPERCLIP_TEST_RUNNER_TLS_KEY!, cert: process.env.NODE_EXTRA_CA_CERTS! }); @@ -310,6 +313,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( }; const first = hardRestart ? await executeInController(firstOptions) : await executeFirst(); expect(first.exitCode).toBe(0); + if (scenario === "warm-suspended") expect(await closeIdleWarmNativeSessionsForRestart()) + .toEqual({ closed: 1, busy: 0, failed: 0 }); await finalizeNativeRun({ db: fixture.db, runId: fixture.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); const snapshot = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] }); expect(snapshot.exitCode).toBe(0); @@ -405,6 +410,23 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( ...(perTurn ? { sessionCheckpoint: admission.checkpoint } : {}) } }).where(eq(heartbeatRuns.id, row.id)); }; await admitContinuation(nextOptions.execution); + let continuationLeaseOwner: string | undefined; + if (scenario === "finalizer-wait") { + const retryAt = new Date(Date.now() + 60_000); + await fixture.db.update(nativeRunFinalizations).set({ leaseOwner: "fixture-finalizer", leaseExpiresAt: retryAt }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + await expect(executePaperclipNativeSession(nextOptions)).rejects.toBeInstanceOf(NativeControllerRecoveryWaitingError); + const [waiting] = await fixture.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, nextRunId)); + expect(waiting).toMatchObject({ phase: "retryable_failure", attempt: 0, nextAttemptAt: retryAt }); + const calls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] }); + expect(calls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(1); + await fixture.db.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null }) + .where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [claim] = await claimNativeSessionResumptions({ db: fixture.db, runnerInstanceId: next.runnerInstanceId, + runIds: [nextRunId], now: retryAt }); + expect(claim).toBeDefined(); + continuationLeaseOwner = claim.leaseOwner; + } if (scenario.endsWith("-read-retry")) { let rejectedReads = 0; await expect(executePaperclipNativeSession({ ...nextOptions, @@ -422,7 +444,8 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( // Retry with the partial PRP projection left by the failed read. No // fixture writes a suspended receipt or changes the surviving Runner. } - const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession(nextOptions); + const second = diskless ? await executeInController(nextOptions) : await executePaperclipNativeSession({ ...nextOptions, + ...(continuationLeaseOwner ? { leaseOwner: continuationLeaseOwner } : {}) }); expect(second.exitCode).toBe(0); if (scenario === "diskless-live-inspection") expect("inspectionOverridden" in second && second.inspectionOverridden).toBe(true); const historyAfter = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/filesystem/codex-home/fake-codex-state.json`] }); @@ -476,9 +499,9 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", ( const finalCalls = await runner.execute({ command: "cat", args: [`${remoteCwd}/provider-calls.log`] }); expect(finalCalls.stdout.split("\n").filter(line => line === "turn/start")).toHaveLength(3); expect(finalCalls.stdout.split("\n").filter(line => line === "thread/start")).toHaveLength(1); - expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (resumedAfterSuspension ? 1 : 0)); + expect(finalCalls.stdout.split("\n").filter(line => line === "process-start")).toHaveLength(providerStarts + (suspendedBeforeThird ? 1 : 0)); const finalProcess = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-process.identity`] }); - if (resumedAfterSuspension) expect(finalProcess.stdout).not.toBe(processAfter.stdout); + if (suspendedBeforeThird) expect(finalProcess.stdout).not.toBe(processAfter.stdout); else expect(finalProcess.stdout).toBe(processAfter.stdout); expect(await closeIdleWarmNativeSessionsForRestart()).toEqual({ closed: perTurn ? 0 : 1, busy: 0, failed: 0 }); const suspended = await runner.execute({ command: "sh", args: ["-c", `cat '${remoteCwd}'/.paperclip-runtime/paperclip-runner/sessions/*/runner/runner-state.json`] }); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 725f07f2ea0..94fa60242b5 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12967,9 +12967,10 @@ describe("runnerd provider runtime wiring", () => { }, ); - it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "external-checkpoint", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => { + it.each((["codex", "claude"] as const).flatMap(provider => (["unchanged", "external-checkpoint", "suspension-receipt", "suspension-active-owner", "changed", "runtime-context", "retirement-failed", "stale-owner", "active-owner", "missing-owner", "foreign-environment", "plugin-replaced", "plugin-managed-github"] as const).filter(scenario => provider === "codex" || scenario.startsWith("plugin-")).map(scenario => [provider, scenario] as const)))("rotates an owned warm %s remote runner before checking changed configuration: %s", async (provider, scenario) => { const plugin = scenario === "plugin-replaced" || scenario === "plugin-managed-github"; - const reusable = ["unchanged", "external-checkpoint", "plugin-managed-github"].includes(scenario); + const suspensionReceipt = scenario.startsWith("suspension-"); + const reusable = suspensionReceipt || ["unchanged", "external-checkpoint", "plugin-managed-github"].includes(scenario); const { permissionPolicy: _permissionPolicy, ...claudeProfile } = QUALIFIED_ACPX_PROFILES.claude; const first = { ...scopedNativeExecution(`warm-admission-${provider}-${scenario}`), schema: "paperclip.native-execution-input.v5" as const, @@ -12978,7 +12979,7 @@ describe("runnerd provider runtime wiring", () => { if (provider === "claude") first.session.driverKind = "acpx_runtime"; first.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 60_000 }; const next = { ...first, binding: { ...first.binding, runId: `${first.binding.runId}-next` }, - session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: ["unchanged", "external-checkpoint", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } }; + session: { ...first.session, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: suspensionReceipt || ["unchanged", "external-checkpoint", "runtime-context", "stale-owner", "missing-owner", "plugin-replaced", "plugin-managed-github"].includes(scenario) ? 60_000 : 90_000 } } }; if (scenario === "runtime-context") { next.runtimeContext = { ...first.runtimeContext, mcp: { ...first.runtimeContext.mcp, digest: "1".repeat(64) } }; @@ -13084,11 +13085,17 @@ describe("runnerd provider runtime wiring", () => { const ordinarySelect = nextDb.select.bind(nextDb); nextDb.select = ((fields?: Record) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson - ? priorNativeRunDb(scenario === "stale-owner" ? older : first, scenario === "active-owner" ? "running" : "succeeded").select(fields as never) + ? priorNativeRunDb(scenario === "stale-owner" ? older : first, ["active-owner", "suspension-active-owner"].includes(scenario) ? "running" : "succeeded").select(fields as never) : ordinarySelect(fields as never)) as Db["select"]; try { await executePaperclipNativeSession({ db: firstDb, execution: first, runnerInstanceId: identity.runnerInstanceId, useRunnerd: true, runnerExecutionTarget: pluginTarget as never,...pluginOptions }); + if (suspensionReceipt) { + // Remote suspension leaves a local identity receipt, while the Runner + // owns its full state in the sandbox. A resumed turn must retire it. + await writeFile(join(root, "plugin-runner-state.json"), JSON.stringify(durableRunnerState(identity, "suspended"))); + await rm(join(root, "runner/runner-state.json")); + } if (scenario === "stale-owner") await writeRunnerAuthority(root, { ...identity, runId: older.binding.runId }, "ready"); if (scenario === "missing-owner") { await closeWarmNativeSessionsForEnvironment({ environmentId, reason: "fixture owner loss" }); @@ -13101,12 +13108,14 @@ describe("runnerd provider runtime wiring", () => { runnerInstanceId: `runner-next-${scenario}`, useRunnerd: true, runnerExecutionTarget: (scenario === "foreign-environment" ? { ...target, environmentId: `${environmentId}-other` } : plugin ? {...pluginTarget,leaseId:"next-core-lease"} : target) as never,...pluginOptions }); - if (["stale-owner", "active-owner", "missing-owner", "foreign-environment"].includes(scenario)) { - await expect(continuation).rejects.toThrow(scenario === "active-owner" + if (["stale-owner", "active-owner", "suspension-active-owner", "missing-owner", "foreign-environment"].includes(scenario)) { + await expect(continuation).rejects.toThrow(["active-owner", "suspension-active-owner"].includes(scenario) ? "prior_owner_active" : "prior_owner_terminal_state_indeterminate"); expect(state.execute).toHaveBeenCalledOnce(); expect(firstSession.close).not.toHaveBeenCalled(); expect(order).toEqual([]); + if (suspensionReceipt) expect(JSON.parse(await readFile(join(root, "plugin-runner-state.json"), "utf8"))) + .toMatchObject({ ...identity, lifecycle: "suspended" }); } else if (scenario === "retirement-failed") { await expect(continuation).rejects.toThrow("retirement unconfirmed"); expect(state.execute).toHaveBeenCalledOnce(); @@ -13120,8 +13129,9 @@ describe("runnerd provider runtime wiring", () => { } if (scenario === "plugin-managed-github") expect(rebind).toHaveBeenCalledOnce(); expect(order).toEqual(reusable ? ["execute-next"] : ["retire", "execute-next"]); - if (["unchanged", "external-checkpoint"].includes(scenario)) { + if (["unchanged", "external-checkpoint", "suspension-receipt"].includes(scenario)) { if (!persistedController) await writeRunnerAuthority(root, { ...identity, runId: next.binding.runId }, "ready"); + if (suspensionReceipt) await rm(join(root, "runner/runner-state.json")); const third = { ...next, binding: { ...next.binding, runId: `${next.binding.runId}-third` } }; const thirdDb = leaseDb(third); const thirdSelect = thirdDb.select.bind(thirdDb); @@ -13139,6 +13149,11 @@ describe("runnerd provider runtime wiring", () => { runnerExecutionTarget: pluginTarget as never,...pluginOptions })).resolves.toMatchObject({ exitCode: 0 }); expect(firstSession.close).not.toHaveBeenCalled(); expect(state.execute).toHaveBeenCalledTimes(3); + if (suspensionReceipt) { + expect(existsSync(join(root, "plugin-runner-state.json"))).toBe(false); + expect(JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")).identity) + .toMatchObject({ ...identity, runId: next.binding.runId }); + } if (restore) { expect(restore.mock.calls.map(([, binding]) => binding.runId)).toEqual([first.binding.runId, next.binding.runId]); expect(existsSync(root)).toBe(true); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c68407087e2..7264f55e6f1 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9844,7 +9844,8 @@ async function executePaperclipNativeSessionWithinScope( ); }, onSession: async (session) => { - if (session && usesPluginTask) { + if (session && input.runnerExecutionTarget?.kind === "remote" + && input.runnerExecutionTarget.transport === "sandbox") { rmSync(resolve(scopedRunnerdStateRoot(input.execution), "plugin-runner-state.json"), { force: true }); } if (session && session === existingWarmSession) remoteWarmRecoveryPending = false; diff --git a/server/src/services/native-runtime/provider-api-placement.test.ts b/server/src/services/native-runtime/provider-api-placement.test.ts new file mode 100644 index 00000000000..dc1389b8c3c --- /dev/null +++ b/server/src/services/native-runtime/provider-api-placement.test.ts @@ -0,0 +1,56 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { buildNativeExecutionInput } from "./native-execution-input.js"; +import { nativeRuntimeContextFixture } from "./runtime-context.test-fixture.js"; +import { resolveProviderApiPlacement } from "./provider-api-placement.js"; +import { evaluateExecutionAllowlist } from "../execution-allowlist.js"; + +const placement = { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" }; +const agent = { adapterType: "paperclip_runner", adapterConfig: { + provider: "cursor_cloud", model: "caller-model", repoUrl: "https://github.com/example/qualification", +} }; +function execution(provider: "cursor_cloud" | "codex") { + return buildNativeExecutionInput({ companyId: randomUUID(), agentId: randomUUID(), runId: randomUUID(), provider, + model: "caller-model", ...(provider === "cursor_cloud" ? { cursorCloud: { schema: "paperclip.cursor-cloud.config.v1" as const, repos: [{ url: "https://github.com/example/qualification" }] } } : {}), + issue: { id: randomUUID(), identifier: "API-1", title: "Bound task", description: null, workMode: "standard" }, + taskPrompt: "Complete the task", normalizedSessionId: randomUUID(), + workspace: { id: randomUUID(), cwd: "/private/controller", repoUrl: null, repoRef: null, branchName: null }, + completionContract: { id: randomUUID(), sha256: `sha256:${"a".repeat(64)}`, schemaVersion: "paperclip.run-result.v1", + contract: { revision: "1", objective: "Complete the task", criteria: [{ id: "objective", requirement: "Complete the task" }] } }, + runtimeContext: nativeRuntimeContextFixture(), + }); +} + +describe("provider API controller placement", () => { + it("derives API placement only from a validated native Cursor contract while local agent code stays denied", () => { + expect(resolveProviderApiPlacement({ agent })).toEqual(placement); + expect(resolveProviderApiPlacement({ agent: { ...agent, adapterType: "cursor_cloud" } })).toBeNull(); + expect(resolveProviderApiPlacement({ agent: { ...agent, adapterConfig: { provider: "codex", executionPlacement: placement } } })).toBeNull(); + expect(evaluateExecutionAllowlist({ managedSandboxOnly: true }, { driver: "local", provider: null }).allowed).toBe(false); + }); + it.each(["command", "extraArgs", "cwd", "workspaceStrategy", "workspaceRuntime", "executionPlacement", "providerApiPlacement", "controllerPlacement", "apiOnly"])("refuses configurable host authority through %s", key => { + expect(() => resolveProviderApiPlacement({ agent: { ...agent, adapterConfig: { ...agent.adapterConfig, [key]: key === "extraArgs" ? ["--execute"] : "injected" } } })).toThrow(); + }); + it("requires exact model and refuses unrelated environment or a preconfigured home", () => { + for (const patch of [{ model: "" }, { env: { NODE_OPTIONS: "--require=untrusted" } }, { homeMode: "preconfigured" }]) { + expect(() => resolveProviderApiPlacement({ agent: { ...agent, adapterConfig: { ...agent.adapterConfig, ...patch } } })).toThrow(); + } + }); + it("restores the admitted provider rather than reinterpreting a later agent configuration", () => { + const nativeExecutionInput = execution("cursor_cloud"); + const run = { runtimeMode: "native", runnerProfileJson: { backend: "cursor_cloud_api", executionPlacement: placement, nativeExecutionInput } }; + expect(resolveProviderApiPlacement({ agent: { ...agent, adapterConfig: { provider: "codex" } }, run })).toEqual(placement); + expect(() => resolveProviderApiPlacement({ agent, run: { ...run, runnerProfileJson: { ...run.runnerProfileJson, backend: "codex_app_server" } } })).toThrow("binding_mismatch"); + expect(() => resolveProviderApiPlacement({ agent, run: { ...run, runnerProfileJson: { ...run.runnerProfileJson, nativeExecutionInput: execution("codex") } } })).toThrow("binding_mismatch"); + expect(resolveProviderApiPlacement({ agent, run: { runtimeMode: "native", runnerProfileJson: { nativeExecutionInput: execution("codex") } } })).toBeNull(); + }); + it("binds historical Cursor input to its backend and recorded driver before granting API placement", () => { + const nativeExecutionInput = execution("cursor_cloud"); + const run = { runtimeMode: "native", driverKind: "cursor_cloud_api", runnerProfileJson: { backend: "cursor_cloud_api", nativeExecutionInput } }; + expect(resolveProviderApiPlacement({ agent, run })).toEqual(placement); + expect(resolveProviderApiPlacement({ agent, run: { ...run, driverKind: null } })).toEqual(placement); + expect(resolveProviderApiPlacement({ agent, run: { ...run, driverKind: null, runnerProfileJson: { nativeExecutionInput } } })).toEqual(placement); + expect(() => resolveProviderApiPlacement({ agent, run: { ...run, runnerProfileJson: { ...run.runnerProfileJson, backend: "codex_app_server" } } })).toThrow("binding_mismatch"); + expect(() => resolveProviderApiPlacement({ agent, run: { ...run, driverKind: "codex_app_server" } })).toThrow("binding_mismatch"); + }); +}); diff --git a/server/src/services/native-runtime/provider-api-placement.ts b/server/src/services/native-runtime/provider-api-placement.ts new file mode 100644 index 00000000000..3029508a8e9 --- /dev/null +++ b/server/src/services/native-runtime/provider-api-placement.ts @@ -0,0 +1,60 @@ +import { parseObject } from "../../adapters/utils.js"; +import { unprocessable } from "../../errors.js"; +import { isExecutionForcedToKubernetes, type ExecutionPolicy } from "../execution-allowlist.js"; +import { parseNativeExecutionInput } from "../../vendor/paperclip-runner/index.js"; +import { resolvePaperclipRunnerNativeProviderInput } from "./provider-profile.js"; + +export interface ProviderApiPlacement { + kind: "provider_api"; + provider: "cursor_cloud"; + controller: "paperclip"; +} + +export function managedCursorApiDefaultEnabled(experimental: { + enableManagedSandboxOnly?: boolean; enableNativeRunner?: boolean; +} | null | undefined): boolean { + return experimental?.enableManagedSandboxOnly === true && experimental.enableNativeRunner === true; +} + +export function assertProviderApiPlacementPolicy(policy: ExecutionPolicy): void { + if (isExecutionForcedToKubernetes(policy)) { + throw unprocessable("Cursor Cloud API execution is unavailable under the Kubernetes-only policy.", { + code: "provider_api_kubernetes_execution_unsupported", + }); + } +} + +/** Held setup may be incomplete. This only skips unused Computer preparation; + * active hire and verification must still derive a fully validated placement. */ +export function isProviderApiSetupDraft(agent: { adapterType?: string; adapterConfig?: unknown }): boolean { + return agent.adapterType === "paperclip_runner" && parseObject(agent.adapterConfig).provider === "cursor_cloud"; +} + +/** Placement is server-owned. A provider's API client is not an agent Computer. */ +export function resolveProviderApiPlacement(input: { + agent: { adapterType: string; adapterConfig: unknown }; + run?: { runtimeMode: string | null; runnerProfileJson: unknown; driverKind?: string | null }; +}): ProviderApiPlacement | null { + const profile = parseObject(input.run?.runnerProfileJson); + const persisted = input.run?.runtimeMode === "native" && profile.nativeExecutionInput !== undefined + ? parseNativeExecutionInput(profile.nativeExecutionInput) : null; + if (persisted?.provider.kind === "cursor_cloud" + && ((profile.backend != null && profile.backend !== "cursor_cloud_api") + || (input.run?.driverKind != null && input.run.driverKind !== "cursor_cloud_api"))) { + throw new Error("provider_api_placement_binding_mismatch"); + } + const saved = profile.executionPlacement; + const savedRecord = parseObject(saved); + if (saved !== undefined && (Object.keys(savedRecord).sort().join(",") !== "controller,kind,provider" + || savedRecord.kind !== "provider_api" || savedRecord.provider !== "cursor_cloud" || savedRecord.controller !== "paperclip" + || persisted?.provider.kind !== "cursor_cloud" || profile.backend !== "cursor_cloud_api")) { + throw new Error("provider_api_placement_binding_mismatch"); + } + if (persisted) return persisted.provider.kind === "cursor_cloud" + ? { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" } : null; + const config = parseObject(input.agent.adapterConfig); + if (input.agent.adapterType !== "paperclip_runner" || config.provider !== "cursor_cloud") return null; + // Validate the provider's complete launch contract before granting placement. + resolvePaperclipRunnerNativeProviderInput({ backend: "cursor_cloud_api", adapterConfig: config }); + return { kind: "provider_api", provider: "cursor_cloud", controller: "paperclip" }; +} diff --git a/server/src/services/native-runtime/provider-profile.ts b/server/src/services/native-runtime/provider-profile.ts index 301c9b590c5..d1d7cb0bc93 100644 --- a/server/src/services/native-runtime/provider-profile.ts +++ b/server/src/services/native-runtime/provider-profile.ts @@ -182,7 +182,7 @@ export function resolvePaperclipRunnerCursorCloudConfig(config: Record config[key] !== undefined && config[key] !== "" && !(Array.isArray(config[key]) && config[key].length === 0)); if (config.lifecycleMode !== undefined && config.lifecycleMode !== "per_turn") unsupported.push("lifecycleMode"); if (config.homeMode === "preconfigured") unsupported.push("homeMode"); diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index 893dfefaef6..ec63a72325e 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -3,7 +3,7 @@ import { agents, companies, environmentLeases, heartbeatRuns, issues, nativeRunF import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import { parseNativeExecutionInput, type NativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { currentNativeControllerIdentity, evaluateNativeControllerTakeover } from "./native-restart-recovery.js"; -import { readNativeControllerRecovery } from "./native-controller-recovery.js"; +import { NativeControllerRecoveryWaitingError, readNativeControllerRecovery } from "./native-controller-recovery.js"; export type RemoteWarmSessionRecovery = { identity: Record; @@ -71,11 +71,14 @@ export async function inspectRemoteWarmSessionRecovery(input: { .from(heartbeatRuns).innerJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) .where(and(eq(heartbeatRuns.id, identity.runId), eq(heartbeatRuns.companyId, binding.companyId), eq(heartbeatRuns.agentId, binding.agentId), eq(heartbeatRuns.nativeIssueId, binding.issueId))); - if (!prior || prior.run.status !== "succeeded" || prior.owner.phase !== "committed" || - !prior.owner.resultId || prior.owner.leaseOwner || !prior.owner.controllerPid || + if (!prior || prior.run.status !== "succeeded" || !prior.owner.resultId || !prior.owner.controllerPid || !prior.owner.controllerProcessStartedAt || !prior.owner.controllerBootId) return deny(); const previous = parseNativeExecutionInput(prior.run.runnerProfileJson?.nativeExecutionInput); if (previous.binding.runId !== identity.runId || !input.matchesExecution(previous)) return deny(); + if (prior.owner.leaseOwner && prior.owner.leaseExpiresAt && prior.owner.leaseExpiresAt > new Date()) { + throw new NativeControllerRecoveryWaitingError(prior.owner.leaseExpiresAt); + } + if (prior.owner.phase !== "committed" || prior.owner.leaseOwner) return deny(); const checkpoint = input.loadCheckpoint(previous); if (!checkpoint || checkpoint.identity.runId !== identity.runId || checkpoint.identity.companyId !== binding.companyId || checkpoint.identity.issueId !== binding.issueId || diff --git a/server/src/services/pre-dispatch-execution-evidence.ts b/server/src/services/pre-dispatch-execution-evidence.ts new file mode 100644 index 00000000000..b7b9aa93408 --- /dev/null +++ b/server/src/services/pre-dispatch-execution-evidence.ts @@ -0,0 +1,32 @@ +import { and, eq, inArray, isNotNull, isNull, or, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations } from "@paperclipai/db"; +import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; +import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; + +export function executionStartedEventPredicate() { + return or(isNotNull(heartbeatRunEvents.sourceEventId), inArray(heartbeatRunEvents.eventType, + ["adapter.invoke", PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, "harness.ready", "session.started", + "session.resumed", "session.updated", "turn.started", "provider.event", "provider.rpc_result", "tool.execution.started"])); +} + +/** Both dispatch cancellation policies require this same retained-evidence + * proof. Each caller still owns its terminal status and admission receipt. */ +export function noRetainedExecutionPredicate() { + return and( + isNull(heartbeatRuns.startedAt), isNull(heartbeatRuns.runtimeModeResolvedAt), + isNull(heartbeatRuns.processPid), isNull(heartbeatRuns.processGroupId), isNull(heartbeatRuns.processStartedAt), + isNull(heartbeatRuns.nativeIssueId), isNull(heartbeatRuns.nativeSessionId), isNull(heartbeatRuns.sessionIdAfter), + isNull(heartbeatRuns.controllerBootId), isNull(heartbeatRuns.controllerLeaseExpiresAt), isNull(heartbeatRuns.executionStage), + sql`coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive'`, + sql`coalesce(${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA}`, + sql`not (coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) ?| array[ + 'nativeExecutionInput', 'nativeWorkspaceSync', 'remoteWorkspace', 'nativeWorkspaceFinalizationOwner', 'sessionCheckpoint'])`, + sql`not exists (select 1 from ${nativeRunFinalizations} + where ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} and ${nativeRunFinalizations.runId} = ${heartbeatRuns.id})`, + sql`not exists (select 1 from ${environmentLeases} + where ${environmentLeases.companyId} = ${heartbeatRuns.companyId} and ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id})`, + sql`not exists (select 1 from ${heartbeatRunEvents} + where ${and(eq(heartbeatRunEvents.companyId, heartbeatRuns.companyId), + eq(heartbeatRunEvents.runId, heartbeatRuns.id), executionStartedEventPredicate())})`, + ); +} diff --git a/server/src/services/pre-dispatch-execution-wait.ts b/server/src/services/pre-dispatch-execution-wait.ts index 228d1a12d74..b91bbc878be 100644 --- a/server/src/services/pre-dispatch-execution-wait.ts +++ b/server/src/services/pre-dispatch-execution-wait.ts @@ -1,7 +1,7 @@ import { and, eq, sql } from "drizzle-orm"; -import { environmentLeases, heartbeatRunEvents, heartbeatRuns, issueRecoveryActions, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { heartbeatRuns, issueRecoveryActions, type Db } from "@paperclipai/db"; import { canonicalExecutionUuid } from "./execution-identity.js"; -import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; +import { noRetainedExecutionPredicate } from "./pre-dispatch-execution-evidence.js"; import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "./workspace-restore-recovery-state.js"; /** Admission denied by an existing hold is not a provider attempt. Require the @@ -18,27 +18,7 @@ function preDispatchExecutionWaitPredicate() { AND (${heartbeatRuns.issueId} IS NULL OR ${heartbeatRuns.issueId} = ${canonicalExecutionUuid(sql`${heartbeatRuns.contextSnapshot}->>'issueId'`)}) AND ${canonicalExecutionUuid(sql`${heartbeatRuns.resultJson}->'executionWait'->>'recoveryActionId'`)} IS NOT NULL - AND ${heartbeatRuns.startedAt} IS NULL AND ${heartbeatRuns.runtimeModeResolvedAt} IS NULL - AND ${heartbeatRuns.processPid} IS NULL AND ${heartbeatRuns.processGroupId} IS NULL - AND ${heartbeatRuns.processStartedAt} IS NULL AND ${heartbeatRuns.nativeIssueId} IS NULL - AND ${heartbeatRuns.nativeSessionId} IS NULL AND ${heartbeatRuns.sessionIdAfter} IS NULL - AND ${heartbeatRuns.controllerBootId} IS NULL AND ${heartbeatRuns.controllerLeaseExpiresAt} IS NULL - AND ${heartbeatRuns.executionStage} IS NULL - AND coalesce(${heartbeatRuns.resultJson}->>'workspaceRestoreFailure', '') <> 'restore_unsafe_archive' - AND coalesce(${heartbeatRuns.resultJson}->'workspaceRestoreRecovery'->>'schema', '') <> ${LEGACY_WORKSPACE_RECOVERY_SCHEMA} - AND NOT EXISTS (SELECT 1 FROM ${environmentLeases} - WHERE ${environmentLeases.companyId} = ${heartbeatRuns.companyId} - AND ${environmentLeases.heartbeatRunId} = ${heartbeatRuns.id}) - AND NOT EXISTS (SELECT 1 FROM ${nativeRunFinalizations} - WHERE ${nativeRunFinalizations.companyId} = ${heartbeatRuns.companyId} - AND ${nativeRunFinalizations.runId} = ${heartbeatRuns.id}) - AND NOT EXISTS (SELECT 1 FROM ${heartbeatRunEvents} - WHERE ${heartbeatRunEvents.companyId} = ${heartbeatRuns.companyId} - AND ${heartbeatRunEvents.runId} = ${heartbeatRuns.id} - AND (${heartbeatRunEvents.sourceEventId} IS NOT NULL OR ${heartbeatRunEvents.eventType} IN - ('adapter.invoke', ${PROCESS_START_REQUESTED}, ${PROCESS_IDENTITY_RECORDED}, 'harness.ready', - 'session.started', 'session.resumed', 'session.updated', 'turn.started', - 'provider.event', 'provider.rpc_result', 'tool.execution.started'))), false)`; + AND ${noRetainedExecutionPredicate()}, false)`; } export async function isPreDispatchExecutionWaitVerified(db: Db, run: typeof heartbeatRuns.$inferSelect): Promise { diff --git a/server/src/services/pre-dispatch-review-wait.ts b/server/src/services/pre-dispatch-review-wait.ts index 7448d5f8e55..68454275e1e 100644 --- a/server/src/services/pre-dispatch-review-wait.ts +++ b/server/src/services/pre-dispatch-review-wait.ts @@ -1,13 +1,43 @@ -import { and, eq, inArray, isNotNull, or } from "drizzle-orm"; +import { and, eq, inArray, isNotNull } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; -import { PROCESS_IDENTITY_RECORDED, PROCESS_START_REQUESTED } from "./native-local-process-stop.js"; import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; +import { executionStartedEventPredicate, noRetainedExecutionPredicate } from "./pre-dispatch-execution-evidence.js"; +import type { QueuedRunStalenessErrorCode } from "../modules/run-dispatch/index.js"; type Run = typeof heartbeatRuns.$inferSelect; type DispatchFields = "startedAt" | "runtimeModeResolvedAt" | "processPid" | "processGroupId" | "processStartedAt" | "nativeIssueId" | "nativeSessionId" | "sessionIdAfter" | "controllerBootId" | "controllerLeaseExpiresAt" | "executionStage" | "contextSnapshot"; +const staleDispatchReconciliationExemptions = { + issue_not_found: true, + issue_not_in_progress: true, + issue_assignee_changed: true, + issue_waiting_for_response: true, + issue_continuation_waiting_on_review: true, + issue_dependencies_blocked: true, + issue_blocked: true, + issue_execution_lock_changed: true, + issue_terminal_status: true, + issue_review_participant_changed: true, + // This code requires its own executionWait receipt in pre-dispatch-execution-wait.ts. + execution_reconciliation_required: false, +} satisfies Record; +const staleDispatchErrorCodes = Object.entries(staleDispatchReconciliationExemptions) + .filter(([, exempt]) => exempt).map(([errorCode]) => errorCode); + +/** A dispatch cancellation is not an interrupted provider attempt. Historical + * rows need no new receipt, but every retained execution reference must agree. */ +export function neverStartedDispatchCancellationPredicate() { + return and( + eq(heartbeatRuns.runtimeMode, "legacy"), eq(heartbeatRuns.status, "cancelled"), + inArray(heartbeatRuns.errorCode, staleDispatchErrorCodes), + isNotNull(heartbeatRuns.finishedAt), + noRetainedExecutionPredicate(), + ); +} + /** The queued-run gate owns this receipt before execution authority is claimed. * A deliberate review or connection response wait has no provider actions to reconcile. The error code * alone is not proof; missing or conflicting dispatch evidence retains the hold. */ @@ -39,6 +69,17 @@ export function isPreDispatchReviewWait( * retained execution or cleanup evidence. Historical false holds must be * checked against the same evidence that gates explicit user admission. */ export async function isPreDispatchReviewWaitVerified(db: Db, run: Run): Promise { + if (run.runtimeMode === "legacy" && run.status === "cancelled" && run.finishedAt !== null && + staleDispatchErrorCodes.some(errorCode => errorCode === run.errorCode) && + [run.startedAt, run.runtimeModeResolvedAt, run.processPid, run.processGroupId, run.processStartedAt, + run.nativeIssueId, run.nativeSessionId, run.sessionIdAfter, run.controllerBootId, + run.controllerLeaseExpiresAt, run.executionStage].every(value => value === null) && + run.resultJson?.workspaceRestoreFailure !== "restore_unsafe_archive" && !hasRequiredWorkspaceRecovery(run.resultJson)) { + const [unstarted] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, run.companyId), eq(heartbeatRuns.id, run.id), neverStartedDispatchCancellationPredicate(), + )).limit(1); + if (unstarted) return true; + } if (!isPreDispatchReviewWait(run)) return false; const [coordinator] = await db.select({ id: nativeRunFinalizations.runId }).from(nativeRunFinalizations).where(and( eq(nativeRunFinalizations.companyId, run.companyId), eq(nativeRunFinalizations.runId, run.id), @@ -52,10 +93,7 @@ export async function isPreDispatchReviewWaitVerified(db: Db, run: Run): Promise : !hasRemoteTerminationReceipt(lease))) return false; const [execution] = await db.select({ id: heartbeatRunEvents.id }).from(heartbeatRunEvents).where(and( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), - or(isNotNull(heartbeatRunEvents.sourceEventId), - inArray(heartbeatRunEvents.eventType, ["adapter.invoke", PROCESS_START_REQUESTED, PROCESS_IDENTITY_RECORDED, - "harness.ready", "session.started", "session.resumed", "session.updated", "turn.started", - "provider.event", "provider.rpc_result", "tool.execution.started"])), + executionStartedEventPredicate(), )).limit(1); return !execution; } diff --git a/server/src/services/projects.ts b/server/src/services/projects.ts index 45891a218a6..5af27d351a7 100644 --- a/server/src/services/projects.ts +++ b/server/src/services/projects.ts @@ -43,6 +43,7 @@ import { parseProjectExecutionWorkspacePolicy } from "./execution-workspace-poli import { mergeProjectWorkspaceRuntimeConfig, readProjectWorkspaceRuntimeConfig } from "./project-workspace-runtime-config.js"; import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; import { recordProjectLifecycleEvent, recordResourceCreationEvent } from "./resource-lifecycle-events.js"; +import { hasUnsettledProjectCharges } from "../modules/execution-admission/index.js"; type ProjectTransaction = Parameters[0]>[0]; @@ -1085,6 +1086,11 @@ export function projectService(db: Db, options: { captureWorkspaceUpdates?: bool const [retained] = await tx.select({ id: executionWorkspaces.id }).from(executionWorkspaces) .where(eq(executionWorkspaces.projectId, id)).limit(1); if (retained) throw conflict("This project still supplies access policy for retained task workspaces. Archive the project or explicitly clean up its workspaces before deleting it."); + if (await hasUnsettledProjectCharges(tx as unknown as Db, { companyId: project.companyId, projectId: id })) { + throw conflict("Task executions admitted to this project have not finished saving. Wait for them to settle before deleting the project.", { + code: "project_execution_admission_pending", + }); + } await tx.select({ id: projectWorkspaces.id }).from(projectWorkspaces).where(eq(projectWorkspaces.projectId, id)).for("update"); await recordProjectLifecycleEvent(tx as unknown as Db, project.companyId, id, "delete"); await notifyDeliveryWork(tx, DELIVERY_QUEUES.skillSourceSync); diff --git a/server/src/services/remote-workspace-migration-batch.ts b/server/src/services/remote-workspace-migration-batch.ts new file mode 100644 index 00000000000..baa3f090739 --- /dev/null +++ b/server/src/services/remote-workspace-migration-batch.ts @@ -0,0 +1,34 @@ +import { and, asc, desc, isNotNull, sql } from "drizzle-orm"; +import { executionWorkspaces, type Db } from "@paperclipai/db"; +import type { StorageService } from "../storage/types.js"; +import { offlineWorkspaceMigrationBlockers } from "./remote-workspace-migration-tree.js"; +import { migrateRemoteWorkspace } from "./remote-workspace-migration.js"; + +/** The Cloud maintenance process owns the stop-all-deployments precondition. + * Imported records leave the pending set, so a restart resumes the same owner + * journals without moving the task binding or importing over remote authority. */ +export async function migrateWorkspaceTrees(db: Db, input: { + storage: StorageService; signal: AbortSignal; phase: (name: string) => void; +}) { + const blockers = await offlineWorkspaceMigrationBlockers(db); + if (blockers.length) throw new Error(blockers[0]); + let migrated = 0; + for (;;) { + input.signal.throwIfAborted(); + // Match the final ephemeral-storage contract, not just the importer's + // supported subset. Silently skipping an unsupported provider or unsettled + // lifecycle state would abandon files when the original volume is detached. + const rows = await db.select({ companyId: executionWorkspaces.companyId, workspaceId: executionWorkspaces.id }) + .from(executionWorkspaces).where(and(isNotNull(executionWorkspaces.cwd), + sql`${executionWorkspaces.providerType} <> 'plugin'`, + sql`coalesce(${executionWorkspaces.metadata}->'remoteWorkspace'->>'schema', '') <> 'paperclip.remote-workspace.v1'`, + )).orderBy(desc(sql`length(${executionWorkspaces.cwd})`), asc(executionWorkspaces.id)).limit(25); + if (!rows.length) return migrated; + for (const workspace of rows) { + input.signal.throwIfAborted(); + input.phase(`workspace:${workspace.workspaceId}`); + await migrateRemoteWorkspace({ db, storage: input.storage, ...workspace, apply: true, writersStopped: true, signal: input.signal }); + migrated++; + } + } +} diff --git a/server/src/services/remote-workspace-migration-git.ts b/server/src/services/remote-workspace-migration-git.ts new file mode 100644 index 00000000000..66d82639c38 --- /dev/null +++ b/server/src/services/remote-workspace-migration-git.ts @@ -0,0 +1,142 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { and, desc, eq, gte, or, sql } from "drizzle-orm"; +import { executionWorkspaces, projectWorkspaces, workspaceOperations, type Db } from "@paperclipai/db"; +import { PROJECT_REPOSITORIES_DIR } from "@paperclipai/adapter-utils/git-workspace-sync"; +import type { StorageService } from "../storage/types.js"; +import { createIssueGitRemoteAuthProvider } from "./git-credentials.js"; +import { applyGitRestorePointToWorktree, ensureGitRestorePointCommit, loadGitRestorePoint } from "./git-restore-point-storage.js"; +import { restoreGitWorkspaceContents } from "./git-workspace-contents.js"; +import { ensureSavedGitCommit, gitRestorePointExpired, managedSourcePaths, readGitCacheRestoration, + verifiedGitCacheSource, type GitCacheRestoration } from "./git-workspace-restoration.js"; +import { localRepositoryPath } from "./local-repository-path.js"; +import { materializeManagedProjectWorkspace } from "./managed-repository-checkout.js"; + +type Workspace = typeof executionWorkspaces.$inferSelect; +export type ArchivedGitMigrationSource = { + commit: string; + sourcePath: string; + cleanupOperationId: string | null; + recipe: GitCacheRestoration | null; +}; +const execute = promisify(execFile); +const unavailable = () => new Error("remote_workspace_migration_archived_git_source_unavailable"); +async function git(cwd: string, args: string[]) { + return (await execute("git", ["-c", "core.fsmonitor=false", "-c", `core.hooksPath=${os.devNull}`, "-C", cwd, ...args], { + timeout: 120_000, maxBuffer: 1024 * 1024, + env: { ...process.env, GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: os.devNull, + GIT_TERMINAL_PROMPT: "0", GIT_OPTIONAL_LOCKS: "0" }, + })).stdout.trim(); +} + +/** A removed branch must be reconstructed from its saved commit or its exact + * successful deletion receipt. Today's default branch is never a substitute. */ +export async function inspectArchivedGitMigrationSource(db: Pick, workspace: Workspace): Promise { + if (workspace.status !== "archived" || !workspace.repoUrl || !workspace.branchName || !workspace.sourceIssueId) return null; + const recipe = readGitCacheRestoration(workspace); + if (recipe) { + if (gitRestorePointExpired(recipe)) throw new Error("remote_workspace_migration_restore_point_expired"); + return { commit: recipe.commit, sourcePath: recipe.sourcePath, recipe, cleanupOperationId: null }; + } + if (workspace.providerType !== "git_worktree" || !workspace.closedAt || workspace.metadata?.gitBranchOwnershipVersion !== 1) return null; + const [operation] = await db.select().from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id), + eq(workspaceOperations.phase, "worktree_cleanup"), eq(workspaceOperations.status, "succeeded"), + or(eq(workspaceOperations.command, `git branch -d ${workspace.branchName}`), and( + sql`${workspaceOperations.metadata}->>'cleanupAction' = 'branch_delete'`, + sql`${workspaceOperations.metadata}->>'workspaceId' = ${workspace.id}`, + sql`${workspaceOperations.metadata}->>'branchName' = ${workspace.branchName}`, + sql`${workspaceOperations.metadata}->>'workspacePath' = ${workspace.providerRef ?? workspace.cwd}`, + )), gte(workspaceOperations.startedAt, workspace.closedAt), + )).orderBy(desc(workspaceOperations.startedAt), desc(workspaceOperations.id)).limit(1); + const receipt = operation?.stdoutExcerpt?.trim(), prefix = `Deleted branch ${workspace.branchName} (was `; + if (!operation || !receipt?.startsWith(prefix) || !receipt.endsWith(").")) return null; + const abbreviation = receipt.slice(prefix.length, -2); + if (!/^[a-f0-9]{7,40}$/.test(abbreviation)) throw unavailable(); + const [projectWorkspace] = workspace.projectWorkspaceId ? await db.select({ cwd: projectWorkspaces.cwd, repoUrl: projectWorkspaces.repoUrl }) + .from(projectWorkspaces).where(and(eq(projectWorkspaces.id, workspace.projectWorkspaceId), + eq(projectWorkspaces.companyId, workspace.companyId), workspace.projectId ? eq(projectWorkspaces.projectId, workspace.projectId) : undefined)) : []; + const sourcePath = projectWorkspace?.cwd ?? managedSourcePaths(workspace)[0]; + if (!sourcePath || projectWorkspace?.repoUrl && projectWorkspace.repoUrl !== workspace.repoUrl) throw unavailable(); + await verifiedGitCacheSource(workspace, sourcePath); + const commit = await git(sourcePath, ["rev-parse", "--verify", `${abbreviation}^{commit}`]); + if (!/^[a-f0-9]{40}$/.test(commit)) throw unavailable(); + return { commit, sourcePath, cleanupOperationId: operation.id, recipe: null }; +} + +/** Restore only into private migration scratch. Original repositories, task + * folders, lifecycle state and restore-point metadata remain unchanged. */ +export async function materializeArchivedGitMigrationSource(input: { + db: Db; storage: StorageService; workspace: Workspace; source: ArchivedGitMigrationSource; scratch: string; +}) { + const { workspace, source } = input, cwd = path.join(input.scratch, "retained-git"); + if (!workspace.sourceIssueId || !workspace.repoUrl || !workspace.branchName) throw unavailable(); + const resolveGitAuth = createIssueGitRemoteAuthProvider(input.db, workspace.companyId, workspace.sourceIssueId); + const present = await fs.lstat(source.sourcePath).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (present) { + const verified = await verifiedGitCacheSource(workspace, source.sourcePath); + await fs.mkdir(cwd, { mode: 0o700 }); await git(cwd, ["init", "--quiet"]); + // Fetch objects, never the current working tree of a shared project root. + const presentCommit = await git(verified, ["cat-file", "-e", `${source.commit}^{commit}`]).then(() => true, () => false); + if (presentCommit) { + await git(cwd, ["fetch", "--no-tags", "--no-recurse-submodules", "--", verified, source.commit]); + } else if (source.recipe?.restorePoint && !source.recipe.restorePoint.headOnRemote) { + // The archived branch can have been pruned from the base while its + // encrypted bundle still owns the unpushed commits. Import only that + // bundle's explicit prerequisites, then let the restore-point owner + // verify and import the saved history below. + const { manifest } = await loadGitRestorePoint({ storage: input.storage, workspace, ref: source.recipe.restorePoint }); + for (const commit of manifest.prerequisites) { + const local = await git(verified, ["cat-file", "-e", `${commit}^{commit}`]).then(() => true, () => false); + if (local) await git(cwd, ["fetch", "--no-tags", "--no-recurse-submodules", "--", verified, commit]); + else await ensureSavedGitCommit(cwd, workspace.repoUrl, commit, resolveGitAuth); + } + } + } else { + if (!source.recipe) throw unavailable(); + await materializeManagedProjectWorkspace(cwd, { repoUrl: workspace.repoUrl, resolveGitAuth }); + } + if (source.recipe?.restorePoint) await ensureGitRestorePointCommit({ storage: input.storage, workspace, + ref: source.recipe.restorePoint, repositoryCwd: cwd }); + await ensureSavedGitCommit(cwd, workspace.repoUrl, source.commit, resolveGitAuth); + await git(cwd, ["check-ref-format", `refs/heads/${workspace.branchName}`]); + await git(cwd, ["checkout", "--detach", source.commit]); + await git(cwd, ["update-ref", `refs/heads/${workspace.branchName}`, source.commit]); + await git(cwd, ["symbolic-ref", "HEAD", `refs/heads/${workspace.branchName}`]); + await git(cwd, ["config", "remote.origin.url", workspace.repoUrl]); + if (source.recipe?.restorePoint) await applyGitRestorePointToWorktree({ storage: input.storage, workspace, + ref: source.recipe.restorePoint, repositoryCwd: cwd, worktreePath: cwd }); + const approvedLocalSources = new Set(); + await restoreGitWorkspaceContents({ cwd, companyId: workspace.companyId, workspaceId: workspace.id, + projectId: workspace.projectId, contents: source.recipe?.contents, storage: input.storage, resolveGitAuth, + authorizeSource: async saved => { + if (!workspace.projectId) throw unavailable(); + const [row] = await input.db.select().from(projectWorkspaces).where(and(eq(projectWorkspaces.id, saved.id), + eq(projectWorkspaces.companyId, workspace.companyId), eq(projectWorkspaces.projectId, workspace.projectId))); + if (!row || row.repoUrl !== saved.repoUrl) throw unavailable(); + const local = localRepositoryPath(saved.repoUrl); if (local) approvedLocalSources.add(local); + }, + prepareLocalSource: async local => { + if (!approvedLocalSources.has(local)) throw unavailable(); + const identity = await fs.realpath(local); + return async () => { if (await fs.realpath(local) !== identity) throw unavailable(); }; + }, + }); + // A checkout records scratch-file inode and time fields in its index. Those + // fields must not change the saved archive identity after an upload reply is + // lost. Rebuild only these reconstructed indexes from their staged trees; + // the original repository and unstaged files remain untouched. + for (const repository of [cwd, ...(source.recipe?.contents?.checkouts ?? []) + .map(checkout => path.join(cwd, PROJECT_REPOSITORIES_DIR, checkout.name))]) { + const tree = await git(repository, ["write-tree"]); + await git(repository, ["read-tree", "--empty"]); + await git(repository, ["read-tree", tree]); + } + return cwd; +} diff --git a/server/src/services/remote-workspace-migration-tree.ts b/server/src/services/remote-workspace-migration-tree.ts new file mode 100644 index 00000000000..3c7deed9113 --- /dev/null +++ b/server/src/services/remote-workspace-migration-tree.ts @@ -0,0 +1,148 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { and, eq, inArray, ne, sql } from "drizzle-orm"; +import { executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns, workspaceOperations, type Db } from "@paperclipai/db"; +import type { StorageService } from "../storage/types.js"; +import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; +import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; + +type ReadDb = Pick; +type Workspace = typeof executionWorkspaces.$inferSelect; +export type ImportedWorkspaceChild = { + workspaceId: string; snapshotId: string; operationId: string; relativePath: string; manifestDigest: string; +}; +const overlaps = (a: string, b: string) => a === b || a.startsWith(`${b}/`) || b.startsWith(`${a}/`); +const rootOf = (workspace: Pick) => workspace.cwd ?? workspace.providerRef; + +// Missing historical paths still resolve through their nearest surviving +// parent, so a symlinked ancestor cannot hide an alias from the writer fence. +async function physicalPath(value: string): Promise { + let current = path.resolve(value); const missing: string[] = []; + for (;;) { + try { return path.join(await fs.realpath(current), ...missing.reverse()); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT" || current === path.dirname(current)) throw error; + missing.push(path.basename(current)); current = path.dirname(current); + } + } +} + +export function validateImportedWorkspaceChildren(raw: unknown): ImportedWorkspaceChild[] { + if (!Array.isArray(raw) || raw.some(child => !child || typeof child !== "object" + || typeof child.relativePath !== "string" || !child.relativePath.startsWith(".paperclip/worktrees/") + || child.relativePath.split("/").some((part: string) => !part || part === "." || part === "..") + || !/^[a-f0-9-]{36}$/.test(child.workspaceId) || !/^[a-f0-9-]{36}$/.test(child.snapshotId) + || !/^[a-f0-9-]{36}$/.test(child.operationId) || !/^[a-f0-9]{64}$/.test(child.manifestDigest))) + throw new Error("nested_workspace_coverage_invalid"); + return raw; +} + +export async function offlineWorkspaceMigrationBlockers(db: ReadDb) { + if (process.env.PAPERCLIP_EPHEMERAL_MIGRATION !== "true") return ["offline_maintenance_required"]; + const [active] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns) + .where(inArray(heartbeatRuns.status, ["running", "queued"])).limit(1); + return active ? ["active_execution_requires_drain"] : []; +} + +/** Only the maintenance entry point can acknowledge that every old deployment + * has stopped. A task-level migration hold cannot fence other path aliases. */ +export async function workspaceMigrationLayoutBlockers(db: ReadDb, workspace: Workspace, writersStopped = false) { + if (writersStopped) return offlineWorkspaceMigrationBlockers(db); + const cwd = rootOf(workspace); + if (!cwd) return []; + const local = await db.select({ id: executionWorkspaces.id, cwd: executionWorkspaces.cwd, + providerRef: executionWorkspaces.providerRef }).from(executionWorkspaces).where(and( + eq(executionWorkspaces.companyId, workspace.companyId), ne(executionWorkspaces.id, workspace.id), + inArray(executionWorkspaces.providerType, ["local_fs", "git_worktree"]), + )); + const root = await physicalPath(cwd); + for (const row of local) if (rootOf(row) && overlaps(root, await physicalPath(rootOf(row)!))) + return ["overlapping_workspace_requires_maintenance"]; + const migrated = await db.select({ cwd: workspaceOperations.cwd }).from(workspaceOperations).where(and( + eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.phase, "remote_workspace_migration"), + eq(workspaceOperations.status, "succeeded"), ne(workspaceOperations.executionWorkspaceId, workspace.id), + )); + for (const row of migrated) if (row.cwd && (await physicalPath(row.cwd)).startsWith(`${root}/.paperclip/worktrees/`)) + return ["overlapping_workspace_requires_maintenance"]; + return []; +} + +/** Runtime-created nested task folders can be omitted only after an exact + * committed import covers their current bytes. Ordinary user folders stay in + * the parent archive. The snapshot owner retains the resulting dependencies. */ +export async function prepareNestedWorkspaceImports(input: { + db: Db; storage: StorageService; workspace: Workspace; scratch: string; signal?: AbortSignal; + recordedChildren?: ImportedWorkspaceChild[]; +}): Promise { + const sourceRoot = rootOf(input.workspace); + if (!sourceRoot) return []; + if (input.recordedChildren && !input.recordedChildren.length) return []; + const root = await physicalPath(sourceRoot); + const rows = await input.db.select({ operation: workspaceOperations, snapshot: executionWorkspaceSnapshots }) + .from(workspaceOperations).innerJoin(executionWorkspaceSnapshots, and( + eq(executionWorkspaceSnapshots.companyId, workspaceOperations.companyId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, workspaceOperations.executionWorkspaceId), + eq(executionWorkspaceSnapshots.sourceOperationId, workspaceOperations.id), + )).where(and(eq(workspaceOperations.companyId, input.workspace.companyId), + ne(workspaceOperations.executionWorkspaceId, input.workspace.id), eq(workspaceOperations.phase, "remote_workspace_migration"), + eq(workspaceOperations.status, "succeeded"), inArray(executionWorkspaceSnapshots.status, ["committed", "superseded"]), + sql`${workspaceOperations.metadata}->>'snapshotId' = ${executionWorkspaceSnapshots.id}::text`, + )); + const located = []; + for (const row of rows) { + if (!row.operation.cwd || input.recordedChildren && !input.recordedChildren.some(child => child.operationId === row.operation.id)) continue; + located.push({ ...row, sourcePath: await physicalPath(row.operation.cwd) }); + } + const candidates = located.filter(row => row.sourcePath.startsWith(`${root}/.paperclip/worktrees/`)) + .sort((a, b) => a.sourcePath.length - b.sourcePath.length || a.operation.id.localeCompare(b.operation.id)); + const covered: ImportedWorkspaceChild[] = []; + for (const { operation, snapshot, sourcePath } of candidates) { + input.signal?.throwIfAborted(); + const relativePath = path.relative(root, sourcePath).split(path.sep).join("/"); + if (covered.some(item => relativePath === item.relativePath || relativePath.startsWith(`${item.relativePath}/`))) continue; + const entry = await fs.lstat(sourcePath).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (!entry) { if (input.recordedChildren) throw new Error("nested_workspace_source_changed"); continue; } + if (!entry.isDirectory() || entry.isSymbolicLink()) throw new Error("nested_workspace_source_changed"); + await verifyCoveredWorkspace({ ...input, sourcePath, snapshot, operationId: operation.id, + expectedDigest: operation.metadata?.manifestDigest, depth: 0 }); + covered.push({ workspaceId: snapshot.executionWorkspaceId, snapshotId: snapshot.id, operationId: operation.id, + relativePath, manifestDigest: snapshot.manifestDigest! }); + } + if (input.recordedChildren && !isDeepStrictEqual(covered, input.recordedChildren)) throw new Error("nested_workspace_scope_changed"); + return covered; +} + +async function verifyCoveredWorkspace(input: { + db: Db; storage: StorageService; sourcePath: string; snapshot: typeof executionWorkspaceSnapshots.$inferSelect; + operationId: string; expectedDigest: unknown; scratch: string; signal?: AbortSignal; depth: number; +}) { + input.signal?.throwIfAborted(); + if (input.depth > 32) throw new Error("nested_workspace_coverage_invalid"); + const { snapshot } = input; + const reference = await getWorkspaceSnapshot(input.db, { companyId: snapshot.companyId, + workspaceId: snapshot.executionWorkspaceId, snapshotId: snapshot.id }); + if (!reference || reference.manifestDigest !== input.expectedDigest || snapshot.sourceOperationId !== input.operationId) + throw new Error("nested_workspace_snapshot_changed"); + const children = validateImportedWorkspaceChildren(snapshot.metadata?.importedWorkspaceChildren ?? []); + const area = await fs.mkdtemp(path.join(input.scratch, "covered-")); + for (const child of children) { + const [saved] = await input.db.select({ operation: workspaceOperations, snapshot: executionWorkspaceSnapshots }) + .from(executionWorkspaceSnapshots).innerJoin(workspaceOperations, and( + eq(workspaceOperations.companyId, executionWorkspaceSnapshots.companyId), eq(workspaceOperations.id, executionWorkspaceSnapshots.sourceOperationId), + )).where(and(eq(executionWorkspaceSnapshots.companyId, snapshot.companyId), eq(executionWorkspaceSnapshots.id, child.snapshotId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, child.workspaceId), eq(workspaceOperations.id, child.operationId), + eq(workspaceOperations.status, "succeeded"), eq(workspaceOperations.phase, "remote_workspace_migration"), + inArray(executionWorkspaceSnapshots.status, ["committed", "superseded"]))); + const sourcePath = path.join(input.sourcePath, child.relativePath); + if (!saved?.operation.cwd || await physicalPath(saved.operation.cwd) !== sourcePath) throw new Error("nested_workspace_coverage_invalid"); + await verifyCoveredWorkspace({ ...input, sourcePath, snapshot: saved.snapshot, operationId: child.operationId, + expectedDigest: child.manifestDigest, scratch: area, depth: input.depth + 1 }); + } + const archivePath = path.join(area, "saved.pcws"), restored = path.join(area, "restored"); + await loadWorkspaceSnapshotArchive({ storage: input.storage, reference, targetPath: archivePath, signal: input.signal }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir: restored, expectedManifestDigest: reference.manifestDigest, signal: input.signal }); + const current = await captureRemoteWorkspaceDirectory({ cwd: input.sourcePath, archivePath: path.join(area, "current.pcws"), + exclude: children.map(child => child.relativePath), signal: input.signal }); + if (current.manifestDigest !== reference.manifestDigest) throw new Error("nested_workspace_source_changed"); +} diff --git a/server/src/services/remote-workspace-migration.test.ts b/server/src/services/remote-workspace-migration.test.ts index 0e55b224eab..bce64e9e66d 100644 --- a/server/src/services/remote-workspace-migration.test.ts +++ b/server/src/services/remote-workspace-migration.test.ts @@ -4,19 +4,29 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; +import { createServer, type Server } from "node:http"; +import { pathToFileURL } from "node:url"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { activityLog, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, +import { activityLog, agents, heartbeatRuns, applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, executionWorkspaceRuntimeLeases, executionWorkspaces, issues, projects, projectWorkspaces, workspaceOperations, workspaceRuntimeServices } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; import { createStorageService } from "../storage/service.js"; import type { StorageService } from "../storage/types.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; +import { migrateWorkspaceTrees } from "./remote-workspace-migration-batch.js"; import { inventoryRemoteWorkspaceMigration, migrateRemoteWorkspace } from "./remote-workspace-migration.js"; import { getWorkspaceSnapshot, loadWorkspaceSnapshotArchive } from "./remote-workspace-snapshot-storage.js"; import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; -import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; +import { captureTaskDirectorySnapshot, removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; +import { captureRestorePoint, sealRestorePoint } from "./git-restore-point.js"; +import type { TaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js"; +import { resolveManagedProjectWorkspaceDir } from "../home-paths.js"; +import { cleanupExecutionWorkspaceArtifacts } from "./workspace-runtime.js"; +import { workspaceOperationService } from "./workspace-operations.js"; +import { managedProjectCheckoutName } from "./managed-project-checkout-name.js"; +import { captureGitWorkspaceContents, readManagedCheckoutContents } from "./git-workspace-contents.js"; const execute = promisify(execFile); const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C", cwd, ...args], { env: { ...process.env, @@ -28,6 +38,7 @@ if (url && !/^remote_workspace_[a-z_]*test$/.test(new URL(url).pathname.slice(1) const support = url ? { supported: true } : await getEmbeddedPostgresTestSupport(); describe.skipIf(!support.supported)("remote workspace migration", () => { let db: ReturnType, root: string, storage: StorageService; + const servers: Server[] = []; let temporary: Awaited> | undefined; beforeAll(async () => { if (url) { await applyPendingMigrations(url); db = createDb(url); } @@ -37,9 +48,10 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { beforeEach(async () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "61".repeat(32)); root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-migration-test-")); storage = createStorageService(createLocalDiskStorageProvider(path.join(root, "objects"))); }); afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())))); vi.unstubAllEnvs(); await db.delete(activityLog); await db.delete(workspaceOperations); await db.delete(executionWorkspaceSnapshots); await db.delete(workspaceRuntimeServices); await db.delete(executionWorkspaceRuntimeLeases); - await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); + await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(issues); await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); await db.delete(companies); await fs.rm(root, { recursive: true, force: true }); }); async function seed() { @@ -53,6 +65,392 @@ describe.skipIf(!support.supported)("remote workspace migration", () => { return { companyId, workspaceId, issueId: issue!.id, cwd }; } async function workspace(id: string) { return (await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, id)))[0]!; } + async function nestedFixture() { + const parent = await seed(), workspaceId = randomUUID(); + const cwd = path.join(parent.cwd, ".paperclip", "worktrees", "child"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "child.txt"), "saved child edits\n"); + const [issue] = await db.insert(issues).values({ companyId: parent.companyId, title: "Nested task" }).returning(); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: parent.companyId, sourceIssueId: issue!.id, + name: "Child workspace", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd, providerRef: cwd }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, issue!.id)); + return { parent, child: { companyId: parent.companyId, workspaceId, issueId: issue!.id, cwd } }; + } + it("requires offline maintenance for overlapping physical workspace paths", async () => { + const { parent, child } = await nestedFixture(); + await expect(migrateRemoteWorkspace({ db, storage, ...child, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + expect(await fs.readFile(path.join(child.cwd, "child.txt"), "utf8")).toBe("saved child edits\n"); + }); + it("imports nested task files before their parent and retains independently restorable copies", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + await fs.rm(parent.cwd, { recursive: true }); + for (const [name, fixture] of [["parent", parent], ["child", child]] as const) { + const authority = readRemoteWorkspaceAuthority((await workspace(fixture.workspaceId)).metadata)!; + const ref = await getWorkspaceSnapshot(db, { ...fixture, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, `${name}.pcws`), targetDir = path.join(root, `restored-${name}`); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, name === "parent" ? "saved.txt" : "child.txt"), "utf8")) + .toBe(name === "parent" ? "original saved work\n" : "saved child edits\n"); + if (name === "parent") { + expect(await fs.stat(path.join(targetDir, ".paperclip", "worktrees", "child")).catch(() => null)).toBeNull(); + const [snapshot] = await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, ref!.id)); + expect(snapshot.metadata?.importedWorkspaceChildren).toEqual([expect.objectContaining({ workspaceId: child.workspaceId, + relativePath: ".paperclip/worktrees/child" })]); + } + } + }); + it("requires the offline entry point even when a caller asserts writers stopped", async () => { + const { parent } = await nestedFixture(); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("offline_maintenance_required"); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).rejects.toThrow("offline_maintenance_required"); + }); + it.each(["running", "queued"])("refuses offline archived imports while any execution is %s", async status => { + const f = await seed(); await retainDirectory(f); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const [agent] = await db.insert(agents).values({ companyId: f.companyId, name: "Unrelated writer" }).returning(); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: agent!.id, status }); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow("active_execution_requires_drain"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + }); + it("detects overlapping folders reached through symlinked parent directories", async () => { + const f = await seed(), alias = path.join(root, "alias"); await fs.symlink(root, alias); + await db.insert(executionWorkspaces).values({ companyId: f.companyId, name: "Physical alias", mode: "isolated_workspace", + strategyType: "project_primary", providerType: "local_fs", cwd: path.join(alias, "original") }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + }); + it.each(["current", "legacy"])("keeps the parent capture scope when children migrate between retries (%s)", async journal => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const put = storage.putFile.bind(storage); let fail = true; + storage.putFile = async input => { const saved = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return saved; }; + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("upload reply lost"); + if (journal === "legacy") { + const [operation] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.executionWorkspaceId, parent.workspaceId)); + const metadata = { ...operation!.metadata }; delete metadata.nestedWorkspaceScope; + await db.update(workspaceOperations).set({ metadata }).where(eq(workspaceOperations.id, operation!.id)); + } + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)!; + const reference = await getWorkspaceSnapshot(db, { ...parent, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "retried-parent.pcws"), targetDir = path.join(root, "retried-parent"); + await loadWorkspaceSnapshotArchive({ storage, reference: reference!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: reference!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, ".paperclip/worktrees/child/child.txt"), "utf8")).toBe("saved child edits\n"); + }); + it("blocks distinct legacy records that alias the same folder", async () => { + const f = await seed(), id = randomUUID(); + await db.insert(executionWorkspaces).values({ id, companyId: f.companyId, name: "Historical alias", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd: f.cwd }); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("overlapping_workspace_requires_maintenance"); + }); + it.each(["cleanup_failed", "adapter_managed"])("blocks enrollment instead of abandoning unsupported local files (%s)", async kind => { + const f = await seed(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await db.update(executionWorkspaces).set(kind === "cleanup_failed" ? { status: kind } : { providerType: kind }) + .where(eq(executionWorkspaces.id, f.workspaceId)); + await expect(migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })) + .rejects.toThrow(kind === "cleanup_failed" ? "workspace_not_active" : "workspace_provider_unsupported"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + it.each(["changed", "corrupt", "late-change", "abort"])("keeps the parent local when nested coverage is unsafe (%s)", async mode => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata)!; + const reference = await getWorkspaceSnapshot(db, { ...child, snapshotId: authority.snapshotId! }); + const controller = new AbortController(); + if (mode === "changed") await fs.writeFile(path.join(child.cwd, "child.txt"), "changed child bytes"); + if (mode === "corrupt") await storage.deleteObject(child.companyId, reference!.objectKey); + if (mode === "late-change" || mode === "abort") { + const put = storage.putFile.bind(storage); let once = true; + storage.putFile = async input => { const result = await put(input); if (once) { once = false; + if (mode === "abort") controller.abort(); else await fs.writeFile(path.join(child.cwd, "child.txt"), "changed during upload"); + } return result; }; + } + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true, signal: controller.signal })).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(parent.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + if (mode === "abort") { + await migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true }); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)?.snapshotId).toBeTruthy(); + } + }); + it("resumes an interrupted maintenance batch in leaf-first order without duplicating imports", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const seen: string[] = [], controller = new AbortController(); + await expect(migrateWorkspaceTrees(db, { storage, signal: controller.signal, phase: name => { + seen.push(name); if (name === `workspace:${parent.workspaceId}`) controller.abort(); + } })).rejects.toThrow(); + expect(seen).toEqual([`workspace:${child.workspaceId}`, `workspace:${parent.workspaceId}`]); + const childAuthority = readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata); + expect(await migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).toBe(1); + expect(await migrateWorkspaceTrees(db, { storage, signal: new AbortController().signal, phase: () => {} })).toBe(0); + expect(readRemoteWorkspaceAuthority((await workspace(child.workspaceId)).metadata)).toEqual(childAuthority); + }); + it("checks transitive nested coverage before omitting a child tree", async () => { + const { parent, child } = await nestedFixture(); vi.stubEnv("PAPERCLIP_EPHEMERAL_MIGRATION", "true"); + const id = randomUUID(), cwd = path.join(child.cwd, ".paperclip", "worktrees", "grandchild"); + await fs.mkdir(cwd, { recursive: true }); await fs.writeFile(path.join(cwd, "deep.txt"), "deep work"); + const [issue] = await db.insert(issues).values({ companyId: parent.companyId, title: "Grandchild task" }).returning(); + await db.insert(executionWorkspaces).values({ id, companyId: parent.companyId, sourceIssueId: issue!.id, + name: "Grandchild", mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", cwd }); + await migrateRemoteWorkspace({ db, storage, companyId: parent.companyId, workspaceId: id, apply: true, writersStopped: true }); + await migrateRemoteWorkspace({ db, storage, ...child, apply: true, writersStopped: true }); + await fs.writeFile(path.join(cwd, "deep.txt"), "new deep work"); + await expect(migrateRemoteWorkspace({ db, storage, ...parent, apply: true, writersStopped: true })).rejects.toThrow("nested_workspace_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(parent.workspaceId)).metadata)).toBeNull(); + }); + async function gitArchiveFixture(repoUrl = "https://example.invalid/repository.git") { + const f = await seed(), projectId = randomUUID(), projectWorkspaceId = randomUUID(), branchName = "saved-task"; + vi.stubEnv("PAPERCLIP_HOME", root); vi.stubEnv("PAPERCLIP_INSTANCE_ID", "default"); + const base = resolveManagedProjectWorkspaceDir({ companyId: f.companyId, projectId, repoName: "repository" }); + await fs.mkdir(base, { recursive: true }); await git(base, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(base, "base.txt"), "base\n"); await git(base, "add", "."); await git(base, "commit", "-qm", "base"); + await git(base, "remote", "add", "origin", repoUrl); const upstream = await git(base, "rev-parse", "HEAD"); + const cwd = path.join(base, ".paperclip", "worktrees", branchName); + await git(base, "worktree", "add", "-b", branchName, cwd); + await fs.copyFile(path.join(f.cwd, "saved.txt"), path.join(cwd, "saved.txt")); await fs.rm(f.cwd, { recursive: true }); + await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "saved task commit"); const commit = await git(cwd, "rev-parse", "HEAD"); + await db.insert(projects).values({ id: projectId, companyId: f.companyId, name: "Archived project" }); + await db.insert(projectWorkspaces).values({ id: projectWorkspaceId, companyId: f.companyId, projectId, name: "Source", cwd: base, repoUrl }); + await db.update(executionWorkspaces).set({ cwd, providerRef: cwd, projectId, projectWorkspaceId, repoUrl, baseRef: "main", + branchName, strategyType: "git_worktree", providerType: "git_worktree", metadata: { createdByRuntime: true, gitBranchOwnershipVersion: 1 } }).where(eq(executionWorkspaces.id, f.workspaceId)); + return { ...f, cwd, base, upstream, commit, projectId, projectWorkspaceId, repoUrl, branchName }; + } + it.each(["legacy", "verified", "retry"])("recovers an archived Git workspace from its exact branch-deletion receipt (%s)", async receiptKind => { + const f = await gitArchiveFixture(), closedAt = new Date(); + await db.update(executionWorkspaces).set({ status: "archived", closedAt }).where(eq(executionWorkspaces.id, f.workspaceId)); + if (receiptKind !== "verified") { + await git(f.base, "merge", "--ff-only", f.branchName); + await git(f.base, "worktree", "remove", f.cwd); + const receipt = await git(f.base, "branch", "-d", f.branchName); + await db.insert(workspaceOperations).values({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, + phase: "worktree_cleanup", status: "succeeded", command: `git branch -d ${f.branchName}`, + stdoutExcerpt: receipt, startedAt: new Date(), finishedAt: new Date() }); + } else { + const cleaned = await cleanupExecutionWorkspaceArtifacts({ workspace: await workspace(f.workspaceId), + projectWorkspace: { cwd: f.base, cleanupCommand: null }, expectedBranchHeadSha: f.commit, runCleanupCommands: false, + recorder: workspaceOperationService(db).createRecorder({ companyId: f.companyId, executionWorkspaceId: f.workspaceId, issueId: f.issueId }) }); + expect(cleaned.warnings).toEqual([]); + } + await fs.writeFile(path.join(f.base, "base.txt"), "newer default branch\n"); + await git(f.base, "add", "base.txt"); await git(f.base, "commit", "-qm", "new default branch work"); + if (receiptKind === "retry") { + const put = storage.putFile.bind(storage); let fail = true; + storage.putFile = async input => { const result = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return result; }; + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("upload reply lost"); + } + await migrateRemoteWorkspace({ db, storage, ...f, apply: true }); + const row = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(row.metadata)!; + expect(row.status).toBe("archived"); expect(authority.source).toMatchObject({ kind: "git", ref: f.commit }); + await fs.rm(f.base, { recursive: true }); + const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "legacy-git.pcws"), targetDir = path.join(root, "restored-legacy-git"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await git(targetDir, "rev-parse", "HEAD")).toBe(f.commit); + expect(await fs.readFile(path.join(targetDir, "base.txt"), "utf8")).toBe("base\n"); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + async function serveRepository() { + const directory = path.join(root, "http"), bare = path.join(directory, "repository.git"); + await fs.mkdir(bare, { recursive: true }); await git(bare, "init", "--bare", "--initial-branch=main"); + const server = createServer(async (request, response) => { + const relative = new URL(request.url!, "http://localhost").pathname.slice(1); + const target = path.resolve(directory, relative); + if (!target.startsWith(`${directory}/`)) { response.writeHead(403).end(); return; } + try { response.end(await fs.readFile(target)); } catch { response.writeHead(404).end(); } + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); servers.push(server); + const address = server.address(); if (!address || typeof address === "string") throw new Error("test server not bound"); + return { bare, repoUrl: `http://127.0.0.1:${address.port}/repository.git` }; + } + it.each(["present", "pruned", "retry", "missing-base", "nested", "unapproved-nested", "missing", "expired", "corrupt"])("preserves archived Git edits or refuses an unavailable restore point (%s)", async state => { + const remote = state === "missing-base" ? await serveRepository() : null; + const f = await gitArchiveFixture(remote?.repoUrl); + if (remote) { await git(remote.bare, "fetch", f.base, "main:refs/heads/main"); await git(remote.bare, "update-server-info"); } + await fs.writeFile(path.join(f.cwd, "saved.txt"), "staged edit\n"); await git(f.cwd, "add", "saved.txt"); + await fs.writeFile(path.join(f.cwd, "saved.txt"), "unstaged edit\n"); + const captured = (await captureRestorePoint({ cwd: f.cwd, upstreamCommits: [f.upstream] }))!; + const id = randomUUID(), generation = 1, sealed = await sealRestorePoint(captured.payload, + { companyId: f.companyId, workspaceId: f.workspaceId, restorePointId: id, generation }); + const objectKey = `${f.companyId}/git-restore-points/${f.workspaceId}/${generation}/${id}.pcrp`; + await storage.putFile({ companyId: f.companyId, namespace: "git-restore-points", objectKey, + originalFilename: "git.pcrp", contentType: "application/octet-stream", body: sealed.ciphertext }); + const restorePoint = { schema: "paperclip.git-restore-point.v1", id, generation, objectKey, byteSize: sealed.ciphertext.length, + sha256: sealed.sha256, key: sealed.key, head: captured.manifest.head, index: captured.manifest.index, + worktree: captured.manifest.worktree, ignored: captured.manifest.ignored, modes: captured.manifest.modes, headOnRemote: false }; + let contents; + if (state === "nested" || state === "unapproved-nested") { + const sourceCwd = path.join(root, "nested-source"); + await fs.mkdir(sourceCwd); await git(sourceCwd, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(sourceCwd, "nested.txt"), "nested source work\n"); + await git(sourceCwd, "add", "."); await git(sourceCwd, "commit", "-qm", "nested source"); + const source = { id: randomUUID(), repoUrl: pathToFileURL(sourceCwd).href, repoRef: "main", cwd: sourceCwd }; + const name = managedProjectCheckoutName(source), nestedCwd = path.join(f.cwd, ".paperclip-repositories", name); + await git(f.cwd, "clone", source.repoUrl, nestedCwd); await git(nestedCwd, "checkout", "--detach"); + await fs.mkdir(path.join(nestedCwd, ".claude")); + await fs.writeFile(path.join(nestedCwd, ".claude", "settings.local.json"), '{"permissions":{"allow":["Read"]}}'); + contents = await captureGitWorkspaceContents(f.cwd, { companyId: f.companyId, workspaceId: f.workspaceId }, + [await readManagedCheckoutContents(nestedCwd, name, source)]); + if (state === "nested") await db.insert(projectWorkspaces).values({ ...source, companyId: f.companyId, projectId: f.projectId, name: "Nested source" }); + } + await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date(), metadata: { + createdByRuntime: true, gitBranchOwnershipVersion: 1, gitCacheRestoration: { schema: "paperclip.git-cache-restoration.v2", companyId: f.companyId, + workspaceId: f.workspaceId, sourceIssueId: f.issueId, repoUrl: f.repoUrl, commit: f.commit, branchName: f.branchName, + sourceKind: "managed_project", projectId: f.projectId, sourcePath: f.base, worktreePath: f.cwd, + gitBranchOwnershipVersion: 1, verifiedAt: new Date().toISOString(), restorePoint, contents }, + } }).where(eq(executionWorkspaces.id, f.workspaceId)); + await git(f.base, "worktree", "remove", "--force", f.cwd); await git(f.base, "branch", "-D", f.branchName); + if (state === "missing-base") await fs.rm(f.base, { recursive: true }); + if (state === "unapproved-nested") { + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect((await workspace(f.workspaceId)).status).toBe("archived"); + return; + } + if (state === "pruned") { + await git(f.base, "reflog", "expire", "--expire=now", "--all"); + await git(f.base, "gc", "--prune=now"); + await expect(git(f.base, "cat-file", "-e", `${f.commit}^{commit}`)).rejects.toThrow(); + } + if (["missing", "expired", "corrupt"].includes(state)) { + if (state === "missing") await storage.deleteObject(f.companyId, objectKey); + else { + const row = await workspace(f.workspaceId); + const recipe = row.metadata!.gitCacheRestoration as Record; + await db.update(executionWorkspaces).set({ metadata: { ...row.metadata, gitCacheRestoration: { ...recipe, + restorePoint: { ...restorePoint, ...(state === "expired" ? { expiredAt: new Date().toISOString() } : { sha256: "00".repeat(32) }) }, + } } }).where(eq(executionWorkspaces.id, f.workspaceId)); + } + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + const row = await workspace(f.workspaceId); + expect(row).toMatchObject({ status: "archived", providerType: "git_worktree", cwd: f.cwd }); + expect(readRemoteWorkspaceAuthority(row.metadata)).toBeNull(); + expect(await fs.stat(f.cwd).catch(() => null)).toBeNull(); + return; + } + if (state === "retry") { + const put = storage.putFile.bind(storage); let fail = true; + storage.putFile = async input => { const result = await put(input); if (fail) { fail = false; throw new Error("upload reply lost"); } return result; }; + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("upload reply lost"); + } + await migrateRemoteWorkspace({ db, storage, ...f, apply: true }); + const authority = readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)!; + await fs.rm(f.base, { recursive: true, force: true }); await fs.rm(path.join(root, "nested-source"), { recursive: true, force: true }); await storage.deleteObject(f.companyId, objectKey); + const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "retained-git.pcws"), targetDir = path.join(root, "restored-retained-git"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await git(targetDir, "rev-parse", "HEAD")).toBe(f.commit); + if (state === "nested") { + expect(await fs.readFile(path.join(targetDir, ".paperclip-repositories", contents!.checkouts[0]!.name, "nested.txt"), "utf8")).toBe("nested source work\n"); + expect(JSON.parse(await fs.readFile(path.join(targetDir, ".paperclip-repositories", contents!.checkouts[0]!.name, ".claude/settings.local.json"), "utf8"))) + .toEqual({ permissions: { allow: ["Read"] } }); + } + expect(await git(targetDir, "show", ":saved.txt")).toBe("staged edit"); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("unstaged edit\n"); + }); + async function retainDirectory(f: Awaited>) { + const captured = await captureTaskDirectorySnapshot(f.cwd), id = randomUUID(), generation = 1; + const sealed = await sealRestorePoint(captured.payload, { companyId: f.companyId, workspaceId: f.workspaceId, restorePointId: id, generation }); + const objectKey = `${f.companyId}/git-restore-points/${f.workspaceId}/${generation}/task-directory/${id}.pcrp`; + await storage.putFile({ companyId: f.companyId, namespace: "git-restore-points", objectKey, + originalFilename: "retained.pcrp", contentType: "application/octet-stream", body: sealed.ciphertext }); + const ref: TaskDirectoryRestorePointRef = { schema: "paperclip.task-directory-restore-point.v1", id, generation, + objectKey, sha256: sealed.sha256, byteSize: sealed.ciphertext.length, key: sealed.key, + digest: captured.digest, files: captured.files, excluded: captured.excluded }; + await db.update(executionWorkspaces).set({ status: "archived", closedAt: new Date(), metadata: { + isolatedTaskDirectoryRetention: { schema: "paperclip.task-directory-retention.v1", generation, removed: true, + trash: path.join(root, "removed-trash"), restorePoint: ref }, + } }).where(eq(executionWorkspaces.id, f.workspaceId)); + await fs.rm(f.cwd, { recursive: true }); + return ref; + } + it("imports an archived directory from its encrypted restore point after the old folder is gone", async () => { + const f = await seed(), retained = await retainDirectory(f); + expect(await inventoryRemoteWorkspaceMigration(db, f)).toMatchObject({ eligible: true }); + await migrateRemoteWorkspace({ db, storage, ...f, apply: true }); + const saved = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(saved.metadata)!; + expect(saved.status).toBe("archived"); + expect(await fs.stat(f.cwd).catch(() => null)).toBeNull(); + await storage.deleteObject(f.companyId, retained.objectKey); + const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + const archivePath = path.join(root, "retained.pcws"), targetDir = path.join(root, "restored-retained"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + it.each(["missing", "expired", "corrupt"])("does not replace an unavailable retained directory with an empty workspace (%s)", async failure => { + const f = await seed(), retained = await retainDirectory(f); + if (failure === "missing") await storage.deleteObject(f.companyId, retained.objectKey); + else { + const row = await workspace(f.workspaceId); + const retention = row.metadata!.isolatedTaskDirectoryRetention as Record; + await db.update(executionWorkspaces).set({ metadata: { ...row.metadata, isolatedTaskDirectoryRetention: { + ...retention, restorePoint: { ...retained, ...(failure === "expired" ? { expiredAt: new Date().toISOString() } : { sha256: "00".repeat(32) }) }, + } } }).where(eq(executionWorkspaces.id, f.workspaceId)); + } + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow(); + const row = await workspace(f.workspaceId); + expect(row).toMatchObject({ status: "archived", providerType: "local_fs", cwd: f.cwd }); + expect(readRemoteWorkspaceAuthority(row.metadata)).toBeNull(); + expect(await fs.stat(f.cwd).catch(() => null)).toBeNull(); + }); + it("keeps newly restored local files when they appear during an archived import", async () => { + const f = await seed(); await retainDirectory(f); + const put = storage.putFile.bind(storage); + storage.putFile = async input => { + const result = await put(input); + await fs.mkdir(f.cwd); await fs.writeFile(path.join(f.cwd, "saved.txt"), "new local edits\n"); + return result; + }; + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("remote_workspace_migration_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("new local edits\n"); + }); + it("preserves an archived workspace without reopening its task or losing its files", async () => { + const f = await seed(), closedAt = new Date("2026-10-20T12:00:00Z"); + await db.update(executionWorkspaces).set({ status: "archived", closedAt }).where(eq(executionWorkspaces.id, f.workspaceId)); + await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).resolves.toMatchObject({ applied: true }); + const saved = await workspace(f.workspaceId), authority = readRemoteWorkspaceAuthority(saved.metadata)!; + expect(saved.status).toBe("archived"); expect(saved.closedAt).toEqual(closedAt); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(issue).toMatchObject({ status: "done", executionWorkspaceId: f.workspaceId }); + const ref = await getWorkspaceSnapshot(db, { ...f, snapshotId: authority.snapshotId! }); + await fs.rm(f.cwd, { recursive: true }); + const archivePath = path.join(root, "archived.pcws"), targetDir = path.join(root, "restored-archived"); + await loadWorkspaceSnapshotArchive({ storage, reference: ref!, targetPath: archivePath }); + await restoreRemoteWorkspaceArchive({ archivePath, targetDir, expectedManifestDigest: ref!.manifestDigest }); + expect(await fs.readFile(path.join(targetDir, "saved.txt"), "utf8")).toBe("original saved work\n"); + }); + it("imports a historical workspace without replacing the task's current workspace", async () => { + const f = await seed(), currentId = randomUUID(), currentCwd = path.join(root, "current"); + await fs.mkdir(currentCwd); await fs.writeFile(path.join(currentCwd, "saved.txt"), "newer current work\n"); + await db.insert(executionWorkspaces).values({ id: currentId, companyId: f.companyId, sourceIssueId: f.issueId, + mode: "isolated_workspace", strategyType: "project_primary", providerType: "local_fs", name: "Current workspace", cwd: currentCwd }); + await db.update(issues).set({ executionWorkspaceId: currentId }).where(eq(issues.id, f.issueId)); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).resolves.toMatchObject({ applied: true }); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(issue?.executionWorkspaceId).toBe(currentId); + expect(await fs.readFile(path.join(currentCwd, "saved.txt"), "utf8")).toBe("newer current work\n"); + expect(readRemoteWorkspaceAuthority((await workspace(currentId)).metadata)).toBeNull(); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)?.snapshotId).toBeTruthy(); + }); + it("refuses publication when the task changes its workspace during import", async () => { + const f = await seed(), put = storage.putFile.bind(storage); + storage.putFile = async input => { + const result = await put(input); + await db.update(issues).set({ executionWorkspaceId: null }).where(eq(issues.id, f.issueId)); + return result; + }; + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("remote_workspace_migration_source_changed"); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toBeNull(); + expect(await fs.readFile(path.join(f.cwd, "saved.txt"), "utf8")).toBe("original saved work\n"); + await expect(migrateRemoteWorkspace({ db, storage, ...f, apply: true })).rejects.toThrow("remote_workspace_migration_source_changed"); + }); it("defaults to dry run and preserves both database authority and local files", async () => { const f = await seed(); expect(await migrateRemoteWorkspace({ db, storage, ...f })).toMatchObject({ eligible: true, applied: false, alreadyMigrated: false }); expect((await workspace(f.workspaceId)).providerType).toBe("local_fs"); diff --git a/server/src/services/remote-workspace-migration.ts b/server/src/services/remote-workspace-migration.ts index b26dcfcf1b7..91cdc407c22 100644 --- a/server/src/services/remote-workspace-migration.ts +++ b/server/src/services/remote-workspace-migration.ts @@ -14,7 +14,10 @@ import { withExecutionWorkspaceLifecycleLock } from "./execution-workspace-lifec import { captureRemoteWorkspaceDirectory, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { beginWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot } from "./remote-workspace-snapshot-storage.js"; import { notifyRemoteWorkspaceSnapshots } from "./remote-workspace-snapshot-retention.js"; -import { removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; +import { materializeTaskDirectorySnapshot, removeTaskDirectoryTree } from "./isolated-task-directory-snapshot.js"; +import { loadTaskDirectoryRestorePoint, mayBeIsolatedTaskDirectoryWorkspace, readTaskDirectoryRetention, validateTaskDirectoryRestorePointRef } from "./isolated-task-directory-retention.js"; +import { inspectArchivedGitMigrationSource, materializeArchivedGitMigrationSource, type ArchivedGitMigrationSource } from "./remote-workspace-migration-git.js"; +import { prepareNestedWorkspaceImports, validateImportedWorkspaceChildren, workspaceMigrationLayoutBlockers } from "./remote-workspace-migration-tree.js"; const SCHEMA = "paperclip.remote-workspace-migration.v1"; type Workspace = typeof executionWorkspaces.$inferSelect; @@ -32,29 +35,55 @@ function fingerprint(workspace: Workspace) { providerType: workspace.providerType, cwd: workspace.cwd, providerRef: workspace.providerRef, repoUrl: workspace.repoUrl, baseRef: workspace.baseRef, branchName: workspace.branchName, status: workspace.status, closedAt: workspace.closedAt?.toISOString(), metadata })).digest("hex"); } +const gitSourceProof = (source: ArchivedGitMigrationSource | null) => source ? { + commit: source.commit, sourcePath: source.sourcePath, cleanupOperationId: source.cleanupOperationId, +} : null; -async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string }) { +async function inspect(db: ReadDb, input: { companyId: string; workspaceId: string; operationId?: string; writersStopped?: boolean }) { const [workspace] = await db.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))); if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); const authority = readRemoteWorkspaceAuthority(workspace.metadata); const marker = object(workspace.metadata?.remoteWorkspaceMigration); const allowedOperation = input.operationId ?? (marker.schema === SCHEMA && typeof marker.operationId === "string" ? marker.operationId : undefined); const reasons: string[] = []; - if (authority) return { workspace, reasons, authority, marker, source: authority.source }; + if (authority) return { workspace, reasons, authority, marker, source: authority.source, issueBinding: null, retainedDirectory: null, archivedGitSource: null }; if (!["local_fs", "git_worktree"].includes(workspace.providerType)) reasons.push("workspace_provider_unsupported"); - if (!["active", "idle", "in_review"].includes(workspace.status)) reasons.push("workspace_not_active"); + if (!["active", "idle", "in_review", "archived"].includes(workspace.status)) reasons.push("workspace_not_active"); if (!workspace.sourceIssueId) reasons.push("task_binding_required"); const nestedMigration = workspace.metadata?.nestedCloneMigrationPending; if (nestedMigration != null && (typeof nestedMigration !== "object" || Array.isArray(nestedMigration) || Object.keys(nestedMigration).length > 0)) reasons.push("nested_clone_migration_unresolved"); const cwd = workspace.cwd ?? workspace.providerRef; + let retainedDirectory = null as ReturnType; + let archivedGitSource: ArchivedGitMigrationSource | null = null; if (!cwd || !path.isAbsolute(cwd)) reasons.push("local_source_missing"); else { - const entry = await fs.lstat(cwd).catch(() => null); - if (!entry?.isDirectory() || entry.isSymbolicLink()) reasons.push("local_source_unsafe_or_missing"); + const entry = await fs.lstat(cwd).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (!entry && workspace.status === "archived" && mayBeIsolatedTaskDirectoryWorkspace(workspace)) { + const retained = readTaskDirectoryRetention(workspace); + if (retained?.removed === true && retained.restorePoint && retained.generation === retained.restorePoint.generation) { + if (retained.restorePoint.expiredAt) reasons.push("restore_point_expired"); + else { + try { validateTaskDirectoryRestorePointRef(retained.restorePoint, workspace); retainedDirectory = retained; } + catch { reasons.push("restore_point_invalid"); } + } + } + } + if (!entry && workspace.status === "archived" && !retainedDirectory) { + try { archivedGitSource = await inspectArchivedGitMigrationSource(db, workspace); } + catch (error) { reasons.push(error instanceof Error && error.message === "remote_workspace_migration_restore_point_expired" + ? "restore_point_expired" : "archived_git_source_unavailable"); } + } + if (!retainedDirectory && !archivedGitSource && (!entry?.isDirectory() || entry.isSymbolicLink())) reasons.push("local_source_unsafe_or_missing"); + if (entry?.isDirectory() && !entry.isSymbolicLink()) reasons.push(...await workspaceMigrationLayoutBlockers(db, workspace, input.writersStopped)); } const [issue] = workspace.sourceIssueId ? await db.select().from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))) : []; - if (workspace.sourceIssueId && (!issue || issue.executionWorkspaceId !== workspace.id)) reasons.push("task_binding_changed"); + // Historical records still own their saved files after the task selects a + // different workspace. Importing them must not reopen or rebind that task. + if (workspace.sourceIssueId && !issue) reasons.push("task_binding_changed"); const [project] = workspace.projectId ? await db.select({ policy: projects.executionWorkspacePolicy }).from(projects) .where(and(eq(projects.companyId, workspace.companyId), eq(projects.id, workspace.projectId))) : []; const issuePolicy = object(issue?.executionWorkspaceSettings), projectPolicy = object(project?.policy); @@ -109,13 +138,13 @@ async function inspect(db: ReadDb, input: { companyId: string; workspaceId: stri eq(executionWorkspaceRepositories.executionWorkspaceId, workspace.id))); if (repositories.some(repository => repository.state !== "ready")) reasons.push("repository_preparation_unresolved"); const source: RemoteWorkspaceSource = workspace.repoUrl ? { kind: "git", repoUrl: workspace.repoUrl, - ref: workspace.baseRef, branchName: workspace.branchName } : { kind: "empty" }; + ref: archivedGitSource?.commit ?? workspace.baseRef, branchName: workspace.branchName } : { kind: "empty" }; if (repositories.length) source.repositories = repositories.map(repository => ({ id: repository.id, relativePath: repository.relativePath, repoUrl: repository.repoUrl, ref: repository.requestedRef, ...(repository.pinnedCommit ? { pinnedCommit: repository.pinnedCommit } : {}) })); - return { workspace, reasons: [...new Set(reasons)], authority, marker, source }; + return { workspace, reasons: [...new Set(reasons)], authority, marker, source, issueBinding: issue?.executionWorkspaceId ?? null, retainedDirectory, archivedGitSource }; } -export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string }) { +export async function inventoryRemoteWorkspaceMigration(db: Db, input: { companyId: string; workspaceId: string; writersStopped?: boolean }) { const result = await inspect(db, input); return { companyId: input.companyId, workspaceId: input.workspaceId, eligible: result.reasons.length === 0, alreadyMigrated: Boolean(result.authority), reasons: result.reasons, localPath: result.workspace.cwd ?? result.workspace.providerRef, @@ -126,6 +155,7 @@ export async function inventoryRemoteWorkspaceMigration(db: Db, input: { company /** Administrative import. The CLI enforces external deployment prerequisites. */ export async function migrateRemoteWorkspace(input: { db: Db; storage: StorageService; companyId: string; workspaceId: string; apply?: boolean; source?: RemoteWorkspaceSource; + writersStopped?: boolean; signal?: AbortSignal; }) { if (!input.apply) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: false }; return withExecutionWorkspaceLifecycleLock(input.db, input.workspaceId, async lock => { @@ -134,7 +164,7 @@ export async function migrateRemoteWorkspace(input: { if (!workspace) throw new Error("remote_workspace_migration_workspace_missing"); if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, workspace.companyId), eq(issues.id, workspace.sourceIssueId))).for("update"); const current = await inspect(tx as unknown as Db, input); - if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "" }; + if (current.authority) return { workspace, authority: current.authority, operation: null, source: current.source, sourceFingerprint: "", issueBinding: current.issueBinding, retainedDirectory: null, archivedGitSource: null }; if (current.reasons.length) throw new Error(`remote_workspace_migration_blocked:${current.reasons.join(",")}`); const source = remoteWorkspaceSourceSchema.parse(input.source ?? current.source); const sourceFingerprint = fingerprint(workspace); @@ -144,33 +174,75 @@ export async function migrateRemoteWorkspace(input: { const [operation] = await tx.select().from(workspaceOperations).where(and(eq(workspaceOperations.companyId, workspace.companyId), eq(workspaceOperations.executionWorkspaceId, workspace.id), eq(workspaceOperations.id, marker.operationId), eq(workspaceOperations.phase, "remote_workspace_migration"))); if (!operation) throw new Error("remote_workspace_migration_operation_missing"); - return { workspace, authority: null, operation, source, sourceFingerprint }; + // Older operations only admitted the task's current workspace. Preserve + // that binding when resuming an import journaled before this field existed. + const issueBinding = Object.hasOwn(operation.metadata ?? {}, "sourceIssueBinding") + ? operation.metadata!.sourceIssueBinding : workspace.id; + if (issueBinding !== current.issueBinding) throw new Error("remote_workspace_migration_source_changed"); + if (Boolean(operation.metadata?.retainedDirectory) !== Boolean(current.retainedDirectory)) throw new Error("remote_workspace_migration_source_changed"); + if (canonical(operation.metadata?.archivedGitSource ?? null) !== canonical(gitSourceProof(current.archivedGitSource))) throw new Error("remote_workspace_migration_source_changed"); + return { workspace, authority: null, operation, source, sourceFingerprint, issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource }; } const [operation] = await tx.insert(workspaceOperations).values({ companyId: workspace.companyId, executionWorkspaceId: workspace.id, issueId: workspace.sourceIssueId, cwd: workspace.cwd ?? workspace.providerRef, phase: "remote_workspace_migration", status: "running", - metadata: { schema: SCHEMA, sourceFingerprint, source } }).returning(); + metadata: { schema: SCHEMA, sourceFingerprint, source, sourceIssueBinding: current.issueBinding, + nestedWorkspaceScope: { version: 1, state: "pending" }, + ...(current.archivedGitSource ? { archivedGitSource: gitSourceProof(current.archivedGitSource) } : {}), + ...(current.retainedDirectory ? { retainedDirectory: { restorePointId: current.retainedDirectory.restorePoint.id, + generation: current.retainedDirectory.generation, digest: current.retainedDirectory.restorePoint.digest } } : {}) } }).returning(); await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`); await tx.update(executionWorkspaces).set({ metadata: { ...workspace.metadata, remoteWorkspaceMigration: { schema: SCHEMA, operationId: operation!.id, state: "running", sourceFingerprint, source, } }, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)); - return { workspace, authority: null, operation: operation!, source, sourceFingerprint }; + return { workspace, authority: null, operation: operation!, source, sourceFingerprint, issueBinding: current.issueBinding, retainedDirectory: current.retainedDirectory, archivedGitSource: current.archivedGitSource }; }); if (reserved.authority) return { ...await inventoryRemoteWorkspaceMigration(input.db, input), applied: true, replayed: true }; - const operation = reserved.operation!; + let operation = reserved.operation!; const snapshot = await beginWorkspaceSnapshot(input.db, { companyId: input.companyId, workspaceId: input.workspaceId, operationId: operation.id, generation: 1, epoch: 1 }); const scratch = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-migration-")); try { + input.signal?.throwIfAborted(); + const retained = reserved.retainedDirectory?.restorePoint; + const sourceRoot = retained ? await materializeTaskDirectorySnapshot({ + payload: await loadTaskDirectoryRestorePoint({ storage: input.storage, workspace: reserved.workspace, ref: retained }), + target: path.join(scratch, "retained-source"), expectedDigest: retained.digest, + }) : reserved.archivedGitSource ? await materializeArchivedGitMigrationSource({ db: input.db, storage: input.storage, + workspace: reserved.workspace, source: reserved.archivedGitSource, scratch }) : reserved.workspace.cwd ?? reserved.workspace.providerRef!; + const scope = object(operation.metadata?.nestedWorkspaceScope); + if (operation.metadata?.nestedWorkspaceScope !== undefined && (scope.version !== 1 || !["pending", "fixed"].includes(String(scope.state)))) + throw new Error("nested_workspace_scope_invalid"); + // Journals predating scoped captures included every child. Preserve that + // archive identity even if a later batch has already imported the leaves. + const recordedChildren = scope.state === "fixed" ? validateImportedWorkspaceChildren(scope.children) + : scope.state === "pending" ? undefined : []; + const nested = input.writersStopped && !retained && !reserved.archivedGitSource + ? await prepareNestedWorkspaceImports({ ...input, workspace: reserved.workspace, scratch, recordedChildren }) : []; + if (scope.state === "pending") { + const [saved] = await input.db.update(workspaceOperations).set({ metadata: { ...operation.metadata, + nestedWorkspaceScope: { version: 1, state: "fixed", children: nested } }, updatedAt: new Date() }) + .where(and(eq(workspaceOperations.companyId, input.companyId), eq(workspaceOperations.id, operation.id), + eq(workspaceOperations.status, "running"))).returning(); + if (!saved) throw new Error("remote_workspace_migration_operation_changed"); + operation = saved; + } + const exclude = nested.map(child => child.relativePath); const archivePath = path.join(scratch, "workspace.pcws"); - const captured = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath }); + const captured = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath, exclude, signal: input.signal }); const ref = await uploadWorkspaceSnapshot({ db: input.db, storage: input.storage, snapshotId: snapshot.id, - companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest }); + companyId: input.companyId, workspaceId: input.workspaceId, archivePath, manifestDigest: captured.manifestDigest, signal: input.signal }); const restoredArchive = path.join(scratch, "verified.pcws"), restored = path.join(scratch, "restored"); - await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive }); - await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest }); + await loadWorkspaceSnapshotArchive({ storage: input.storage, reference: ref, targetPath: restoredArchive, signal: input.signal }); + await restoreRemoteWorkspaceArchive({ archivePath: restoredArchive, targetDir: restored, expectedManifestDigest: ref.manifestDigest, signal: input.signal }); const currentArchive = path.join(scratch, "current.pcws"); - const current = await captureRemoteWorkspaceDirectory({ cwd: reserved.workspace.cwd ?? reserved.workspace.providerRef!, archivePath: currentArchive }); + const current = await captureRemoteWorkspaceDirectory({ cwd: sourceRoot, archivePath: currentArchive, exclude, signal: input.signal }); if (current.manifestDigest !== captured.manifestDigest) throw new Error("remote_workspace_migration_source_changed"); + if (nested.length) { + const verifyScratch = path.join(scratch, "verify-nested"); await fs.mkdir(verifyScratch); + const verified = await prepareNestedWorkspaceImports({ ...input, workspace: reserved.workspace, scratch: verifyScratch, recordedChildren: nested }); + if (canonical(verified) !== canonical(nested)) throw new Error("nested_workspace_source_changed"); + } + input.signal?.throwIfAborted(); await lock.execute(sql`select 1`); const publication = await input.db.transaction(async tx => { const [workspace] = await tx.select().from(executionWorkspaces).where(and(eq(executionWorkspaces.companyId, input.companyId), eq(executionWorkspaces.id, input.workspaceId))).for("update"); @@ -181,15 +253,28 @@ export async function migrateRemoteWorkspace(input: { || object(workspace.metadata?.remoteWorkspaceMigration).operationId !== operation.id) throw new Error("remote_workspace_migration_publication_changed"); if (workspace.sourceIssueId) await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, workspace.sourceIssueId)).for("update"); const validation = await inspect(tx as unknown as Db, { ...input, operationId: operation.id }); + if (validation.issueBinding !== reserved.issueBinding) throw new Error("remote_workspace_migration_source_changed"); + if (Boolean(validation.retainedDirectory) !== Boolean(reserved.retainedDirectory)) throw new Error("remote_workspace_migration_source_changed"); + if (canonical(gitSourceProof(validation.archivedGitSource)) !== canonical(gitSourceProof(reserved.archivedGitSource))) throw new Error("remote_workspace_migration_source_changed"); if (validation.reasons.length) throw new Error(`remote_workspace_migration_blocked:${validation.reasons.join(",")}`); const [uploaded] = await tx.select().from(executionWorkspaceSnapshots).where(and(eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.executionWorkspaceId, workspace.id), eq(executionWorkspaceSnapshots.id, ref.id))).for("update"); if (!uploaded || uploaded.status !== "uploaded" || uploaded.sourceOperationId !== operation.id) throw new Error("remote_workspace_migration_snapshot_changed"); + for (const child of [...nested].sort((a, b) => a.snapshotId.localeCompare(b.snapshotId))) { + const [saved] = await tx.select().from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, workspace.companyId), eq(executionWorkspaceSnapshots.id, child.snapshotId), + eq(executionWorkspaceSnapshots.executionWorkspaceId, child.workspaceId), + )).for("update"); + if (!saved || !["committed", "superseded"].includes(saved.status) || saved.manifestDigest !== child.manifestDigest + || saved.sourceOperationId !== child.operationId) throw new Error("nested_workspace_snapshot_changed"); + } const authority: RemoteWorkspaceAuthority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: ref.id, source: reserved.source, location: null, writerRunId: null, dirty: false }; const metadata: Record = { ...workspace.metadata, remoteWorkspace: authority }; delete metadata.remoteWorkspaceMigration; await notifyRemoteWorkspaceSnapshots(tx, `workspace:${workspace.id}`); - await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id)); + await tx.update(executionWorkspaceSnapshots).set({ status: "committed", deleteAfter: null, + metadata: { ...uploaded.metadata, ...(nested.length ? { importedWorkspaceChildren: nested } : {}) }, + updatedAt: new Date() }).where(eq(executionWorkspaceSnapshots.id, ref.id)); await tx.update(executionWorkspaces).set({ providerType: "cloud_sandbox", providerRef: null, cwd: `/workspaces/${workspace.id}`, metadata, updatedAt: new Date() }).where(eq(executionWorkspaces.id, workspace.id)); await tx.update(workspaceOperations).set({ status: "succeeded", finishedAt: new Date(), updatedAt: new Date(), diff --git a/server/src/services/remote-workspace-snapshot-retention.test.ts b/server/src/services/remote-workspace-snapshot-retention.test.ts index d008d393d38..079d5a0206f 100644 --- a/server/src/services/remote-workspace-snapshot-retention.test.ts +++ b/server/src/services/remote-workspace-snapshot-retention.test.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { createServer } from "node:http"; import type { AddressInfo } from "node:net"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { applyPendingMigrations, closeRegisteredClients, companies, createDb, executionWorkspaceSnapshots, executionWorkspaces, issueRecoveryActions, issues, workspaceOperations } from "@paperclipai/db"; import { createLocalDiskStorageProvider } from "../storage/local-disk-provider.js"; @@ -131,6 +131,41 @@ describe.skipIf(!support.supported)("remote workspace snapshot retention", () => for (const key of f.keys) expect((await storage.headObject(f.companyId, key)).exists).toBe(false); expect(await owner.nextDeadlineAt()).toBeNull(); }); + it("uses the dependency index when a company has many unrelated snapshots", async () => { + const child = await seed(); + await db.execute(sql`insert into execution_workspace_snapshots + (company_id, execution_workspace_id, source_operation_id, generation, epoch, status) + select ${child.companyId}::uuid, gen_random_uuid(), gen_random_uuid(), 1, 1, 'committed' + from generate_series(1, 2500)`); + await db.insert(executionWorkspaceSnapshots).values({ companyId: child.companyId, executionWorkspaceId: randomUUID(), + sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed", + metadata: { importedWorkspaceChildren: [{ workspaceId: child.workspaceId, snapshotId: child.row.id }] } }); + await db.execute(sql`analyze execution_workspace_snapshots`); + const plan = await db.execute(sql`explain (format json) select id from execution_workspace_snapshots + where company_id = ${child.companyId}::uuid and status <> 'deleted' + and metadata ? 'importedWorkspaceChildren' + and metadata->'importedWorkspaceChildren' @> ${JSON.stringify([{ workspaceId: child.workspaceId, snapshotId: child.row.id }])}::jsonb + limit 1`); + expect(JSON.stringify(plan)).toContain("execution_workspace_snapshots_imported_children_idx"); + }); + it("pins imported child bytes until the covering parent snapshot is deleted", async () => { + const child = await seed(), parentId = randomUUID(), workspaceId = randomUUID(); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: child.companyId, name: "Imported parent", mode: "isolated_workspace", strategyType: "project_primary", + metadata: { remoteWorkspace: { snapshotId: parentId } } }); + await db.insert(executionWorkspaceSnapshots).values({ id: parentId, companyId: child.companyId, executionWorkspaceId: workspaceId, + sourceOperationId: randomUUID(), generation: 1, epoch: 1, status: "committed", + metadata: { importedWorkspaceChildren: [{ workspaceId: child.workspaceId, snapshotId: child.row.id }] } }); + const owner = service(); + expect(await owner.tick()).toMatchObject({ pinned: 1, deleted: 0 }); + expect(await owner.nextDeadlineAt()).toBeNull(); + expect((await storage.headObject(child.companyId, child.keys[0]!)).exists).toBe(true); + await db.update(executionWorkspaces).set({ metadata: {} }).where(eq(executionWorkspaces.id, workspaceId)); + await db.update(executionWorkspaceSnapshots).set({ status: "superseded", deleteAfter: new Date(now.getTime() - 1) }) + .where(eq(executionWorkspaceSnapshots.id, parentId)); + await owner.tick(); await owner.tick(); + expect((await storage.headObject(child.companyId, child.keys[0]!)).exists).toBe(false); + expect(await owner.nextDeadlineAt()).toBeNull(); + }); it("purges exact-key S3 versions and delete markers before settling retention", async () => { const s3 = await versionedS3Fixture(); try { diff --git a/server/src/services/remote-workspace-snapshot-retention.ts b/server/src/services/remote-workspace-snapshot-retention.ts index 22977698304..747339eff06 100644 --- a/server/src/services/remote-workspace-snapshot-retention.ts +++ b/server/src/services/remote-workspace-snapshot-retention.ts @@ -23,6 +23,14 @@ async function pinned(db: ReadDb, row: Snapshot) { eq(executionWorkspaces.companyId, row.companyId), eq(executionWorkspaces.id, row.executionWorkspaceId), )); if ((workspace?.metadata?.remoteWorkspace as { snapshotId?: unknown } | undefined)?.snapshotId === row.id) return true; + const [importedBy] = await db.select({ id: executionWorkspaceSnapshots.id }).from(executionWorkspaceSnapshots).where(and( + eq(executionWorkspaceSnapshots.companyId, row.companyId), ne(executionWorkspaceSnapshots.status, "deleted"), + sql`${executionWorkspaceSnapshots.metadata} ? 'importedWorkspaceChildren'`, + sql`${executionWorkspaceSnapshots.metadata}->'importedWorkspaceChildren' @> ${JSON.stringify([ + { workspaceId: row.executionWorkspaceId, snapshotId: row.id }, + ])}::jsonb`, + )).limit(1); + if (importedBy) return true; const [operation] = await db.select({ id: workspaceOperations.id }).from(workspaceOperations).where(and( eq(workspaceOperations.companyId, row.companyId), eq(workspaceOperations.executionWorkspaceId, row.executionWorkspaceId), or(inArray(workspaceOperations.status, ["running", "pending"]), @@ -138,7 +146,8 @@ export function remoteWorkspaceSnapshotRetentionService(db: Db, options: { .where(and(eq(executionWorkspaceSnapshots.id, claimed.id), eq(executionWorkspaceSnapshots.status, "delete_requested"))); // A parent inspected earlier in this page can now lose its last pin. // Revisit it once instead of waiting for an unrelated execution event. - if (claimed.parentSnapshotId) rescanRequested = true; + if (claimed.parentSnapshotId || Array.isArray(claimed.metadata?.importedWorkspaceChildren) + && claimed.metadata.importedWorkspaceChildren.length > 0) rescanRequested = true; result.deleted++; } catch (error) { if (signal?.aborted) throw error; diff --git a/server/src/services/remote-workspace-transport.test.ts b/server/src/services/remote-workspace-transport.test.ts index 4b1d2f79987..88b13a85991 100644 --- a/server/src/services/remote-workspace-transport.test.ts +++ b/server/src/services/remote-workspace-transport.test.ts @@ -308,7 +308,7 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); execute.mockResolvedValueOnce({ ...failure, timedOut: true, stderr: gitOperation }); await expect(captureRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) - .rejects.toThrow(/^remote_workspace_operation_failed$/); + .rejects.toThrow(/^remote_workspace_operation_timed_out: archive_helper \(configured timeout 300000ms\)$/); }); it.each([ @@ -347,6 +347,35 @@ describe("remote workspace transport safety with real sandbox commands", () => { .rejects.toThrow(/^remote_workspace_operation_failed$/); }); + it("identifies a timed-out preparation without exposing transport output or retrying it", async () => { + const { remote, execute } = target(); + const resolveGitAuth = vi.fn(async () => null); + execute.mockResolvedValueOnce({ exitCode: 0, stdout: '{"cloneRequired":true}', stderr: "", + timedOut: false, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + execute.mockResolvedValueOnce({ exitCode: null, stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR", + timedOut: true, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + + await expect(prepareRemoteWorkspaceDirectory({ target: remote, remoteCwd: path.join(workspace, "clone"), + source: { kind: "git", repoUrl: "https://example.invalid/source.git", ref: null }, + preparationIdentity: "a".repeat(64), resolveGitAuth })) + .rejects.toThrow(/^remote_workspace_operation_timed_out: prepare \(configured timeout 300000ms\)$/); + expect(execute).toHaveBeenCalledTimes(2); + expect(resolveGitAuth).toHaveBeenCalledTimes(1); + expect(await fs.readdir(workspace)).toEqual([]); + }); + + it("reports the cleanup deadline without treating a timeout as a completed operation", async () => { + await fs.writeFile(path.join(workspace, "saved.txt"), "preserve the workspace\n"); + const { remote, execute } = target(); + execute.mockResolvedValueOnce({ exitCode: 0, stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR", + timedOut: true, signal: null, pid: process.pid, startedAt: new Date().toISOString() }); + + await expect(cleanupRemoteWorkspaceArchive({ target: remote, remoteCwd: workspace, operationId: randomUUID() })) + .rejects.toThrow(/^remote_workspace_operation_timed_out: cleanup \(configured timeout 10000ms\)$/); + expect(execute).toHaveBeenCalledTimes(1); + expect(await fs.readFile(path.join(workspace, "saved.txt"), "utf8")).toBe("preserve the workspace\n"); + }); + it("never follows a preexisting snapshot receipt temporary symlink", async () => { await fs.writeFile(path.join(workspace, "result.txt"), "synthetic saved result\n"); const operationId = randomUUID(); diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 8355f2e4111..794a6d500f6 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -27,12 +27,23 @@ export const REMOTE_WORKSPACE_SANDBOX_DISK_FULL = // Node reports ENOSPC or EDQUOT. Git and tar child processes report the text. const DISK_FULL_CHECK = String.raw`const diskFull=e=>['ENOSPC','EDQUOT'].includes(e?.code)||/No space left on device|Disk quota exceeded/i.test(String(e?.stderr??'')+String(e?.message??''));`; -async function command(target: RemoteWorkspaceTarget, args: string[], env?: Record, timeoutMs = 300_000) { +type RemoteWorkspaceOperation = "prepare" | "repository" | "archive_helper" | "capture" | "restore" | "cleanup"; + +export class RemoteWorkspaceOperationTimeoutError extends Error { + constructor(operation: RemoteWorkspaceOperation, timeoutMs: number) { + super(`remote_workspace_operation_timed_out: ${operation} (configured timeout ${timeoutMs}ms)`); + this.name = "RemoteWorkspaceOperationTimeoutError"; + } +} + +async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, + args: string[], env?: Record, timeoutMs = 300_000) { const result = await target.runner!.execute({ command: "node", args, cwd: target.remoteCwd, env: { NODE_OPTIONS: "", NODE_PATH: "", ...env }, timeoutMs, bypassSession: true }); - if (result.exitCode !== 0 || result.timedOut) { + if (result.timedOut) throw new RemoteWorkspaceOperationTimeoutError(operation, timeoutMs); + if (result.exitCode !== 0) { const gitOperation = "remote_workspace_archive_git_operation_in_progress: finish or abort the Git operation before saving the workspace"; - const reported = result.timedOut ? "" : result.stderr.trim(); + const reported = result.stderr.trim(); throw new Error([gitOperation, REMOTE_WORKSPACE_SANDBOX_DISK_FULL].includes(reported) ? reported : "remote_workspace_operation_failed"); } return result.stdout; @@ -95,14 +106,14 @@ export async function prepareRemoteWorkspaceDirectory(input: { preparationIdentity: string; }) { if (input.source.kind === "git") { - const prior = JSON.parse(await command(input.target, ["-e", PREPARE, input.remoteCwd, + const prior = JSON.parse(await command(input.target, "prepare", ["-e", PREPARE, input.remoteCwd, JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity, authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; cloneRequired?: boolean }; if (prior.cwd === input.remoteCwd) return; if (prior.cloneRequired !== true) throw new Error("remote_workspace_root_changed"); } const auth = input.source.kind === "git" ? await input.resolveGitAuth(input.source.repoUrl) : null; - const output = await command(input.target, ["-e", PREPARE, input.remoteCwd, + const output = await command(input.target, "prepare", ["-e", PREPARE, input.remoteCwd, JSON.stringify({ ...input.source, preparationIdentity: input.preparationIdentity, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env); const result = JSON.parse(output) as { cwd: string }; @@ -150,7 +161,7 @@ export async function prepareRemoteRepository(input: { const branch=await git(['symbolic-ref','--short','-q','HEAD']).catch(()=>null);process.stdout.write(JSON.stringify({cwd:root,pinnedCommit:input.pinnedCommit||head,branchName:branch})); })().catch(()=>{process.stderr.write('remote_workspace_repository_failed');process.exitCode=1});`; const payload = { ...input, target: undefined, resolveGitAuth: undefined }; - const prior = JSON.parse(await command(input.target, ["-e", script, input.remoteCwd, + const prior = JSON.parse(await command(input.target, "repository", ["-e", script, input.remoteCwd, JSON.stringify({ ...payload, authArgs: [], authEnvKeys: [], probeOnly: true })])) as { cwd?: string; pinnedCommit?: string; branchName?: string | null; cloneRequired?: boolean; }; @@ -159,7 +170,7 @@ export async function prepareRemoteRepository(input: { return { cwd: prior.cwd, pinnedCommit: prior.pinnedCommit!, branchName: prior.branchName ?? null }; } const auth = await input.resolveGitAuth(input.repoUrl); - const output = await command(input.target, ["-e", script, input.remoteCwd, + const output = await command(input.target, "repository", ["-e", script, input.remoteCwd, JSON.stringify({ ...payload, authArgs: auth?.configArgs ?? [], authEnvKeys: Object.keys(auth?.env ?? {}) })], auth?.env); const value = JSON.parse(output) as { cwd: string; pinnedCommit: string; branchName: string | null }; if (value.cwd !== input.remoteCwd || !/^[a-f0-9]{40,64}$/.test(value.pinnedCommit)) throw new Error("remote_workspace_repository_receipt_invalid"); @@ -180,7 +191,7 @@ async function installArchiveHelper(target: RemoteWorkspaceTarget, remoteCwd: st try{await fs.writeFile(temp,Buffer.from(body,'base64'),{mode:0o600,flag:'wx'});await fs.rename(temp,file);} catch(e){if(e?.code!=='EEXIST')await fs.rm(temp,{force:true}).catch(()=>{});throw e;} })().catch(error=>{if(diskFull(error))process.stderr.write(${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)});process.exitCode=1});`; - await command(target, ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); + await command(target, "archive_helper", ["-e", install, remoteCwd, directory, helper, Buffer.from(remoteWorkspaceArchiveScript()).toString("base64")]); return { directory, helper, archive: path.posix.join(directory, "workspace.pcws"), receipt: path.posix.join(directory, "receipt.json") }; } @@ -197,7 +208,7 @@ export async function cleanupRemoteWorkspaceArchive(input: { if(!entry.isDirectory()||entry.isSymbolicLink())throw Error('unsafe');} await fs.rm(dir,{recursive:true,force:true}); })().catch(()=>{process.stderr.write('remote_workspace_export_cleanup_failed');process.exitCode=1});`; - await command(input.target, ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000); + await command(input.target, "cleanup", ["-e", script, input.remoteCwd, input.operationId], undefined, 10_000); } export async function captureRemoteWorkspaceArchive(input: { @@ -234,7 +245,7 @@ export async function captureRemoteWorkspaceArchive(input: { process.stderr.write(typeof error?.stderr==='string'&&error.stderr.trim()===gitOperation?gitOperation :diskFull(error)?${JSON.stringify(REMOTE_WORKSPACE_SANDBOX_DISK_FULL)}:'remote_workspace_capture_failed');process.exitCode=1; });`; - const { layout, ...receipt } = JSON.parse(await command(input.target, ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { + const { layout, ...receipt } = JSON.parse(await command(input.target, "capture", ["-e", capture, staged.helper, input.remoteCwd, staged.directory])) as { manifestDigest: string; sha256: string; byteSize: number; entries: number; layout: "legacy" | "captured"; }; if (!["legacy", "captured"].includes(layout) || !/^[a-f0-9]{64}$/.test(receipt.manifestDigest) || !/^[a-f0-9]{64}$/.test(receipt.sha256) @@ -284,7 +295,7 @@ export async function restoreRemoteWorkspaceDirectory(input: { await fs.writeFile(path.join(runtime,'workspace-restored.json'),JSON.stringify({identity,manifestDigest:digest}),{flag:'wx',mode:0o600}); if(await fs.realpath(path.dirname(root))!==path.dirname(root))throw Error('unsafe_root');await fs.rename(staged,root); })().catch(()=>{process.stderr.write('remote_workspace_restore_failed');process.exitCode=1});`; - await command(input.target, ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]); + await command(input.target, "restore", ["-e", restore, staged.helper, staged.archive, input.remoteCwd, input.manifestDigest, input.restorationIdentity]); // Publish the restored location before cleanup can fail, stall, or lose its // reply. The owner journals this receipt and then reclaims disposable staging. return { operationId, stagingRoot: parent, cleanupPending: true }; diff --git a/server/src/services/remote-workspaces.integration.test.ts b/server/src/services/remote-workspaces.integration.test.ts index 077cb768037..e1c6e2d93c3 100644 --- a/server/src/services/remote-workspaces.integration.test.ts +++ b/server/src/services/remote-workspaces.integration.test.ts @@ -919,17 +919,28 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem expect(await preserveRemoteWorkspacePreparationFailure(db, prepared.lease)).toBeNull(); }, 60_000); - it("records and retries export cleanup after a lost reply without another upload", async () => { + it.each([ + ["a lost reply", "remote_workspace_export_cleanup_failed"], + ["a timeout", "remote_workspace_operation_timed_out: cleanup (configured timeout 10000ms)"], + ["an untrusted error", "remote_workspace_export_cleanup_failed"], + ] as const)("records and retries export cleanup after %s without another upload", async (failure, diagnostic) => { const reserved = await reserve({ kind: "empty" }), first = await target("cleanup-sandbox"); const prepared = await prepareRemoteWorkspace({ db, storage, workspace: reserved, runId: ids.run, lease: await lease(), target: first }); await fs.writeFile(path.join(prepared.remoteRoot, "saved.txt"), "keep working files\n"); const sandbox = first as Extract; const execute = sandbox.runner!.execute.bind(sandbox.runner!); - let lost = false; + let lost = false, cleanupAttempts = 0; sandbox.runner!.execute = async input => { const result = await execute(input); - if (!lost && input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) { - lost = true; throw new Error("lost cleanup reply"); + if (input.args?.[1]?.includes("remote_workspace_export_cleanup_failed") && result.exitCode === 0) { + cleanupAttempts++; + if (lost) return result; + lost = true; + if (failure === "a timeout") return { ...result, timedOut: true, + stdout: "SYNTHETIC_PRIVATE_STDOUT", stderr: "SYNTHETIC_PRIVATE_STDERR" }; + throw new Error(failure === "an untrusted error" + ? "remote_workspace_operation_timed_out: cleanup (configured timeout 10000ms)\nSYNTHETIC_PRIVATE_STDERR" + : "lost cleanup reply"); } return result; }; @@ -940,11 +951,13 @@ suite("remote workspace lifecycle with real Postgres and independent filesystem const [before] = await db.select().from(workspaceOperations).where(and(eq(workspaceOperations.heartbeatRunId, ids.run), eq(workspaceOperations.phase, "remote_workspace_snapshot"), eq(workspaceOperations.status, "succeeded"), eq(workspaceOperations.id, (await db.select().from(executionWorkspaceSnapshots).where(eq(executionWorkspaceSnapshots.id, saved.snapshotId)))[0]!.sourceOperationId))); - expect(before?.metadata?.exportCleanup).toMatchObject({ state: "pending", lastError: "remote_workspace_export_cleanup_failed" }); + expect(before?.metadata?.exportCleanup).toMatchObject({ state: "pending", lastError: diagnostic }); + expect(cleanupAttempts).toBe(1); expect(await saveRemoteWorkspace(input)).toMatchObject({ snapshotId: saved.snapshotId, generation: saved.generation }); expect(upload).toHaveBeenCalledOnce(); const [after] = await db.select().from(workspaceOperations).where(eq(workspaceOperations.id, before!.id)); expect(after?.metadata?.exportCleanup).toMatchObject({ state: "cleaned", lastError: null }); + expect(cleanupAttempts).toBe(2); expect(await fs.readFile(path.join(prepared.remoteRoot, "saved.txt"), "utf8")).toBe("keep working files\n"); }, 60_000); diff --git a/server/src/services/remote-workspaces.ts b/server/src/services/remote-workspaces.ts index bb5849d2b55..231896c716c 100644 --- a/server/src/services/remote-workspaces.ts +++ b/server/src/services/remote-workspaces.ts @@ -25,7 +25,7 @@ import { canonicalExecutionUuid } from "./execution-identity.js"; import { persistActivity } from "./activity-log.js"; import { captureRemoteWorkspaceArchive, cleanupRemoteWorkspaceArchive, prepareRemoteWorkspaceDirectory, prepareRemoteRepository, remoteWorkspaceRepositoryPath, - requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, + RemoteWorkspaceOperationTimeoutError, requireRemoteWorkspaceTarget, restoreRemoteWorkspaceDirectory, } from "./remote-workspace-transport.js"; import { beginWorkspaceSnapshot, getWorkspaceSnapshot, loadWorkspaceSnapshotArchive, uploadWorkspaceSnapshot, @@ -281,16 +281,19 @@ async function cleanupSavedExports(input: { db: Db; companyId: string; workspace for (const row of rows) { await input.assertOwned(); let cleaned = false; + let lastError: string | null = null; try { await cleanupRemoteWorkspaceArchive({ target: input.target, remoteCwd: row.phase === "remote_workspace_restore" ? path.posix.dirname(input.location.remoteCwd) : input.location.remoteCwd, operationId: row.id }); cleaned = true; - } catch { /* Keep a durable retry obligation; the acknowledged snapshot is safe. */ } + } catch (error) { + lastError = error instanceof RemoteWorkspaceOperationTimeoutError ? error.message : "remote_workspace_export_cleanup_failed"; + } await input.assertOwned(); await input.db.update(workspaceOperations).set({ metadata: sql`jsonb_set(coalesce(${workspaceOperations.metadata}, '{}'::jsonb), '{exportCleanup}', coalesce(${workspaceOperations.metadata}->'exportCleanup', '{}'::jsonb) || - ${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError: cleaned ? null : "remote_workspace_export_cleanup_failed" })}::jsonb)`, + ${JSON.stringify({ state: cleaned ? "cleaned" : "pending", lastError })}::jsonb)`, updatedAt: new Date() }) .where(and(eq(workspaceOperations.id, row.id), eq(workspaceOperations.companyId, input.companyId), inArray(workspaceOperations.status, ["succeeded", "failed"]))); diff --git a/server/src/services/scoped-deadline-work.ts b/server/src/services/scoped-deadline-work.ts index cdd69a6801e..6ce95086ad0 100644 --- a/server/src/services/scoped-deadline-work.ts +++ b/server/src/services/scoped-deadline-work.ts @@ -18,6 +18,8 @@ export function registerScopedDeadlineWork( decodeScope(hint: string): Scope | undefined; matches(entry: Entry, scope?: Scope): boolean; load(scope?: Scope, signal?: AbortSignal): Promise; + // Owners may retain pending queue priority while refreshing eligibility. + mergeEntry?(pending: Entry, loaded: Entry): Entry; check(entry: Entry, signal?: AbortSignal): Promise; }, canRun: () => boolean, @@ -90,7 +92,8 @@ export function registerScopedDeadlineWork( const loaded = await options.load(scope, signal); const result = Array.isArray(loaded) ? { rows: loaded, complete: true } : loaded; for (const entry of result.rows ?? []) { - entries.set(entry.id, entry); + const pending = entries.get(entry.id); + entries.set(entry.id, pending && options.mergeEntry ? options.mergeEntry(pending, entry) : entry); for (const [other, progress] of scans) if (options.matches(entry, options.decodeScope(other))) progress.seen.add(entry.id); } if (!result.complete) { dirty.add(hint); continue; } diff --git a/tests/e2e/legacy-failure-continuation.spec.ts b/tests/e2e/legacy-failure-continuation.spec.ts index 46ce639e5b5..2fc7d49508d 100644 --- a/tests/e2e/legacy-failure-continuation.spec.ts +++ b/tests/e2e/legacy-failure-continuation.spec.ts @@ -40,6 +40,10 @@ for (const action of ["task_retry", "retry_after_reload", "inbox_retry", "messag } })); if (action === "automatic_message_on_reactivation") { await json(await request.patch(`/api/companies/${company.id}`, { data: { status: "paused" } })); + // Pause cleanup cancels pending wakeups. Finish that operation before + // seeding the historical saved input that reactivation must recover. + await expect.poll(async () => (await json(await request.get(`/api/agents/${agent.id}`))).lifecycleState, + { timeout: 45_000 }).toBe("paused"); } const sourceRunId = randomUUID(); // Seed the historical incident, then exercise ordinary recovery entry points. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index bf76e7c8c3f..f5c240c0df4 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -758,6 +758,9 @@ describe("runner E2E catalog", () => { 'summary:"PAPERCLIP_E2E_PLAN_DONE_nonce"', ); expect(task!.buildPrompt("nonce")).toContain("first call get_task_context"); + expect(task!.buildPrompt("nonce")).toContain("criteria:CURRENT_TURN_CRITERIA"); + expect(task!.buildPrompt("nonce")).toContain("entry for every current criterion ID"); + expect(task!.buildPrompt("nonce")).not.toContain('criterionId:"objective"'); expect(task!.buildPrompt("nonce")).toContain( "identifies the exact revised Plan revision used as the confirmation target as accepted", ); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 4100e707f9d..81e7e4fdf85 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -685,7 +685,7 @@ export const runnerTasks: readonly RunnerTaskFixture[] = [ "Only after the revised plan is accepted, implement it by posting one final visible task-thread response containing exactly " + `PAPERCLIP_E2E_PLAN_DONE_${nonce}` + " and mark the task Done.", - `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. For both the initial Plan and the revised Plan, those two tool calls form one indivisible response sequence: immediately after write_document succeeds, request_human_input must be your next action using that call's returned latestRevisionId. Do not emit assistant text, end the response or heartbeat, or stop after write_document alone before the matching confirmation request succeeds. Do not call paperclip_finish while waiting for either Plan confirmation. When an acceptance wake arrives, first call get_task_context. Treat the wake as valid only when that control-plane result is for the current task and identifies the exact revised Plan revision used as the confirmation target as accepted; otherwise do not finish and continue waiting for the matching revision-bound confirmation. After that verification succeeds, your immediate next action must be the paperclip_finish tool call. Do not call list_documents or any other tool, and do not emit any assistant text, acknowledgement, progress note, or preamble between verification and paperclip_finish. Use the current turn completion revision and criterion IDs (CURRENT_TURN_REVISION below is a placeholder, not a literal). Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:[{criterionId:"objective",status:"satisfied",evidenceRefs:[]}],remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit only PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, + `For a native runner, remain in the requested planning collaboration mode. Call write_document for key \`plan\`, then call request_human_input exactly once with interactionKind \`confirmation\`, targetRevisionId set to the returned latest Plan revision, and continuationPolicy \`wake_assignee\`. For both the initial Plan and the revised Plan, those two tool calls form one indivisible response sequence: immediately after write_document succeeds, request_human_input must be your next action using that call's returned latestRevisionId. Do not emit assistant text, end the response or heartbeat, or stop after write_document alone before the matching confirmation request succeeds. Do not call paperclip_finish while waiting for either Plan confirmation. When an acceptance wake arrives, first call get_task_context. Treat the wake as valid only when that control-plane result is for the current task and identifies the exact revised Plan revision used as the confirmation target as accepted; otherwise do not finish and continue waiting for the matching revision-bound confirmation. After that verification succeeds, your immediate next action must be the paperclip_finish tool call. Do not call list_documents or any other tool, and do not emit any assistant text, acknowledgement, progress note, or preamble between verification and paperclip_finish. Use the current turn completion revision and criterion IDs. CURRENT_TURN_REVISION and CURRENT_TURN_CRITERIA below are placeholders, not literals. CURRENT_TURN_CRITERIA must contain one {criterionId:,status:"satisfied",evidenceRefs:[]} entry for every current criterion ID. Call paperclip_finish exactly once with {reportedWorkDisposition:"done",summary:"PAPERCLIP_E2E_PLAN_DONE_${nonce}",completionClaim:{contractRevision:CURRENT_TURN_REVISION,objectiveSatisfied:true,criteria:CURRENT_TURN_CRITERIA,remainingWork:[]},evidence:[],verification:[]}. Wait for that tool call to succeed, then emit only PAPERCLIP_E2E_PLAN_DONE_${nonce} as the complete final response. Do not write a user-facing final response before paperclip_finish succeeds, and do not call another tool.`, `For a legacy runner, use the public Paperclip API. The first PUT of the \`plan\` issue document creates it. For every later PUT, first GET the current document and set \`baseRevisionId\` to its \`latestRevisionId\`; a 409 means you must GET again and retry with the new latest revision. Create a \`request_confirmation\` targeting the successful PUT response's \`latestRevisionId\` with \`continuationPolicy: wake_assignee\`, and move the issue to \`in_review\` while waiting. After the revised Plan is accepted, write PAPERCLIP_E2E_PLAN_DONE_${nonce} exactly once through one atomic issue PATCH with status \`done\` and that exact comment; do not POST a separate comment or perform a second write.`, "Do not create files, child tasks, or unrelated work, and do not expose credentials.", ].join("\n"), diff --git a/tests/runner-e2e/cursor-cloud-replay-fault.test.ts b/tests/runner-e2e/cursor-cloud-replay-fault.test.ts index a2e34c1771a..4b29286b59f 100644 --- a/tests/runner-e2e/cursor-cloud-replay-fault.test.ts +++ b/tests/runner-e2e/cursor-cloud-replay-fault.test.ts @@ -212,7 +212,7 @@ describe("Cursor Cloud replay bundle treatment", () => { it("the actual driver reconnects the real client with its saved cursor and deduplicates the replayed frame without creating another prompt", async () => { const directory = await root(), events: PrpEvent[] = [], proofs: CursorCloudReplayFaultProof[] = []; const nativeRun = "run-33333333-3333-4333-8333-333333333333", time = "2026-10-28T00:00:00Z"; - let nativeAgent = "", creates = 0, streams = 0; + let nativeAgent = "", creates = 0, streams = 0, cancelled = false; const requests: Array<{ path: string; headers: Headers }> = []; const peer: typeof fetch = async (resource, init) => { const path = new URL(String(resource)).pathname, headers = new Headers(init?.headers); requests.push({ path, headers }); @@ -226,7 +226,8 @@ it("the actual driver reconnects the real client with its saved cursor and dedup const withheld = 'event: assistant\nid: withheld\ndata: {"text":"must be replayed"}\n\n'; return bytes([Buffer.from(first + withheld + (streams === 1 ? "" : `event: assistant\nid: new-cursor\ndata: {"text":"Recovered."}\n\nevent: result\nid: terminal\ndata: {"runId":"${nativeRun}","status":"FINISHED"}\n\n`))]); } - if (path.endsWith(nativeRun)) return Response.json({ id: nativeRun, agentId: nativeAgent, status: streams < 2 ? "RUNNING" : "FINISHED", createdAt: time, updatedAt: time }); + if (path.endsWith("/cancel") && init?.method === "POST") { cancelled = true; return Response.json({ id: nativeRun }); } + if (path.endsWith(nativeRun)) return Response.json({ id: nativeRun, agentId: nativeAgent, status: cancelled ? "CANCELLED" : streams < 2 ? "RUNNING" : "FINISHED", createdAt: time, updatedAt: time }); throw new Error("Unexpected controlled request"); }; const client = new CursorCloudClient({ apiKey: "private-auth-canary", fetch: createCursorCloudReplayFaultFetch({ fetch: peer, digest, record: value => proofs.push(value) }) }); diff --git a/tests/runner-e2e/cursor-cloud-replay-fault.ts b/tests/runner-e2e/cursor-cloud-replay-fault.ts index 29257ab3bab..ada7b5b4bc4 100644 --- a/tests/runner-e2e/cursor-cloud-replay-fault.ts +++ b/tests/runner-e2e/cursor-cloud-replay-fault.ts @@ -101,7 +101,7 @@ export function createCursorCloudReplayFaultFetch(input: { if (cutNextRead) { proof.cutCount += 1; save(); - controller.error(new Error("Injected Cursor Cloud SSE disconnect")); + controller.error(new TypeError("Injected Cursor Cloud SSE disconnect")); void reader.cancel().catch(() => {}); return; } diff --git a/ui/src/adapters/codex-local/config-fields.tsx b/ui/src/adapters/codex-local/config-fields.tsx index 148caa904d9..9e161eb72d6 100644 --- a/ui/src/adapters/codex-local/config-fields.tsx +++ b/ui/src/adapters/codex-local/config-fields.tsx @@ -302,7 +302,7 @@ export function CodexLocalConfigFields({ )} {runnerManaged && runnerProvider === "grok_bot" && } {runnerManaged && isCursorCloud && <> -

Experimental. Runs in Cursor Cloud and requires a local Paperclip computer. Cursor manages command permissions; effective model identity is unavailable. Agent environment settings support only CURSOR_API_KEY. Configure hosted secrets in your Cursor Cloud environment.

+

Experimental. Cursor hosts this agent’s execution environment. A Paperclip computer is not required. Cursor manages command permissions; effective model identity is unavailable. Agent environment settings support only CURSOR_API_KEY. Configure hosted secrets in your Cursor Cloud environment.

{[ { key: "repoUrl", label: "GitHub repository", fallback: "", legacy: undefined, placeholder: "https://github.com/your-org/repo", hint: "Select a repository or a named Cursor cloud environment." }, { key: "repoRef", label: "Starting ref", fallback: config.repoStartingRef ?? "", legacy: "repoStartingRef", placeholder: "Repository default" }, diff --git a/ui/src/components/AgentConfigForm.render.test.tsx b/ui/src/components/AgentConfigForm.render.test.tsx index 7cba9d2a5d6..7877501077d 100644 --- a/ui/src/components/AgentConfigForm.render.test.tsx +++ b/ui/src/components/AgentConfigForm.render.test.tsx @@ -1324,6 +1324,49 @@ describe("AgentConfigForm environment selector", () => { expect(result.container.querySelector('[aria-label="Environment override"]')).toBeNull(); }); + it.each([null, "saved-computer"])("tests native Cursor Cloud without a Computer when the saved target is %s", async defaultEnvironmentId => { + mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true, enableManagedSandboxOnly: true }); + mockInstanceSettingsApi.get.mockResolvedValue({ defaultEnvironmentId: "instance-computer" }); + mockAdaptersApi.list.mockResolvedValue([{ type: "cursor_cloud", supportedRunners: ["paperclip", "legacy"], defaultRunner: "legacy" }]); + const result = await renderForm([], { + adapterType: "paperclip_runner", defaultEnvironmentId, + adapterConfig: { provider: "cursor_cloud", model: "claude-sonnet-4-5", repoUrl: "https://github.com/example/qualification" }, + }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + + expect(result.container.querySelector('[aria-label="Environment override"]')).toBeNull(); + await runTest(result.container); + expect(mockAgentsApi.testEnvironment).toHaveBeenCalledExactlyOnceWith("company-1", "paperclip_runner", expect.objectContaining({ environmentId: null, agentId: "agent-1" })); + expect(mockAgentsApi.qualifyRunner).not.toHaveBeenCalled(); + expect(mockEnvironmentsApi.probe).not.toHaveBeenCalled(); + }); + + it("clears a Computer inherited by a new native Cursor Cloud draft", async () => { + mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true, enableManagedSandboxOnly: true }); + mockAdaptersApi.list.mockResolvedValue([{ type: "cursor_cloud", supportedRunners: ["paperclip", "legacy"], defaultRunner: "legacy" }]); + const result = await renderCreateForm([], { + adapterType: "paperclip_runner", runner: "paperclip", defaultEnvironmentId: "previous-computer", + adapterSchemaValues: { provider: "cursor_cloud", model: "claude-sonnet-4-5" }, + }); + roots.push(result.root); + expect(result.onChange).toHaveBeenCalledWith({ defaultEnvironmentId: "" }); + expect(result.container.querySelector('[aria-label="Environment override"]')).toBeNull(); + }); + + it("tests the native default explicitly when creating Cursor Cloud with automatic runner selection", async () => { + mockInstanceSettingsApi.getExperimental.mockResolvedValue({ enableEnvironments: true, enableManagedSandboxOnly: true }); + mockAdaptersApi.list.mockResolvedValue([{ type: "cursor_cloud", supportedRunners: ["paperclip"], defaultRunner: "paperclip" }]); + const result = await renderCreateForm([], { + adapterType: "cursor_cloud", runner: "auto", model: "claude-sonnet-4-5", + adapterSchemaValues: { provider: "cursor_cloud", repoUrl: "https://github.com/example/qualification" }, + }, { showAdapterTestEnvironmentButton: true }); + roots.push(result.root); + await runTest(result.container); + expect(mockAgentsApi.testEnvironment).toHaveBeenCalledExactlyOnceWith("company-1", "cursor_cloud", expect.objectContaining({ + environmentId: null, runner: "paperclip", + })); + }); + it("requests the native Kimi model catalog while preserving an explicit saved model", async () => { mockAgentsApi.adapterModels.mockResolvedValue([]); const result = await renderForm([], { diff --git a/ui/src/components/AgentConfigForm.tsx b/ui/src/components/AgentConfigForm.tsx index d5f65b5d45e..1bb9a896ba4 100644 --- a/ui/src/components/AgentConfigForm.tsx +++ b/ui/src/components/AgentConfigForm.tsx @@ -659,6 +659,11 @@ export function AgentConfigForm(props: AgentConfigFormProps) { ? (runner === "paperclip" || (runner === "auto" && runnerAvailability?.defaultRunner === "paperclip") ? "paperclip_runner" : harnessType) : adapterType; const isCursorCloudRunner = harnessType === "cursor_cloud" && fieldsAdapterType === "paperclip_runner"; + useEffect(() => { + if (props.mode !== "create" || !isCursorCloudRunner || !props.values.defaultEnvironmentId) return; + props.onChange({ defaultEnvironmentId: "" }); + }, [props.mode, isCursorCloudRunner, isCreate ? props.values.defaultEnvironmentId : undefined, + isCreate ? props.onChange : undefined]); useEffect(() => { if (props.mode !== "create" || !isRunnerHarness) return; const resolvedRunner = runner === "auto" ? runnerAvailability?.defaultRunner : runner; @@ -992,10 +997,10 @@ export function AgentConfigForm(props: AgentConfigFormProps) { // Only applies in create mode and only once the K8s environment is loaded; if // none is available the UI surfaces a notice instead of silently selecting it. useEffect(() => { - if (!isCreate || !set || !forcedKubernetes || !kubernetesEnvironment) return; + if (!isCreate || !set || isCursorCloudRunner || !forcedKubernetes || !kubernetesEnvironment) return; if (currentDefaultEnvironmentId === kubernetesEnvironment.id) return; set({ defaultEnvironmentId: kubernetesEnvironment.id }); - }, [isCreate, set, forcedKubernetes, kubernetesEnvironment, currentDefaultEnvironmentId]); + }, [isCreate, set, isCursorCloudRunner, forcedKubernetes, kubernetesEnvironment, currentDefaultEnvironmentId]); const runnableEnvironments = useMemo( () => environments.filter((environment) => { @@ -1172,7 +1177,7 @@ export function AgentConfigForm(props: AgentConfigFormProps) { let settings = instanceSettings; let environmentList = environments; let managedSandboxOnly = experimentalSettings?.enableManagedSandboxOnly === true; - if (!rawCurrentDefaultEnvironmentId) { + if (!isCursorCloudRunner && !rawCurrentDefaultEnvironmentId) { // The agent has no own environment, so the Test resolves the instance // default, the local default, or the managed sandbox. Resolve the // settings, the environment list, and the managed-sandbox-only policy @@ -1210,7 +1215,7 @@ export function AgentConfigForm(props: AgentConfigFormProps) { // policy is on, probe the managed sandbox the real run uses instead. The // resolver throws when no managed sandbox is available, which the mutation // surfaces as a fail-closed error rather than a local host probe. - const environmentId = resolveAdapterTestEnvironmentId({ + const environmentId = isCursorCloudRunner ? null : resolveAdapterTestEnvironmentId({ agentDefaultEnvironmentId: rawCurrentDefaultEnvironmentId || null, instanceDefaultEnvironmentId: settings?.defaultEnvironmentId ?? null, localDefaultEnvironmentId: resolveLocalDefaultEnvironmentId(environmentList), @@ -1227,6 +1232,9 @@ export function AgentConfigForm(props: AgentConfigFormProps) { const aiConnection = isCreate ? undefined : aiRuntimeConnectionBindingSchema.safeParse( (overlay.runtime.runtimeConfig as Record | undefined)?.aiConnection ?? props.agent.runtimeConfig.aiConnection, ).data; + const testRunnerSelection = isCursorCloudRunner + ? { runner: "paperclip" as const } + : isRunnerHarness && (isCreate ? runner !== "auto" : overlay.runner !== undefined) ? { runner } : {}; if (!isCreate && props.agent.adapterType === "paperclip_runner" && adapterType === "paperclip_runner" && runnerAvailability?.defaultRunner !== "paperclip" && !isExternalRunner && aiConnection?.mode !== "router" && !isDirty && environmentId && canQualifyRunnerEnvironment(environmentList.find(environment => environment.id === environmentId))) { @@ -1246,9 +1254,9 @@ export function AgentConfigForm(props: AgentConfigFormProps) { : adapterConfig.provider === "acpx" && adapterConfig.acpxAgent === "claude" ? "claude_local" : adapterType : adapterType; - return testAgentSetup({ companyId: selectedCompanyId, adapterType, providerAdapter, adapterConfig, agentId, aiConnection, environmentId, ...(isRunnerHarness && (isCreate ? runner !== "auto" : overlay.runner !== undefined) ? { runner } : {}) }); + return testAgentSetup({ companyId: selectedCompanyId, adapterType, providerAdapter, adapterConfig, agentId, aiConnection, environmentId, ...testRunnerSelection }); } - return agentsApi.testEnvironment(selectedCompanyId, adapterType, { adapterConfig, agentId, aiConnection, environmentId, ...(isRunnerHarness && (isCreate ? runner !== "auto" : overlay.runner !== undefined) ? { runner } : {}) }); + return agentsApi.testEnvironment(selectedCompanyId, adapterType, { adapterConfig, agentId, aiConnection, environmentId, ...testRunnerSelection }); }, }); const [testActionPending, setTestActionPending] = useState(false); @@ -1718,7 +1726,7 @@ export function AgentConfigForm(props: AgentConfigFormProps) { )} {/* ---- Execution ---- */} - {forcedKubernetes ? ( + {isCursorCloudRunner ? null : forcedKubernetes ? ( // Instance execution policy forces the managed Kubernetes sandbox // (executionMode=kubernetes): never offer local / non-Kubernetes targets. // Render the environment read-only instead of the selectable picker. diff --git a/ui/src/components/NewAgentDialog.test.tsx b/ui/src/components/NewAgentDialog.test.tsx index e3c5a89ca09..df5d4c3cb5f 100644 --- a/ui/src/components/NewAgentDialog.test.tsx +++ b/ui/src/components/NewAgentDialog.test.tsx @@ -218,6 +218,54 @@ it("offers one Codex harness without a runner/provider tile", async () => { expect(document.querySelectorAll('input[value="codex_local"]')).toHaveLength(1); expect(document.querySelector("select")).toBeNull(); }); +it("opens native Cursor Cloud from the ordinary hosted adapter picker", async () => { + await act(async () => { + cache.setQueryData(queryKeys.health, { status: "ok", cloud: { managed: true } }); + cache.setQueryData(queryKeys.adapters.availability("company-1", null), [...state.adapters, + { type: "cursor_cloud", loaded: true, supportedRunners: ["paperclip"], defaultRunner: "paperclip" }]); + }); + await name(); + const cursor = document.querySelector('input[value="cursor_cloud"]'); + expect(cursor).not.toBeNull(); + await act(async () => cursor!.click()); + await click("Configure agent"); + expect(agents.update).toHaveBeenCalledWith(agents.create.mock.calls[0][1].setupId, expect.objectContaining({ + adapterType: "paperclip_runner", runner: "paperclip", adapterConfig: { provider: "cursor_cloud" }, + }), "company-1"); + const query = new URL(state.navigate.mock.calls[0][0], "http://local").searchParams; + expect(query.get("adapterType")).toBe("paperclip_runner"); + expect(query.get("runnerProvider")).toBe("cursor_cloud"); + expect(query.get("runner")).toBe("paperclip"); +}); + +it("hides explicitly unsupported harnesses while keeping the qualified Cursor API card", async () => { + await act(async () => { + cache.setQueryData(queryKeys.health, { status: "ok", cloud: { managed: true } }); + cache.setQueryData(queryKeys.adapters.availability("company-1", null), [ + { type: "codex_local", loaded: true, supportedRunners: [] }, + { type: "opencode_local", loaded: true, supportedRunners: [] }, + { type: "paperclip_runner", loaded: true, supportedRunners: [] }, + { type: "cursor_cloud", loaded: true, supportedRunners: ["paperclip"], defaultRunner: "paperclip" }, + ]); + }); + await name(); + expect(document.querySelector('input[value="codex_local"]')).toBeNull(); + expect(document.querySelector('input[value="opencode_local"]')).toBeNull(); + expect(document.querySelector('input[value="copilot_runtime"]')).toBeNull(); + expect(document.querySelector('input[value="antigravity_runtime"]')).toBeNull(); + expect(document.querySelector('input[value="cursor_cloud"]')).not.toBeNull(); +}); + +it.each(["gate off", "native unavailable"])("keeps hosted Cursor Cloud hidden when %s", async reason => { + await act(async () => { + cache.setQueryData(queryKeys.health, { status: "ok", cloud: { managed: true } }); + if (reason === "gate off") cache.setQueryData(queryKeys.instance.experimentalSettings, { enableNativeRunner: false }); + cache.setQueryData(queryKeys.adapters.availability("company-1", null), [...state.adapters, + { type: "cursor_cloud", loaded: true, supportedRunners: reason === "native unavailable" ? ["legacy"] : ["paperclip"], defaultRunner: "legacy" }]); + }); + await name(); + expect(document.querySelector('input[value="cursor_cloud"]')).toBeNull(); +}); it.each([true, false, undefined])("opens saved-token Copilot setup without an experimental override (%s)", async enableNativeRunner => { await act(async () => cache.setQueryData(queryKeys.instance.experimentalSettings, { enableNativeRunner })); await name(); diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index 4cffcc90ec3..b628c4e224f 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -23,6 +23,8 @@ import { pendingConnectionIntentInteraction } from "@/fixtures/issueThreadIntera import { TaskChatScrollNavigation, useTaskChatScrollNavigation } from "./task-chat/scroll-navigation"; import { CoordinatorPresentation } from "./coordinator/CoordinatorPresentation"; import type { HeartbeatRunEvent } from "@paperclipai/shared"; +import type { HeartbeatRun } from "@paperclipai/shared"; +import { waitForStoppedRuns } from "@/lib/wait-for-stopped-runs"; const transcriptState = vi.hoisted(() => ({ transcriptByRun: new Map(), @@ -5221,6 +5223,36 @@ describe("TaskChatThread live transcript", () => { }); describe("TaskChatThread composer execution controls", () => { + it("reconciles a timed-out Stop from exact company-bound historical proof after its live action disappears", async () => { + vi.useFakeTimers(); + try { + const run = { id: "stopped-run", status: "running", runtimeMode: "native" as const, invocationSource: "issue", triggerDetail: null, + startedAt: "2026-11-03T15:00:00Z", finishedAt: null, createdAt: "2026-11-03T15:00:00Z", agentId: "agent-1", agentName: "Alex", adapterType: "paperclip_runner" }; + const requested: Partial & { runtimeMode: "native" } = { + id: run.id, companyId: "company", status: "cancelled", runtimeMode: "native", + resultJson: { executionCancellation: { state: "requested" } }, + }; + const onStop = () => waitForStoppedRuns([run.id], { timeoutMs: 1000, getRun: async () => requested as HeartbeatRun }); + const base = { comments: [], onAdd: async () => {}, issueStatus: "in_progress", companyId: "company" }; + render(); + await act(async () => { container.querySelector('[data-testid="task-chat-composer-stop"]')!.click(); }); + await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); + expect(container.querySelector('[role="alert"]')?.textContent).toContain("stop was requested"); + const historical = { runId: run.id, agentId: run.agentId, adapterType: run.adapterType, createdAt: run.createdAt, + startedAt: run.startedAt, finishedAt: run.startedAt, status: "cancelled", runtimeMode: "native" as const, + resultJson: { executionCancellation: { state: "acknowledged" }, nativeCancellation: { + schema: "paperclip.native-cancellation.v1", runId: run.id, companyId: "company", dispatchState: "acknowledged" } } }; + render(); + await act(async () => {}); expect(container.querySelector('[role="alert"]')).not.toBeNull(); + render(); + await act(async () => {}); expect(container.querySelector('[role="alert"]')).not.toBeNull(); + render(); + await act(async () => {}); expect(container.querySelector('[role="alert"]')).not.toBeNull(); + render(); + await act(async () => {}); expect(container.querySelector('[role="alert"]')).toBeNull(); + } finally { vi.useRealTimers(); } + }); it.each(["process", "paperclip_runner"])("passes the task's stop action through for %s execution", async (adapterType) => { const onStop = vi.fn(async () => {}); const run = { diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index 5a271aa9c01..5c8b4682177 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -143,6 +143,7 @@ import { issuesApi } from "@/api/issues"; import { queryKeys } from "@/lib/queryKeys"; import { useQueryClient } from "@tanstack/react-query"; import { TaskChatPresentationProvider } from "@/components/task-chat/presentation-mode"; +import { isRunStopConfirmed } from "@/lib/wait-for-stopped-runs"; const EMPTY_BROWSERS: TaskBrowser[] = []; const COMPACT_RUN_LOG_READ_LIMIT_BYTES = 32_000; @@ -587,6 +588,11 @@ export function TaskChatThread(props: TaskChatThreadProps) { resumeAssigneePending = false, onOpenSkill, } = props; + const confirmedStoppedRunIds = useMemo(() => new Set((linkedRuns ?? []).filter((run) => + companyId && (run.runtimeMode === "legacy" || run.runtimeMode === "native") && isRunStopConfirmed({ + id: run.runId, companyId, status: run.status, runtimeMode: run.runtimeMode, resultJson: run.resultJson ?? null, + }), + ).map((run) => run.runId)), [companyId, linkedRuns]); const retryFailedRunHandler = isTerminalIssueStatus(issueStatus) || interactions?.some((interaction) => interaction.status === "pending") || @@ -3485,6 +3491,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { !("clientStatus" in comment && comment.clientStatus) ).map((comment) => comment.clientRequestId!))} onStop={liveRun ? onCancelRun : undefined} + confirmedStoppedRunIds={confirmedStoppedRunIds} stopPending={stopPending} stopScope={stopScope} workMode={issueWorkMode} diff --git a/ui/src/components/new-agent/AgentBasicsDialog.tsx b/ui/src/components/new-agent/AgentBasicsDialog.tsx index fabfce6ad1c..8c5554496f8 100644 --- a/ui/src/components/new-agent/AgentBasicsDialog.tsx +++ b/ui/src/components/new-agent/AgentBasicsDialog.tsx @@ -89,7 +89,7 @@ export function AgentBasicsDialog({ queryFn: () => adaptersApi.list({ companyId: companyId! }), enabled: open && Boolean(companyId), }); - const nativeRunner = adapters?.find(adapter => adapter.type === "paperclip_runner" && adapter.loaded && !adapter.disabled); + const nativeRunner = adapters?.find(adapter => adapter.type === "paperclip_runner" && adapter.loaded && !adapter.disabled && adapter.supportedRunners?.length !== 0); const copilotAvailable = Boolean(nativeRunner) && PAPERCLIP_RUNNER_ACPX_PROFILES.some(profile => profile.value === "copilot" && profile.qualified); const kimiAvailable = Boolean(nativeRunner) && experimental.data?.enableNativeRunner === true && PAPERCLIP_RUNNER_ACPX_PROFILES.some(profile => profile.value === "kimi" && profile.qualified); @@ -100,18 +100,20 @@ export function AgentBasicsDialog({ (adapter) => adapter.loaded && !adapter.disabled && + adapter.supportedRunners?.length !== 0 && adapter.type !== "paperclip_runner" && isNewAgentAdapterAllowed(adapter.type, { cloud, nativeRunnerEnabled: experimental.data?.enableNativeRunner === true, }) && + (!cloud || adapter.type !== "cursor_cloud" || adapter.supportedRunners?.includes("paperclip")) && !["process", "http"].includes(adapter.type) && !getAdapterDisplay(adapter.type).comingSoon, ), ...(copilotAvailable ? [{ ...nativeRunner!, type: "copilot_runtime" }] : []), ...(antigravityAvailable ? [{ ...nativeRunner!, type: "antigravity_runtime" }] : []), ...(kimiAvailable ? [{ ...nativeRunner!, type: "kimi_runtime" }] : [])]; const experimentalRunnerAvailable = experimental.data?.enableNativeRunner === true - && adapters?.some(adapter => adapter.type === "paperclip_runner" && adapter.loaded && !adapter.disabled); + && adapters?.some(adapter => adapter.type === "paperclip_runner" && adapter.loaded && !adapter.disabled && adapter.supportedRunners?.length !== 0); const validAdapter = choices.some((adapter) => adapter.type === adapterType) || (experimentalRunnerAvailable && adapterType === "paperclip_runner" && (runnerProvider !== "antigravity" || antigravityQualified)); async function submit() { @@ -142,8 +144,10 @@ export function AgentBasicsDialog({ if (active.current) setName(agent.name); } saved.current = agent; - const nextAdapter = ["openai_dot", "copilot_runtime", "antigravity_runtime", "kimi_runtime"].includes(adapterType) ? "paperclip_runner" : adapterType; - const nextProvider = adapterType === "kimi_runtime" ? "kimi" : adapterType === "antigravity_runtime" ? "antigravity" : adapterType === "copilot_runtime" ? "copilot" : adapterType === "openai_dot" ? "openai_dot" : adapterType === "paperclip_runner" ? runnerProvider : "codex"; + const hostedCursor = cloud && adapterType === "cursor_cloud"; + const nextAdapter = hostedCursor || ["openai_dot", "copilot_runtime", "antigravity_runtime", "kimi_runtime"].includes(adapterType) ? "paperclip_runner" : adapterType; + const nextProvider = hostedCursor ? "cursor_cloud" : adapterType === "kimi_runtime" ? "kimi" : adapterType === "antigravity_runtime" ? "antigravity" : adapterType === "copilot_runtime" ? "copilot" : adapterType === "openai_dot" ? "openai_dot" : adapterType === "paperclip_runner" ? runnerProvider : "codex"; + const nextRunner = hostedCursor ? "paperclip" as const : "auto" as const; // Pending providers can show setup while the saved name-only draft stays held. const pendingAntigravity = nextProvider === "antigravity" && !antigravityQualified; if (step === "adapter" || (!creatingRecord && agent.name !== name.trim())) { @@ -155,7 +159,7 @@ export function AgentBasicsDialog({ setupAdapterChoice: { adapterType: nextAdapter, runnerProvider: nextProvider, runner: "auto" } }, } : {}), ...(step === "adapter" && !pendingAntigravity ? { - adapterType: nextAdapter, runner: "auto" as const, + adapterType: nextAdapter, runner: nextRunner, adapterConfig: nextAdapter === "paperclip_runner" ? { provider: ["claude", "grok", "copilot", "antigravity", "kimi"].includes(nextProvider) ? "acpx" : nextProvider, ...(["claude", "grok", "copilot", "antigravity", "kimi"].includes(nextProvider) ? { acpxAgent: nextProvider } : {}), @@ -171,7 +175,7 @@ export function AgentBasicsDialog({ if (!active.current) return; if (step === "name") setStep("adapter"); else onContinue({ name: agent.name, adapterType: nextAdapter, runnerProvider: nextProvider, - runner: "auto", setupAgentId: agent.id }); + runner: nextRunner, setupAgentId: agent.id }); } catch (cause) { if (active.current) setSaveError(cause instanceof Error ? cause.message : "Could not save agent setup."); } finally { diff --git a/ui/src/components/new-agent/NewAgentSetup.tsx b/ui/src/components/new-agent/NewAgentSetup.tsx index e9c319587fb..0defa9a61b9 100644 --- a/ui/src/components/new-agent/NewAgentSetup.tsx +++ b/ui/src/components/new-agent/NewAgentSetup.tsx @@ -315,7 +315,7 @@ function Setup({ let environmentId: string | null = null; let environmentError: string | null = null; try { - environmentId = forced.forced + environmentId = nativeCursorCloud ? null : forced.forced ? (forced.kubernetesEnvironment?.id ?? null) : resolveAdapterTestEnvironmentId({ agentDefaultEnvironmentId: environmentOverride || null, @@ -414,6 +414,7 @@ function Setup({ const cloud = Boolean(useCloudInstance()); const available = (!nativeCursorCloud || experimental.data?.enableNativeRunner === true) && + (!cloud || adapterType !== "cursor_cloud" || nativeCursorCloud) && isNewAgentAdapterAllowed(adapterType, { cloud, nativeRunnerEnabled: experimental.data?.enableNativeRunner === true, @@ -447,12 +448,13 @@ function Setup({ !general.error && !agents.error && !caps.error && - (!(managedOnly || forced.forced) || environmentId), + (nativeCursorCloud || !(managedOnly || forced.forced) || environmentId), ); const needsCurrentSetup = ["codex_local", "claude_local", "opencode_local", "pi_local"].includes(adapterType); const preparingComputer = created?.pauseReason === "setup" && ["preparing", "pausing"].includes(created.lifecycleState ?? ""); - const environmentSelectionSaved = (!environmentOverride && created?.defaultEnvironmentId == null) - || (created?.defaultEnvironmentId ?? null) === (environmentOverride || (forced.forced || managedOnly ? environmentId : null)); + const environmentSelectionSaved = nativeCursorCloud + || (!environmentOverride && created?.defaultEnvironmentId == null) + || (created?.defaultEnvironmentId ?? null) === (environmentOverride || (forced.forced || managedOnly ? environmentId : null)); const ready = (pendingAntigravity || Boolean(created) && environmentSelectionSaved) && !environmentSaving && !preparingComputer && connectionSettingsReady && (!needsCurrentSetup || ( envs.isSuccess && !envs.isFetching && settings.isSuccess && !settings.isFetching @@ -614,7 +616,7 @@ function Setup({ provider: ["claude", "grok", "copilot", "pi", "antigravity", "kimi"].includes(runnerProvider) ? "acpx" : runnerProvider, ...(["claude", "grok", "copilot", "pi", "antigravity", "kimi"].includes(runnerProvider) ? { acpxAgent: runnerProvider } : {}), ...(antigravityCandidate ? { model: antigravityCandidate.model } : {}), - } : {}, defaultEnvironmentId: environmentOverride || (forced.forced || managedOnly ? environmentId : null), + } : {}, defaultEnvironmentId: nativeCursorCloud ? null : environmentOverride || (forced.forced || managedOnly ? environmentId : null), runtimeConfig: buildNewAgentRuntimeConfig({ heartbeatEnabled: false }), }).then(async agent => { await cache.cancelQueries({ queryKey: queryKeys.agents.detail(agent.id) }); @@ -677,7 +679,7 @@ function Setup({ submitted = true; const agent = await agentsApi.update(created.id, { adapterType: requestAdapterType, runner, adapterConfig: config, - defaultEnvironmentId: selectedEnvironment || (forced.forced || managedOnly ? environmentId : null), + defaultEnvironmentId: nativeCursorCloud ? null : selectedEnvironment || (forced.forced || managedOnly ? environmentId : null), runtimeConfig: { ...created.runtimeConfig, ...(binding ? { aiConnection: binding } : {}) }, ...(antigravityCandidate ? { metadata: { ...created.metadata, setupAdapterPending: false, setupAdapterChoice: undefined } } : {}), ...(nextConnection?.applyStoredClaudeLogin ? { applyStoredClaudeLogin: true } : {}), @@ -805,7 +807,7 @@ function Setup({ adapterType: requestAdapterType, ...(["codex_local", "claude_local", "opencode_local", "pi_local", "cursor_cloud"].includes(adapterType) && runner !== "auto" ? { runner } : {}), adapterConfig: config, - defaultEnvironmentId: + defaultEnvironmentId: nativeCursorCloud ? null : environmentOverride || (forced.forced || managedOnly ? environmentId : null), runtimeConfig: { ...buildNewAgentRuntimeConfig({ heartbeatEnabled: false }), ...(aiBinding ? { aiConnection: aiBinding } : {}) }, @@ -995,7 +997,7 @@ function Setup({ This adapter is unavailable. Choose an enabled adapter.

)} - {(managedOnly || forced.forced) && + {!nativeCursorCloud && (managedOnly || forced.forced) && !envs.isPending && !environmentId && (

@@ -1444,7 +1446,7 @@ function Setup({ )} {adapterType === "cursor_cloud" && ( <> - {nativeCursorCloud &&

Experimental. Requires a local Paperclip computer. Cursor manages cloud command permissions; the effective model is not acknowledged.

} + {nativeCursorCloud &&

Experimental. Cursor hosts this agent’s execution environment. A Paperclip computer is not required. Cursor manages cloud command permissions; the effective model is not acknowledged.

}
{ setRunner(value); resetTest(); }} /> } - {adapterType !== "hermes_gateway" && (adapterType !== "cursor_cloud" || nativeCursorCloud) && ( + {adapterType !== "hermes_gateway" && adapterType !== "cursor_cloud" && (

Environment